openstack
/
tripleo-common
Code Issues Proposed changes

add caching the GetParametersAction 

This caching in swift changes the time for the GetParametersAction from
15s to 3s

Added tests for plan.UpdatePlanAction as there were none there.

Change-Id: I0b543e082a3b02a35e3e979339698ae237b32ce4
Closes-Bug: #1647301
This commit is contained in:
Adriano Petrich 2017-03-10 11:06:32 +00:00
parent b69b950a07
commit 78807dff26
11 changed files with 287 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
tripleo_common/actions/base.py
View File

@ -12,6 +12,9 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import json
import zlib
from glanceclient.v2 import client as glanceclient
from heatclient.v1 import client as heatclient
import ironic_inspector_client
@ -22,6 +25,8 @@ from mistral.utils.openstack import keystone as keystone_utils
from mistralclient.api import client as mistral_client
from novaclient.client import Client as nova_client
from swiftclient import client as swift_client
from swiftclient import exceptions as swiftexceptions
from tripleo_common import constants
class TripleOAction(base.Action):
@ -137,3 +142,63 @@ class TripleOAction(base.Action):
)
return client
def _cache_key(self, plan_name, key_name):
return "__cache_{}_{}".format(plan_name, key_name)
def cache_get(self, plan_name, key):
"""Retrieves the stored objects
Returns None if there are any issues or no objects found
"""
swift_client = self.get_object_client()
try:
headers, body = swift_client.get_object(
constants.TRIPLEO_CACHE_CONTAINER,
self._cache_key(plan_name, key)
)
result = json.loads(zlib.decompress(body).decode())
return result
except swiftexceptions.ClientException:
# cache does not exist, ignore
pass
except ValueError:
# the stored json is invalid. Deleting
self.cache_delete(plan_name, key)
return
def cache_set(self, plan_name, key, contents):
"""Stores an object
Allows the storage of jsonable objects except for None
Storing None equals to a cache delete.
"""
swift_client = self.get_object_client()
if contents is None:
self.cache_delete(plan_name, key)
return
try:
swift_client.head_container(constants.TRIPLEO_CACHE_CONTAINER)
except swiftexceptions.ClientException:
swift_client.put_container(constants.TRIPLEO_CACHE_CONTAINER)
swift_client.put_object(
constants.TRIPLEO_CACHE_CONTAINER,
self._cache_key(plan_name, key),
zlib.compress(json.dumps(contents).encode()))
def cache_delete(self, plan_name, key):
swift_client = self.get_object_client()
try:
swift_client.delete_object(
constants.TRIPLEO_CACHE_CONTAINER,
self._cache_key(plan_name, key)
)
except swiftexceptions.ClientException:
# cache or container does not exist. Ignore
pass

2
tripleo_common/actions/heat_capabilities.py
View File

@ -192,6 +192,8 @@ class UpdateCapabilitiesAction(base.TripleOAction):
if env.get('path') not in self.environments:
mistral_env.variables['environments'].remove(env)
self.cache_delete(self.container, "tripleo.parameters.get")
env_kwargs = {
'name': mistral_env.name,
'variables': mistral_env.variables

13
tripleo_common/actions/parameters.py
View File

@ -45,6 +45,12 @@ class GetParametersAction(templates.ProcessTemplatesAction):
"""Gets list of available heat parameters."""
def run(self):
cached = self.cache_get(self.container, "tripleo.parameters.get")
if cached is not None:
return cached
processed_data = super(GetParametersAction, self).run()
# If we receive a 'Result' instance it is because the parent action
@ -67,10 +73,12 @@ class GetParametersAction(templates.ProcessTemplatesAction):
'environment': processed_data['environment'],
'show_nested': True
}
return {
result = {
'heat_resource_tree': orc.stacks.validate(**fields),
'mistral_environment_parameters': params,
}
self.cache_set(self.container, "tripleo.parameters.get", result)
return result
class ResetParametersAction(base.TripleOAction):
@ -92,6 +100,7 @@ class ResetParametersAction(base.TripleOAction):
'variables': wf_env.variables
}
wc.environments.update(**env_kwargs)
self.cache_delete(self.container, "tripleo.parameters.get")
return wf_env
@ -115,6 +124,7 @@ class UpdateParametersAction(base.TripleOAction):
'variables': wf_env.variables
}
wc.environments.update(**env_kwargs)
self.cache_delete(self.container, "tripleo.parameters.get")
return wf_env
@ -181,6 +191,7 @@ class GeneratePasswordsAction(base.TripleOAction):
}
wc.environments.update(**env_kwargs)
self.cache_delete(self.container, "tripleo.parameters.get")
return wf_env.variables['passwords']

1
tripleo_common/actions/plan.py
View File

@ -215,6 +215,7 @@ class UpdatePlanAction(base.TripleOAction, PlanEnvMixin):
# Update mistral environment with contents from plan environment file
variables = json.dumps(plan_env_dict, sort_keys=True)
self.cache_delete(self.container, "tripleo.parameters.get")
try:
mistral.environments.update(
name=self.container, variables=variables)

1
tripleo_common/actions/templates.py
View File

@ -125,6 +125,7 @@ class ProcessTemplatesAction(base.TripleOAction):
try:
# write the template back to the plan container
LOG.info("Writing rendered template %s" % yaml_f)
self.cache_delete(self.container, "tripleo.parameters.get")
swift.put_object(
self.container, yaml_f, r_template)
except swiftexceptions.ClientException as ex:

3
tripleo_common/constants.py
View File

@ -116,3 +116,6 @@ PLAN_ENVIRONMENT = 'plan-environment.yaml'
DEFAULT_DEPLOY_KERNEL_NAME = 'bm-deploy-kernel'
DEFAULT_DEPLOY_RAMDISK_NAME = 'bm-deploy-ramdisk'
# The name for the swift container to host the cache for tripleo
TRIPLEO_CACHE_CONTAINER = "__cache__"

104
tripleo_common/tests/actions/test_base.py
View File

@ -12,6 +12,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import zlib
import mock
from ironicclient.v1 import client as ironicclient
@ -21,6 +23,8 @@ from mistral.utils.openstack import keystone as keystone_utils
from tripleo_common.actions import base
from tripleo_common.tests import base as tests_base
from swiftclient.exceptions import ClientException
@mock.patch.object(context, 'ctx')
@mock.patch.object(keystone_utils, 'get_endpoint_for_project')
@ -40,3 +44,103 @@ class TestActionsBase(tests_base.TestCase):
region_name='ironic-region', retry_interval=5, token=mock.ANY)
mock_endpoint.assert_called_once_with('ironic')
mock_cxt.assert_called_once_with()
def test_cache_key(self, mock_client, mock_endpoint):
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
self.assertEqual(
self.action._cache_key(container, key),
cache_key
)
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_set(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
compressed_json = zlib.compress("{\"foo\": 1}".encode())
self.action.cache_set(container, key, {"foo": 1})
mock_swift.put_object.assert_called_once_with(
cache_container,
cache_key,
compressed_json
)
mock_swift.delete_object.assert_not_called()
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_set_none(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
self.action.cache_set(container, key, None)
mock_swift.put_object.assert_not_called()
mock_swift.delete_object.called_once_with(
cache_container,
cache_key
)
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_get_filled(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
container = "TestContainer"
key = "testkey"
compressed_json = zlib.compress("{\"foo\": 1}".encode())
# test if cache has something in it
mock_swift.get_object.return_value = ([], compressed_json)
result = self.action.cache_get(container, key)
self.assertEqual(result, {"foo": 1})
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_empty(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
mock_swift.get_object.side_effect = ClientException(
"Foo"
)
result = self.action.cache_get(container, key)
self.assertFalse(result)
# delete cache if we have a value
self.action.cache_delete(container, key)
mock_swift.delete_object.assert_called_once_with(
cache_container,
cache_key
)
@mock.patch("tripleo_common.actions.base.swift_client.Connection")
def test_cache_delete(self, mock_conn, mock_client, mock_endpoint):
mock_swift = mock.Mock()
mock_conn.return_value = mock_swift
cache_container = "__cache__"
container = "TestContainer"
key = "testkey"
cache_key = "__cache_TestContainer_testkey"
mock_swift.delete_object.side_effect = ClientException(
"Foo"
)
self.action.cache_delete(container, key)
mock_swift.delete_object.assert_called_once_with(
cache_container,
cache_key
)

17
tripleo_common/tests/actions/test_heat_capabilities.py
View File

@ -236,9 +236,11 @@ class UpdateCapabilitiesActionTest(base.TestCase):
super(UpdateCapabilitiesActionTest, self).setUp()
self.container_name = 'test-container'
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_workflow_client')
def test_run(self, get_workflow_client_mock):
def test_run(self, get_workflow_client_mock, mock_cache):
# setup mistral
mistral = mock.MagicMock()
@ -267,9 +269,16 @@ class UpdateCapabilitiesActionTest(base.TestCase):
]},
action.run())
mock_cache.assert_called_once_with(
self.container_name,
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_workflow_client')
def test_run_purge_missing(self, get_workflow_client_mock):
def test_run_purge_missing(self, get_workflow_client_mock, mock_cache):
# setup mistral
mistral = mock.MagicMock()
@ -297,6 +306,10 @@ class UpdateCapabilitiesActionTest(base.TestCase):
{'path': '/path/to/poc-custom-env.yaml'}
]},
action.run())
mock_cache.assert_called_once_with(
self.container_name,
"tripleo.parameters.get"
)
@mock.patch(
'tripleo_common.actions.base.TripleOAction.get_object_client')

71
tripleo_common/tests/actions/test_parameters.py
View File

@ -137,6 +137,10 @@ _EXISTING_PASSWORDS = {
class GetParametersActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_set')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_get')
@mock.patch('heatclient.common.template_utils.'
'process_multiple_environments_and_files')
@mock.patch('heatclient.common.template_utils.get_template_contents')
@ -149,7 +153,9 @@ class GetParametersActionTest(base.TestCase):
def test_run(self, mock_ctx, mock_get_object_client,
mock_get_workflow_client, mock_get_orchestration_client,
mock_get_template_contents,
mock_process_multiple_environments_and_files):
mock_process_multiple_environments_and_files,
mock_cache_get,
mock_cache_set):
mock_ctx.return_value = mock.MagicMock()
swift = mock.MagicMock(url="http://test.com")
@ -173,8 +179,10 @@ class GetParametersActionTest(base.TestCase):
mock_process_multiple_environments_and_files.return_value = ({}, {})
mock_heat = mock.MagicMock()
mock_heat.stacks.validate.return_value = {}
mock_get_orchestration_client.return_value = mock_heat
mock_cache_get.return_value = None
# Test
action = parameters.GetParametersAction()
action.run()
@ -184,14 +192,26 @@ class GetParametersActionTest(base.TestCase):
show_nested=True,
template={'heat_template_version': '2016-04-30'},
)
mock_cache_get.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
mock_cache_set.assert_called_once_with(
"overcloud",
"tripleo.parameters.get",
{'heat_resource_tree': {}, 'mistral_environment_parameters': None}
)
class ResetParametersActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_workflow_client')
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_workflow_client):
def test_run(self, mock_ctx, mock_get_workflow_client,
mock_cache):
mock_ctx.return_value = mock.MagicMock()
mock_mistral = mock.MagicMock()
@ -204,7 +224,6 @@ class ResetParametersActionTest(base.TestCase):
}
mock_mistral.environments.get.return_value = mock_env
mock_get_workflow_client.return_value = mock_mistral
# Test
action = parameters.ResetParametersAction()
action.run()
@ -215,14 +234,20 @@ class ResetParametersActionTest(base.TestCase):
'environments': [{u'path': u'environments/test.yaml'}],
}
)
mock_cache.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
class UpdateParametersActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_workflow_client')
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_workflow_client):
def test_run(self, mock_ctx, mock_get_workflow_client, mock_cache):
mock_ctx.return_value = mock.MagicMock()
mock_mistral = mock.MagicMock()
@ -249,10 +274,16 @@ class UpdateParametersActionTest(base.TestCase):
'environments': [{u'path': u'environments/test.yaml'}],
'parameter_defaults': {'SomeTestParameter': 42}}
)
mock_cache.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
class UpdateRoleParametersActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.utils.parameters.set_count_and_flavor_params')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_baremetal_client')
@ -263,7 +294,7 @@ class UpdateRoleParametersActionTest(base.TestCase):
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_workflow_client,
mock_get_compute_client, mock_get_baremetal_client,
mock_set_count_and_flavor):
mock_set_count_and_flavor, mock_cache):
mock_ctx.return_value = mock.MagicMock()
mock_mistral = mock.MagicMock()
@ -283,10 +314,16 @@ class UpdateRoleParametersActionTest(base.TestCase):
mock_mistral.environments.update.assert_called_once_with(
name='overcast', variables={'parameter_defaults': params})
mock_cache.assert_called_once_with(
"overcast",
"tripleo.parameters.get"
)
class GeneratePasswordsActionTest(base.TestCase):
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('tripleo_common.utils.passwords.'
@ -296,7 +333,7 @@ class GeneratePasswordsActionTest(base.TestCase):
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_workflow_client,
mock_get_snmpd_readonly_user_password,
mock_get_orchestration_client):
mock_get_orchestration_client, mock_cache):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
@ -324,7 +361,13 @@ class GeneratePasswordsActionTest(base.TestCase):
for password_param_name in constants.PASSWORD_PARAMETER_NAMES:
self.assertTrue(password_param_name in result,
"%s is not in %s" % (password_param_name, result))
mock_cache.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('tripleo_common.utils.passwords.'
@ -337,7 +380,8 @@ class GeneratePasswordsActionTest(base.TestCase):
def test_run_passwords_exist(self, mock_ctx, mock_get_workflow_client,
mock_get_snmpd_readonly_user_password,
mock_create_ssh_keypair,
mock_get_orchestration_client):
mock_get_orchestration_client,
mock_cache):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -367,7 +411,13 @@ class GeneratePasswordsActionTest(base.TestCase):
# ensure old passwords used and no new generation
self.assertEqual(_EXISTING_PASSWORDS, result)
mock_cache.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('tripleo_common.utils.passwords.'
@ -380,7 +430,8 @@ class GeneratePasswordsActionTest(base.TestCase):
def test_passwords_exist_in_heat(self, mock_ctx, mock_get_workflow_client,
mock_get_snmpd_readonly_user_password,
mock_create_ssh_keypair,
mock_get_orchestration_client):
mock_get_orchestration_client,
mock_cache):
mock_get_snmpd_readonly_user_password.return_value = "TestPassword"
mock_create_ssh_keypair.return_value = {'public_key': 'Foo',
@ -416,6 +467,10 @@ class GeneratePasswordsActionTest(base.TestCase):
existing_passwords["AdminPassword"] = "ExistingPasswordInHeat"
# ensure old passwords used and no new generation
self.assertEqual(existing_passwords, result)
mock_cache.assert_called_once_with(
"overcloud",
"tripleo.parameters.get"
)
class GetPasswordsActionTest(base.TestCase):

14
tripleo_common/tests/actions/test_plan.py
View File

@ -259,7 +259,8 @@ class UpdatePlanActionTest(base.TestCase):
mistral_patcher.start()
self.addCleanup(mistral_patcher.stop)
def test_run_success(self):
@mock.patch('tripleo_common.actions.base.TripleOAction.cache_delete')
def test_run_success(self, mock_cache):
action = plan.UpdatePlanAction(self.container_name)
action.run()
@ -269,13 +270,18 @@ class UpdatePlanActionTest(base.TestCase):
)
self.swift.delete_object.assert_called_once()
mock_cache.assert_called_once_with(
"Test-container-3",
"tripleo.parameters.get"
)
self.mistral.environments.update.assert_called_once_with(
name='Test-container-3',
variables=JSON_CONTENTS
)
def test_run_mistral_env_missing(self):
@mock.patch('tripleo_common.actions.base.TripleOAction.cache_delete')
def test_run_mistral_env_missing(self, mock_cache):
self.mistral.environments.update.side_effect = (
mistral_base.APIException)
@ -286,6 +292,10 @@ class UpdatePlanActionTest(base.TestCase):
self.container_name)
self.assertEqual(result.error, error_str)
self.swift.delete_object.assert_not_called()
mock_cache.assert_called_once_with(
"Test-container-3",
"tripleo.parameters.get"
)
def test_run_missing_file(self):
self.swift.get_object.side_effect = swiftexceptions.ClientException(

10
tripleo_common/tests/actions/test_scale.py
View File

@ -40,6 +40,8 @@ class ScaleDownActionTest(base.TestCase):
super(ScaleDownActionTest, self).setUp()
self.image = collections.namedtuple('image', ['id'])
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'cache_delete')
@mock.patch('tripleo_common.actions.base.TripleOAction.'
'get_orchestration_client')
@mock.patch('heatclient.common.template_utils.'
@ -51,7 +53,8 @@ class ScaleDownActionTest(base.TestCase):
@mock.patch('mistral.context.ctx')
def test_run(self, mock_ctx, mock_get_object_client,
mock_get_workflow_client, mock_get_template_contents,
mock_env_files, mock_get_heat_client):
mock_env_files, mock_get_heat_client,
mock_cache):
mock_env_files.return_value = ({}, {})
heatclient = mock.MagicMock()
@ -120,3 +123,8 @@ class ScaleDownActionTest(base.TestCase):
existing=True,
files={},
timeout_mins=240)
mock_cache.assert_called_once_with(
"stack",
"tripleo.parameters.get"
)