Merge "Reduce the calls to heat when downloading config"
This commit is contained in:
Zuul
Gerrit Code Review
commit
a6f37f3742
|
|
@ -145,10 +145,9 @@ LEGACY_API_NETWORK = 'Internal'
|
|||
# Default nested depth when recursing Heat stacks
|
||||
NESTED_DEPTH = 7
|
||||
|
||||
# Default string format for server resource types
|
||||
SERVER_RESOURCE_TYPES = 'OS::TripleO::%sServer'
|
||||
|
||||
# Resource name for deployment resources when using config download
|
||||
TRIPLEO_DEPLOYMENT_RESOURCE = 'TripleODeployment'
|
||||
|
||||
HOST_NETWORK = 'ctlplane'
|
||||
|
||||
EXTERNAL_TASKS = ['external_deploy_tasks']
|
||||
|
|
|
|||
|
|
@ -111,9 +111,7 @@ class TestConfig(base.TestCase):
|
|||
KeyError,
|
||||
self.config.download_config, *args)
|
||||
|
||||
@mock.patch('tripleo_common.utils.config.Config.get_role_data',
|
||||
autospec=True)
|
||||
def test_overcloud_config_upgrade_tasks(self, mock_get_role_data):
|
||||
def test_overcloud_config_upgrade_tasks(self):
|
||||
|
||||
heat = mock.MagicMock()
|
||||
heat.stacks.get.return_value = fakes.create_tht_stack()
|
||||
|
|
@ -140,8 +138,6 @@ class TestConfig(base.TestCase):
|
|||
'tags': 'step1',
|
||||
'when': ['step|int == 1',
|
||||
'existing', 'list']}]}
|
||||
mock_get_role_data.return_value = fake_role
|
||||
|
||||
for role in fake_role:
|
||||
filedir = os.path.join(self.tmp_dir, role)
|
||||
os.makedirs(filedir)
|
||||
|
|
@ -152,34 +148,25 @@ class TestConfig(base.TestCase):
|
|||
self.assertTrue(os.path.isfile(filepath))
|
||||
self.assertEqual(expected_tasks[role], playbook_tasks)
|
||||
|
||||
def test_get_server_data(self):
|
||||
def test_get_server_names(self):
|
||||
heat = mock.MagicMock()
|
||||
self.config = ooo_config.Config(heat)
|
||||
stack = 'overcloud'
|
||||
role_names = ['Controller', 'Compute', 'Custom']
|
||||
heat.resources.list.return_value = ['fakeserver']
|
||||
server_data = self.config.get_server_data(stack, role_names)
|
||||
self.assertEqual(heat.resources.list.call_count, 3)
|
||||
self.assertEqual(
|
||||
heat.resources.list.call_args_list[0],
|
||||
mock.call(stack,
|
||||
filters=dict(type='OS::TripleO::ControllerServer'),
|
||||
nested_depth=constants.NESTED_DEPTH,
|
||||
with_detail=True))
|
||||
self.assertEqual(
|
||||
heat.resources.list.call_args_list[1],
|
||||
mock.call(stack,
|
||||
filters=dict(type='OS::TripleO::ComputeServer'),
|
||||
nested_depth=constants.NESTED_DEPTH,
|
||||
with_detail=True))
|
||||
self.assertEqual(
|
||||
heat.resources.list.call_args_list[2],
|
||||
mock.call(stack,
|
||||
filters=dict(type='OS::TripleO::CustomServer'),
|
||||
nested_depth=constants.NESTED_DEPTH,
|
||||
with_detail=True))
|
||||
self.assertEqual(server_data,
|
||||
['fakeserver', 'fakeserver', 'fakeserver'])
|
||||
self.config.stack_outputs = {
|
||||
'RoleNetHostnameMap': {
|
||||
'Controller': {
|
||||
'ctlplane': [
|
||||
'c0.ctlplane.localdomain',
|
||||
'c1.ctlplane.localdomain',
|
||||
'c2.ctlplane.localdomain']}},
|
||||
'ServerIdData': {
|
||||
'server_ids': {
|
||||
'Controller': [
|
||||
'8269f736',
|
||||
'2af0a373',
|
||||
'c8479674']}}}
|
||||
server_names = self.config.get_server_names()
|
||||
expected = {'2af0a373': 'c1', '8269f736': 'c0', 'c8479674': 'c2'}
|
||||
self.assertEqual(expected, server_names)
|
||||
|
||||
def test_get_deployment_data(self):
|
||||
heat = mock.MagicMock()
|
||||
|
|
@ -212,17 +199,8 @@ class TestConfig(base.TestCase):
|
|||
'data',
|
||||
datafile)
|
||||
config_data = yaml.safe_load(open(config_data_path).read())
|
||||
server_data = []
|
||||
deployment_data = []
|
||||
|
||||
for server in config_data['servers']:
|
||||
server_mock = mock.MagicMock()
|
||||
server_mock.physical_resource_id = server['physical_resource_id']
|
||||
server_mock.name = server['name']
|
||||
server_mock.attributes = {'OS::stack_id': server['OS::stack_id'],
|
||||
'name': server['name']}
|
||||
server_data.append(server_mock)
|
||||
|
||||
for deployment in config_data['deployments']:
|
||||
deployment_mock = mock.MagicMock()
|
||||
deployment_mock.id = deployment['deployment']
|
||||
|
|
@ -233,10 +211,9 @@ class TestConfig(base.TestCase):
|
|||
name=deployment['name']))
|
||||
deployment_data.append(deployment_mock)
|
||||
|
||||
server_id_data = config_data['server_id_data']
|
||||
configs = config_data['configs']
|
||||
|
||||
return server_data, server_id_data, deployment_data, configs
|
||||
return deployment_data, configs
|
||||
|
||||
def _get_config_dict(self, deployment):
|
||||
config = self.configs[deployment.attributes['value']['config']].copy()
|
||||
|
|
@ -254,26 +231,42 @@ class TestConfig(base.TestCase):
|
|||
return yaml.safe_load(open(file_path).read())
|
||||
|
||||
@patch('tripleo_common.utils.config.Config.get_config_dict')
|
||||
@patch('tripleo_common.utils.config.Config.get_server_id_data')
|
||||
@patch('tripleo_common.utils.config.Config.get_deployment_data')
|
||||
@patch('tripleo_common.utils.config.Config.get_server_data')
|
||||
def test_config_download(self, mock_server_data, mock_deployment_data,
|
||||
mock_server_id_data, mock_config_dict):
|
||||
def test_config_download(self, mock_deployment_data, mock_config_dict):
|
||||
heat = mock.MagicMock()
|
||||
self.config = ooo_config.Config(heat)
|
||||
stack = 'overcloud'
|
||||
stack = mock.MagicMock()
|
||||
heat.stacks.get.return_value = stack
|
||||
|
||||
server_data, server_id_data, deployment_data, configs = \
|
||||
stack.outputs = [
|
||||
{'output_key': 'RoleNetHostnameMap',
|
||||
'output_value': {
|
||||
'Controller': {
|
||||
'ctlplane': [
|
||||
'overcloud-controller-0.ctlplane.localdomain']},
|
||||
'Compute': {
|
||||
'ctlplane': [
|
||||
'overcloud-novacompute-0.ctlplane.localdomain',
|
||||
'overcloud-novacompute-1.ctlplane.localdomain',
|
||||
'overcloud-novacompute-2.ctlplane.localdomain']}}},
|
||||
{'output_key': 'ServerIdData',
|
||||
'output_value': {
|
||||
'server_ids': {
|
||||
'Controller': [
|
||||
'00b3a5e1-5e8e-4b55-878b-2fa2271f15ad'],
|
||||
'Compute': [
|
||||
'a7db3010-a51f-4ae0-a791-2364d629d20d',
|
||||
'8b07cd31-3083-4b88-a433-955f72039e2c',
|
||||
'169b46f8-1965-4d90-a7de-f36fb4a830fe']}}}]
|
||||
deployment_data, configs = \
|
||||
self._get_config_data('config_data.yaml')
|
||||
self.configs = configs
|
||||
|
||||
mock_server_data.return_value = server_data
|
||||
mock_server_id_data.return_value = server_id_data
|
||||
mock_deployment_data.return_value = deployment_data
|
||||
mock_config_dict.side_effect = self._get_config_dict
|
||||
|
||||
tmp_path = self.config.download_config(stack, '/tmp')
|
||||
print("config at %s" % tmp_path)
|
||||
self.tmp_dir = self.useFixture(fixtures.TempDir()).path
|
||||
tmp_path = self.config.download_config(stack, self.tmp_dir)
|
||||
|
||||
for f in ['overcloud-controller-0',
|
||||
'overcloud-novacompute-0',
|
||||
|
|
|
|||
|
|
@ -31,36 +31,22 @@ class Config(object):
|
|||
def __init__(self, orchestration_client):
|
||||
self.log = logging.getLogger(__name__ + ".Config")
|
||||
self.client = orchestration_client
|
||||
self.__server_id_data = None
|
||||
self.stack_outputs = {}
|
||||
|
||||
def get_role_data(self, stack):
|
||||
role_data = {}
|
||||
for output in stack.to_dict().get('outputs', {}):
|
||||
if output['output_key'] == 'RoleData':
|
||||
for role in output['output_value']:
|
||||
role_data[role] = output['output_value'][role]
|
||||
return role_data
|
||||
|
||||
def get_role_config(self, stack):
|
||||
role_data = {}
|
||||
for output in stack.to_dict().get('outputs', {}):
|
||||
if output['output_key'] == 'RoleConfig':
|
||||
for role in output['output_value']:
|
||||
role_data[role] = output['output_value'][role]
|
||||
return role_data
|
||||
|
||||
def get_server_data(self, stack, role_names,
|
||||
nested_depth=constants.NESTED_DEPTH):
|
||||
servers = []
|
||||
server_resource_types = [constants.SERVER_RESOURCE_TYPES % r
|
||||
for r in role_names]
|
||||
|
||||
for server_resource_type in server_resource_types:
|
||||
servers += self.client.resources.list(
|
||||
stack,
|
||||
nested_depth=nested_depth,
|
||||
filters=dict(type=server_resource_type),
|
||||
with_detail=True)
|
||||
def get_server_names(self):
|
||||
servers = {}
|
||||
role_node_id_map = self.stack_outputs.get('ServerIdData', {})
|
||||
role_net_hostname_map = self.stack_outputs.get(
|
||||
'RoleNetHostnameMap', {})
|
||||
for role, hostnames in role_net_hostname_map.items():
|
||||
if hostnames:
|
||||
names = hostnames.get(constants.HOST_NETWORK) or []
|
||||
shortnames = [n.split(".%s." % constants.HOST_NETWORK)[0]
|
||||
for n in names]
|
||||
for idx, name in enumerate(shortnames):
|
||||
if 'server_ids' in role_node_id_map:
|
||||
server_id = role_node_id_map['server_ids'][role][idx]
|
||||
servers[server_id] = name
|
||||
return servers
|
||||
|
||||
def get_deployment_data(self, stack,
|
||||
|
|
@ -74,16 +60,11 @@ class Config(object):
|
|||
deployments = sorted(deployments, key=lambda d: d.creation_time)
|
||||
return deployments
|
||||
|
||||
def get_server_id_data(self, stack):
|
||||
for output in stack.to_dict().get('outputs', {}):
|
||||
if output['output_key'] == 'ServerIdData':
|
||||
return output['output_value']['server_ids']
|
||||
|
||||
def get_role_from_server_id(self, stack, server_id):
|
||||
if self.__server_id_data is None:
|
||||
self.__server_id_data = self.get_server_id_data(stack)
|
||||
server_id_data = self.stack_outputs.get('ServerIdData', {}
|
||||
).get('server_ids', {})
|
||||
|
||||
for k, v in self.__server_id_data.items():
|
||||
for k, v in server_id_data.items():
|
||||
if server_id in v:
|
||||
return k
|
||||
|
||||
|
|
@ -166,6 +147,9 @@ class Config(object):
|
|||
def download_config(self, name, config_dir, config_type=None):
|
||||
# Get the stack object
|
||||
stack = self.client.stacks.get(name)
|
||||
self.stack_outputs = {i['output_key']: i['output_value']
|
||||
for i in stack.outputs}
|
||||
|
||||
# Create config directory
|
||||
self._mkdir(config_dir)
|
||||
tmp_path = tempfile.mkdtemp(prefix='tripleo-',
|
||||
|
|
@ -175,7 +159,7 @@ class Config(object):
|
|||
"%s" % tmp_path)
|
||||
|
||||
# Get role data:
|
||||
role_data = self.get_role_data(stack)
|
||||
role_data = self.stack_outputs.get('RoleData', {})
|
||||
for role_name, role in six.iteritems(role_data):
|
||||
role_path = os.path.join(tmp_path, role_name)
|
||||
self._mkdir(role_path)
|
||||
|
|
@ -205,7 +189,7 @@ class Config(object):
|
|||
yaml.safe_dump(data,
|
||||
conf_file,
|
||||
default_flow_style=False)
|
||||
role_config = self.get_role_config(stack)
|
||||
role_config = self.stack_outputs.get('RoleConfig', {})
|
||||
for config_name, config in six.iteritems(role_config):
|
||||
conf_path = os.path.join(tmp_path, config_name + ".yaml")
|
||||
with self._open_file(conf_path) as conf_file:
|
||||
|
|
@ -215,16 +199,15 @@ class Config(object):
|
|||
conf_file.write(config)
|
||||
|
||||
# Get deployment data
|
||||
self.log.info("Getting server data from Heat...")
|
||||
server_data = self.get_server_data(name, role_data.keys())
|
||||
self.log.info("Getting deployment data from Heat...")
|
||||
deployments_data = self.get_deployment_data(name)
|
||||
|
||||
# server_deployments is a dict of server name to a list of deployments
|
||||
# (dicts) associated with that server
|
||||
server_deployments = {}
|
||||
# server_ids is a dict of server_name to server_id for easier lookup
|
||||
server_ids = {}
|
||||
# server_names is a dict of server id to server_name for easier lookup
|
||||
server_names = self.get_server_names()
|
||||
server_ids = dict([(v, k) for (k, v) in server_names.items()])
|
||||
# role_deployment_names is a dict of role names to deployment names for
|
||||
# that role. The deployment names are futher separated in their own
|
||||
# dict with keys of pre_deployment/post_deployment.
|
||||
|
|
@ -232,10 +215,6 @@ class Config(object):
|
|||
|
||||
for deployment in deployments_data:
|
||||
server_id = deployment.attributes['value']['server']
|
||||
server = [s for s in server_data
|
||||
if s.physical_resource_id == server_id or
|
||||
s.attributes.get('OS::stack_id') == server_id].pop()
|
||||
server_ids.setdefault(server.attributes['name'], server_id)
|
||||
config_dict = self.get_config_dict(deployment)
|
||||
|
||||
# deployment_name should be set via the name property on the
|
||||
|
|
@ -252,9 +231,8 @@ class Config(object):
|
|||
[i for i in config_dict['inputs']
|
||||
if i['name'] == 'deploy_server_id'].pop()
|
||||
deploy_server_id_input['value'] = server_id
|
||||
|
||||
server_deployments.setdefault(
|
||||
server.attributes['name'],
|
||||
server_names[server_id],
|
||||
[]).append(config_dict)
|
||||
|
||||
role = self.get_role_from_server_id(stack, server_id)
|
||||
|
|
|
|||
Loading…
Reference in New Issue