328 lines
14 KiB
Python
328 lines
14 KiB
Python
# Copyright 2016 Red Hat, Inc.
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
import json
|
|
import logging
|
|
import os
|
|
import re
|
|
import shutil
|
|
import six
|
|
import tempfile
|
|
import yaml
|
|
|
|
import jinja2
|
|
|
|
from tripleo_common import constants
|
|
|
|
|
|
class Config(object):
|
|
|
|
def __init__(self, orchestration_client):
|
|
self.log = logging.getLogger(__name__ + ".Config")
|
|
self.client = orchestration_client
|
|
self.__server_id_data = None
|
|
|
|
def get_role_data(self, stack):
|
|
role_data = {}
|
|
for output in stack.to_dict().get('outputs', {}):
|
|
if output['output_key'] == 'RoleData':
|
|
for role in output['output_value']:
|
|
role_data[role] = output['output_value'][role]
|
|
return role_data
|
|
|
|
def get_role_config(self, stack):
|
|
role_data = {}
|
|
for output in stack.to_dict().get('outputs', {}):
|
|
if output['output_key'] == 'RoleConfig':
|
|
for role in output['output_value']:
|
|
role_data[role] = output['output_value'][role]
|
|
return role_data
|
|
|
|
def get_server_data(self, stack, role_names,
|
|
nested_depth=constants.NESTED_DEPTH):
|
|
servers = []
|
|
server_resource_types = [constants.SERVER_RESOURCE_TYPES % r
|
|
for r in role_names]
|
|
|
|
for server_resource_type in server_resource_types:
|
|
servers += self.client.resources.list(
|
|
stack,
|
|
nested_depth=nested_depth,
|
|
filters=dict(type=server_resource_type),
|
|
with_detail=True)
|
|
return servers
|
|
|
|
def get_deployment_data(self, stack,
|
|
nested_depth=constants.NESTED_DEPTH):
|
|
deployments = self.client.resources.list(
|
|
stack,
|
|
nested_depth=nested_depth,
|
|
filters=dict(name=constants.TRIPLEO_DEPLOYMENT_RESOURCE),
|
|
with_detail=True)
|
|
# Sort by creation time
|
|
deployments = sorted(deployments, key=lambda d: d.creation_time)
|
|
return deployments
|
|
|
|
def get_server_id_data(self, stack):
|
|
for output in stack.to_dict().get('outputs', {}):
|
|
if output['output_key'] == 'ServerIdData':
|
|
return output['output_value']['server_ids']
|
|
|
|
def get_role_from_server_id(self, stack, server_id):
|
|
if self.__server_id_data is None:
|
|
self.__server_id_data = self.get_server_id_data(stack)
|
|
|
|
for k, v in self.__server_id_data.items():
|
|
if server_id in v:
|
|
return k
|
|
|
|
def get_config_dict(self, deployment):
|
|
if '/' in deployment.attributes['value']['deployment']:
|
|
deployment_stack_id = \
|
|
deployment.attributes['value']['deployment'].split('/')[-1]
|
|
deployment_resource_id = self.client.resources.get(
|
|
deployment_stack_id,
|
|
'TripleOSoftwareDeployment').physical_resource_id
|
|
else:
|
|
deployment_resource_id = \
|
|
deployment.attributes['value']['deployment']
|
|
deployment_rsrc = self.client.software_deployments.get(
|
|
deployment_resource_id)
|
|
config = self.client.software_configs.get(
|
|
deployment_rsrc.config_id)
|
|
|
|
return config.to_dict()
|
|
|
|
def get_jinja_env(self, tmp_path):
|
|
templates_path = os.path.join(
|
|
os.path.dirname(__file__), '..', 'templates')
|
|
self._mkdir(os.path.join(tmp_path, 'templates'))
|
|
env = jinja2.Environment(
|
|
loader=jinja2.FileSystemLoader(templates_path))
|
|
env.trim_blocks = True
|
|
return env, templates_path
|
|
|
|
@staticmethod
|
|
def _open_file(path):
|
|
return os.fdopen(os.open(path,
|
|
os.O_WRONLY | os.O_CREAT, 0o600),
|
|
'w')
|
|
|
|
def _step_tags_to_when(self, sorted_tasks):
|
|
for task in sorted_tasks:
|
|
tag = task.get('tags', '')
|
|
match = re.search('step([0-9]+)', tag)
|
|
if match:
|
|
step = match.group(1)
|
|
whenexpr = task.get('when', None)
|
|
if whenexpr:
|
|
# Handle when: foo and a list of when conditionals
|
|
if not isinstance(whenexpr, list):
|
|
whenexpr = [whenexpr]
|
|
for w in whenexpr:
|
|
when_exists = re.search('step|int == [0-9]', w)
|
|
if when_exists:
|
|
break
|
|
if when_exists:
|
|
# Skip to the next task,
|
|
# there is an existing 'step|int == N'
|
|
continue
|
|
whenexpr.append("step|int == %s" % step)
|
|
task['when'] = whenexpr
|
|
else:
|
|
task.update({"when": "step|int == %s" % step})
|
|
|
|
def _write_playbook_get_tasks(self, tasks, role, filepath):
|
|
playbook = []
|
|
sorted_tasks = sorted(tasks, key=lambda x: x.get('tags', None))
|
|
self._step_tags_to_when(sorted_tasks)
|
|
playbook.append({'name': '%s playbook' % role,
|
|
'hosts': role,
|
|
'tasks': sorted_tasks})
|
|
with self._open_file(filepath) as conf_file:
|
|
yaml.safe_dump(playbook, conf_file, default_flow_style=False)
|
|
return sorted_tasks
|
|
|
|
def _mkdir(self, dirname):
|
|
if not os.path.exists(dirname):
|
|
try:
|
|
os.mkdir(dirname, 0o700)
|
|
except OSError as e:
|
|
message = 'Failed to create: %s, error: %s' % (dirname,
|
|
str(e))
|
|
raise OSError(message)
|
|
|
|
def download_config(self, name, config_dir, config_type=None):
|
|
# Get the stack object
|
|
stack = self.client.stacks.get(name)
|
|
# Create config directory
|
|
self._mkdir(config_dir)
|
|
tmp_path = tempfile.mkdtemp(prefix='tripleo-',
|
|
suffix='-config',
|
|
dir=config_dir)
|
|
self.log.info("Generating configuration under the directory: "
|
|
"%s" % tmp_path)
|
|
|
|
# Get role data:
|
|
role_data = self.get_role_data(stack)
|
|
for role_name, role in six.iteritems(role_data):
|
|
role_path = os.path.join(tmp_path, role_name)
|
|
self._mkdir(role_path)
|
|
for config in config_type or role.keys():
|
|
if config in constants.EXTERNAL_TASKS:
|
|
# external tasks are collected globally, not per-role
|
|
continue
|
|
elif config == 'step_config':
|
|
filepath = os.path.join(role_path, 'step_config.pp')
|
|
with self._open_file(filepath) as step_config:
|
|
step_config.write('\n'.join(step for step in
|
|
role[config]
|
|
if step is not None))
|
|
else:
|
|
if 'upgrade_tasks' in config:
|
|
filepath = os.path.join(role_path, '%s_playbook.yaml' %
|
|
config)
|
|
data = self._write_playbook_get_tasks(
|
|
role[config], role_name, filepath)
|
|
else:
|
|
try:
|
|
data = role[config]
|
|
except KeyError as e:
|
|
message = 'Invalid key: %s, error: %s' % (config,
|
|
str(e))
|
|
raise KeyError(message)
|
|
filepath = os.path.join(role_path, '%s.yaml' % config)
|
|
with self._open_file(filepath) as conf_file:
|
|
yaml.safe_dump(data,
|
|
conf_file,
|
|
default_flow_style=False)
|
|
role_config = self.get_role_config(stack)
|
|
for config_name, config in six.iteritems(role_config):
|
|
conf_path = os.path.join(tmp_path, config_name + ".yaml")
|
|
with self._open_file(conf_path) as conf_file:
|
|
if isinstance(config, list) or isinstance(config, dict):
|
|
yaml.safe_dump(config, conf_file, default_flow_style=False)
|
|
else:
|
|
conf_file.write(config)
|
|
|
|
# Get deployment data
|
|
self.log.info("Getting server data from Heat...")
|
|
server_data = self.get_server_data(name, role_data.keys())
|
|
self.log.info("Getting deployment data from Heat...")
|
|
deployments_data = self.get_deployment_data(name)
|
|
|
|
# server_deployments is a dict of server name to a list of deployments
|
|
# (dicts) associated with that server
|
|
server_deployments = {}
|
|
# server_ids is a dict of server_name to server_id for easier lookup
|
|
server_ids = {}
|
|
# role_deployment_names is a dict of role names to deployment names for
|
|
# that role. The deployment names are futher separated in their own
|
|
# dict with keys of pre_deployment/post_deployment.
|
|
role_deployment_names = {}
|
|
|
|
for deployment in deployments_data:
|
|
server_id = deployment.attributes['value']['server']
|
|
server = [s for s in server_data
|
|
if s.physical_resource_id == server_id or
|
|
s.attributes.get('OS::stack_id') == server_id].pop()
|
|
server_ids.setdefault(server.attributes['name'], server_id)
|
|
config_dict = self.get_config_dict(deployment)
|
|
|
|
# deployment_name should be set via the name property on the
|
|
# Deployment resources in the templates, however, if it's None,
|
|
# default to the name of the parent_resource
|
|
deployment_name = deployment.attributes['value'].get(
|
|
'name', deployment.parent_resource)
|
|
config_dict['deployment_name'] = deployment_name
|
|
|
|
# reset deploy_server_id to the actual server_id since we have to
|
|
# use a dummy server resource to create the deployment in the
|
|
# templates
|
|
deploy_server_id_input = \
|
|
[i for i in config_dict['inputs']
|
|
if i['name'] == 'deploy_server_id'].pop()
|
|
deploy_server_id_input['value'] = server_id
|
|
|
|
server_deployments.setdefault(
|
|
server.attributes['name'],
|
|
[]).append(config_dict)
|
|
|
|
role = self.get_role_from_server_id(stack, server_id)
|
|
role_deployments = role_deployment_names.setdefault(role, {})
|
|
role_pre_deployments = role_deployments.setdefault(
|
|
'pre_deployments', [])
|
|
role_post_deployments = role_deployments.setdefault(
|
|
'post_deployments', [])
|
|
|
|
# special handling of deployments that are run post the deploy
|
|
# steps. We have to look these up based on the
|
|
# physical_resource_id, but these names should be consistent since
|
|
# they are consistent interfaces in our templates.
|
|
if 'ExtraConfigPost' in deployment.physical_resource_id or \
|
|
'PostConfig' in deployment.physical_resource_id:
|
|
if deployment_name not in role_post_deployments:
|
|
role_post_deployments.append(deployment_name)
|
|
else:
|
|
if deployment_name not in role_pre_deployments:
|
|
role_pre_deployments.append(deployment_name)
|
|
|
|
env, templates_path = self.get_jinja_env(tmp_path)
|
|
|
|
templates_dest = os.path.join(tmp_path, 'templates')
|
|
self._mkdir(templates_dest)
|
|
shutil.copyfile(os.path.join(templates_path, 'heat-config.j2'),
|
|
os.path.join(templates_dest, 'heat-config.j2'))
|
|
|
|
group_vars_dir = os.path.join(tmp_path, 'group_vars')
|
|
self._mkdir(group_vars_dir)
|
|
|
|
for server, deployments in server_deployments.items():
|
|
group_var_server_path = os.path.join(group_vars_dir, server)
|
|
group_var_server_template = env.get_template('group_var_server.j2')
|
|
|
|
for d in deployments:
|
|
if isinstance(d['config'], dict):
|
|
d['config'] = json.dumps(d['config'])
|
|
if d['group'] == 'hiera':
|
|
d['scalar'] = False
|
|
else:
|
|
d['scalar'] = True
|
|
|
|
with open(group_var_server_path, 'w') as f:
|
|
f.write(group_var_server_template.render(
|
|
deployments=deployments,
|
|
server_id=server_ids[server]))
|
|
|
|
for role, deployments in role_deployment_names.items():
|
|
group_var_role_path = os.path.join(group_vars_dir, role)
|
|
group_var_role_template = env.get_template('group_var_role.j2')
|
|
|
|
with open(group_var_role_path, 'w') as f:
|
|
f.write(group_var_role_template.render(
|
|
role=role,
|
|
pre_deployments=deployments['pre_deployments'],
|
|
post_deployments=deployments['post_deployments']))
|
|
|
|
for role_name, role in six.iteritems(role_data):
|
|
role_path = os.path.join(tmp_path, role_name)
|
|
|
|
shutil.copyfile(
|
|
os.path.join(templates_path, 'deployments.yaml'),
|
|
os.path.join(role_path, 'deployments.yaml'))
|
|
|
|
self.log.info("The TripleO configuration has been successfully "
|
|
"generated into: %s" % tmp_path)
|
|
return tmp_path
|