Config download support for all deployments

Presently, "openstack overcloud config download" only supports the
OS::HeatSoftwareDeploymentGroup (plural) resources from
deploy-steps.yaml.

This patch adds support to the command to also pull the deployment
resources for each individual server from the OS::Heat::Value resources
called "TripleODeployment", as well as other SoftwareDeploymentGroup
resources not from deploy-steps.j2. See the Depends-On
tripleo-heat-templates patch for how these resources are mapped.

These deployments are applied by additional plays in the generated
playbooks. Instead of having to reimplement each deployment in a native
group:ansible, the ansible tasks just trigger the deployments by calling
55-heat-config remotely with the rendered json deployment data for
whatever deployment is being applied.

This method saves us from having to migrate all these resources to
native ansible (and requring all end users to do so).

Also included is a helper script called tripleo-config-download to run
the config download outside of Mistral.

implements: blueprint ansible-config-download
Change-Id: I7d7f6b831b8566390d8f747fb6f45e879b0392ba
Depends-On: Ic2af634403b1ab2924c383035f770453f39a2cd5

scripts/tripleo-config-download

@@ -0,0 +1,56 @@
+#!/usr/bin/python
+# Copyright 2016 Red Hat, Inc.
+# All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import logging
+import os
+import sys
+
+import os_client_config
+from tripleo_common.utils import config
+
+
+def get_orchestration_client():
+    return os_client_config.make_client('orchestration')
+
+
+def get_args():
+    parser = argparse.ArgumentParser(
+                description=("tripleo-config-download"),
+                formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+    parser.add_argument('--stack-name', '-s',
+                        default='overcloud',
+                        help="Heat stack name")
+    parser.add_argument('--output-dir', '-o',
+                        default='tripleo-config-download',
+                        help="Output directory for downloaded config")
+
+    args = parser.parse_args(sys.argv[1:])
+    return args
+
+if __name__ == '__main__':
+    args = get_args()
+
+    logging.basicConfig()
+    log = logging.getLogger()
+    log.setLevel(logging.INFO)
+
+    if not os.path.exists(args.output_dir):
+        os.mkdir(args.output_dir)
+
+    client = get_orchestration_client()
+    stack_config = config.Config(client)
+    stack_config.download_config(args.stack_name, args.output_dir)

setup.cfg

@@ -30,6 +30,7 @@ scripts =
     scripts/pull-puppet-modules
     scripts/run-validation
     scripts/tripleo-build-images
+    scripts/tripleo-config-download
     scripts/upgrade-non-controller.sh
     scripts/upload-puppet-modules
     scripts/upload-swift-artifacts

tripleo_common/constants.py

@@ -136,3 +136,12 @@ TRIPLEO_CACHE_CONTAINER = "__cache__"
 
 TRIPLEO_UI_LOG_FILE_SIZE = 1e7  # 10MB
 TRIPLEO_UI_LOG_FILENAME = 'tripleo-ui.logs'
+
+# Default nested depth when recursing Heat stacks
+NESTED_DEPTH = 7
+
+# Default string format for server resource types
+SERVER_RESOURCE_TYPES = 'OS::TripleO::%sServer'
+
+# Resource name for deployment resources when using config download
+TRIPLEO_DEPLOYMENT_RESOURCE = 'TripleODeployment'

tripleo_common/templates/deployments.yaml

@@ -0,0 +1,42 @@
+- name: Create /var/lib/heat-config/tripleo-config-download directory for deployment data
+  file:
+    path: /var/lib/heat-config/tripleo-config-download
+    state: directory
+  become: true
+
+- name: "Render deployment file for {{ item }}"
+  template:
+    src: ../templates/heat-config.j2
+    dest: "/var/lib/heat-config/tripleo-config-download/{{ item }}"
+  become: true
+  vars:
+    deployment: "{{ vars[item] | to_json }}"
+
+- name: "Force remove deployed file for {{ item }}"
+  file:
+    path: /var/lib/heat-config/deployed/{{ vars[item].id }}.json
+    state: absent
+  become: true
+  when: force | bool
+
+- name: "Run deployment {{ item }}"
+  shell: |
+    # TODO(emilien) deploying jq this way is a workaround until we figure where to do it
+    # properly. jq is required by this task.
+    yum install -y jq
+    /usr/libexec/os-refresh-config/configure.d/55-heat-config
+    exit $(jq .deploy_status_code /var/lib/heat-config/deployed/{{ vars[item].id }}.notify.json)
+  become: true
+  environment:
+    HEAT_SHELL_CONFIG: /var/lib/heat-config/tripleo-config-download/{{ item }}
+  register: deployment_result
+  ignore_errors: yes
+
+- name: "Output for {{ item }}"
+  debug:
+    msg:
+      - stderr: "{{ deployment_result.stderr.split('\n') }}"
+      - status_code: "{{ deployment_result.rc }}"
+  tags:
+    - output
+  failed_when: deployment_result.rc not in (0, 2)

tripleo_common/templates/group_var_role.j2

@@ -0,0 +1,10 @@
+{{ role }}_pre_deployments:
+{% for deployment in pre_deployments %}
+  - {{ deployment }}
+{% endfor %}
+
+{{ role }}_post_deployments: {% if not post_deployments %} [] {% endif %}
+
+{% for deployment in post_deployments %}
+  - {{ deployment }}
+{% endfor %}

tripleo_common/templates/group_var_server.j2

@@ -0,0 +1,32 @@
+deploy_server_id: {{ server_id }}
+
+{% for deployment in deployments %}
+{{ deployment.get('deployment_name') }}:
+{% if deployment.get('scalar') %}
+  config: |
+{% else %}
+  config:
+{% endif %}
+{{ deployment.get('config') | string | indent(4, true) }}
+  creation_time: "{{ deployment.get('creation_time') }}"
+  deployment_name: {{ deployment.get('deployment_name') }}
+  group: {{ deployment.get('group') }}
+  id: {{ deployment.get('id') }}
+  inputs:
+{% for input in deployment.get('inputs') %}
+    - name: {{ input.get('name') }}
+      description: {{ input.get('description') }}
+      type: {{ input.get('type') }}
+      value: |-
+{{ input.get('value') | string | indent(8, true) }}
+{% endfor %}
+  name: {{ deployment.get('name') }}
+  options: {{ deployment.get('options') }}
+  outputs:
+{% for output in deployment.get('outputs') %}
+    - name: {{ output.get('name') }}
+      description: {{ output.get('description') }}
+      type: {{ output.get('type') }}
+{% endfor %}
+
+{% endfor %}

tripleo_common/templates/heat-config.j2

@@ -0,0 +1 @@
+[{{ deployment }}]

tripleo_common/tests/utils/test_config.py

@@ -27,13 +27,15 @@ class TestConfig(base.TestCase):
     def setUp(self):
         super(TestConfig, self).setUp()
 
+    @patch.object(ooo_config.shutil, 'copyfile')
     @patch.object(ooo_config.Config, '_mkdir')
     @patch.object(ooo_config.Config, '_open_file')
     @mock.patch('tempfile.mkdtemp', autospec=True)
     def test_overcloud_config_generate_config(self,
                                               mock_tmpdir,
                                               mock_open,
-                                              mock_mkdir):
+                                              mock_mkdir,
+                                              mock_copyfile):
         config_type_list = ['config_settings', 'global_config_settings',
                             'logging_sources', 'monitoring_subscriptions',
                             'service_config_settings',
@@ -62,13 +64,15 @@ class TestConfig(base.TestCase):
                                             (role, config))]
         mock_open.assert_has_calls(expected_calls, any_order=True)
 
+    @patch.object(ooo_config.shutil, 'copyfile')
     @patch.object(ooo_config.Config, '_mkdir')
     @patch.object(ooo_config.Config, '_open_file')
     @mock.patch('tempfile.mkdtemp', autospec=True)
     def test_overcloud_config_one_config_type(self,
                                               mock_tmpdir,
                                               mock_open,
-                                              mock_mkdir):
+                                              mock_mkdir,
+                                              mock_copyfile):
 
         expected_config_type = 'config_settings'
         fake_role = [role for role in

tripleo_common/utils/config.py

@@ -12,19 +12,26 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations
 # under the License.
+import json
 import logging
 import os
 import re
+import shutil
 import six
 import tempfile
 import yaml
 
+import jinja2
+
+from tripleo_common import constants
+
 
 class Config(object):
 
     def __init__(self, orchestration_client):
         self.log = logging.getLogger(__name__ + ".Config")
         self.client = orchestration_client
+        self.__server_id_data = None
 
     def get_role_data(self, stack):
         role_data = {}
@@ -42,6 +49,70 @@ class Config(object):
                     role_data[role] = output['output_value'][role]
         return role_data
 
+    def get_server_data(self, stack, role_names,
+                        nested_depth=constants.NESTED_DEPTH):
+        servers = []
+        server_resource_types = [constants.SERVER_RESOURCE_TYPES % r
+                                 for r in role_names]
+
+        for server_resource_type in server_resource_types:
+            servers += self.client.resources.list(
+                stack,
+                nested_depth=nested_depth,
+                filters=dict(type=server_resource_type),
+                with_detail=True)
+        return servers
+
+    def get_deployment_data(self, stack,
+                            nested_depth=constants.NESTED_DEPTH):
+        deployments = self.client.resources.list(
+            stack,
+            nested_depth=nested_depth,
+            filters=dict(name=constants.TRIPLEO_DEPLOYMENT_RESOURCE),
+            with_detail=True)
+        # Sort by creation time
+        deployments = sorted(deployments, key=lambda d: d.creation_time)
+        return deployments
+
+    def get_server_id_data(self, stack):
+        for output in stack.to_dict().get('outputs', {}):
+            if output['output_key'] == 'ServerIdData':
+                return output['output_value']['server_ids']
+
+    def get_role_from_server_id(self, stack, server_id):
+        if self.__server_id_data is None:
+            self.__server_id_data = self.get_server_id_data(stack)
+
+        for k, v in self.__server_id_data.items():
+            if server_id in v:
+                return k
+
+    def get_config_dict(self, deployment):
+        if '/' in deployment.attributes['value']['deployment']:
+            deployment_stack_id = \
+                deployment.attributes['value']['deployment'].split('/')[-1]
+            deployment_resource_id = self.client.resources.get(
+                deployment_stack_id,
+                'TripleOSoftwareDeployment').physical_resource_id
+        else:
+            deployment_resource_id = \
+                deployment.attributes['value']['deployment']
+        deployment_rsrc = self.client.software_deployments.get(
+            deployment_resource_id)
+        config = self.client.software_configs.get(
+            deployment_rsrc.config_id)
+
+        return config.to_dict()
+
+    def get_jinja_env(self, tmp_path):
+        templates_path = os.path.join(
+            os.path.dirname(__file__), '..', 'templates')
+        self._mkdir(os.path.join(tmp_path, 'templates'))
+        env = jinja2.Environment(
+            loader=jinja2.FileSystemLoader(templates_path))
+        env.trim_blocks = True
+        return env, templates_path
+
     @staticmethod
     def _open_file(path):
         return os.fdopen(os.open(path,
@@ -102,6 +173,7 @@ class Config(object):
                                     dir=config_dir)
         self.log.info("Generating configuration under the directory: "
                       "%s" % tmp_path)
+
         # Get role data:
         role_data = self.get_role_data(stack)
         for role_name, role in six.iteritems(role_data):
@@ -137,6 +209,113 @@ class Config(object):
             conf_path = os.path.join(tmp_path, config_name + ".yaml")
             with self._open_file(conf_path) as conf_file:
                 conf_file.write(config)
+
+        # Get deployment data
+        self.log.info("Getting server data from Heat...")
+        server_data = self.get_server_data(name, role_data.keys())
+        self.log.info("Getting deployment data from Heat...")
+        deployments_data = self.get_deployment_data(name)
+
+        # server_deployments is a dict of server name to a list of deployments
+        # (dicts) associated with that server
+        server_deployments = {}
+        # server_ids is a dict of server_name to server_id for easier lookup
+        server_ids = {}
+        # role_deployment_names is a dict of role names to deployment names for
+        # that role. The deployment names are futher separated in their own
+        # dict with keys of pre_deployment/post_deployment.
+        role_deployment_names = {}
+
+        for deployment in deployments_data:
+            server_id = deployment.attributes['value']['server']
+            server = [s for s in server_data
+                      if s.physical_resource_id == server_id or
+                      s.attributes.get('OS::stack_id') == server_id].pop()
+            server_ids.setdefault(server.attributes['name'], server_id)
+            config_dict = self.get_config_dict(deployment)
+
+            # deployment_name should be set via the name property on the
+            # Deployment resources in the templates, however, if it's None,
+            # default to the name of the parent_resource
+            deployment_name = deployment.attributes['value'].get(
+                'name', deployment.parent_resource)
+            config_dict['deployment_name'] = deployment_name
+
+            # reset deploy_server_id to the actual server_id since we have to
+            # use a dummy server resource to create the deployment in the
+            # templates
+            deploy_server_id_input = \
+                [i for i in config_dict['inputs']
+                 if i['name'] == 'deploy_server_id'].pop()
+            deploy_server_id_input['value'] = server_id
+
+            server_deployments.setdefault(
+                server.attributes['name'],
+                []).append(config_dict)
+
+            role = self.get_role_from_server_id(stack, server_id)
+            role_deployments = role_deployment_names.setdefault(role, {})
+            role_pre_deployments = role_deployments.setdefault(
+                'pre_deployments', [])
+            role_post_deployments = role_deployments.setdefault(
+                'post_deployments', [])
+
+            # special handling of deployments that are run post the deploy
+            # steps. We have to look these up based on the
+            # physical_resource_id, but these names should be consistent since
+            # they are consistent interfaces in our templates.
+            if 'ExtraConfigPost' in deployment.physical_resource_id or \
+                    'PostConfig' in deployment.physical_resource_id:
+                if deployment_name not in role_post_deployments:
+                    role_post_deployments.append(deployment_name)
+            else:
+                if deployment_name not in role_pre_deployments:
+                    role_pre_deployments.append(deployment_name)
+
+        env, templates_path = self.get_jinja_env(tmp_path)
+
+        templates_dest = os.path.join(tmp_path, 'templates')
+        self._mkdir(templates_dest)
+        shutil.copyfile(os.path.join(templates_path, 'heat-config.j2'),
+                        os.path.join(templates_dest, 'heat-config.j2'))
+
+        group_vars_dir = os.path.join(tmp_path, 'group_vars')
+        self._mkdir(group_vars_dir)
+
+        for server, deployments in server_deployments.items():
+            group_var_server_path = os.path.join(group_vars_dir, server)
+            group_var_server_template = env.get_template('group_var_server.j2')
+
+            for d in deployments:
+                if isinstance(d['config'], dict):
+                    d['config'] = json.dumps(d['config'])
+                if d['group'] == 'hiera':
+                    d['scalar'] = False
+                else:
+                    d['scalar'] = True
+
+            with open(group_var_server_path, 'w') as f:
+                f.write(group_var_server_template.render(
+                    deployments=deployments,
+                    server_id=server_ids[server]))
+
+        for role, deployments in role_deployment_names.items():
+            group_var_role_path = os.path.join(group_vars_dir, role)
+            group_var_role_template = env.get_template('group_var_role.j2')
+
+            with open(group_var_role_path, 'w') as f:
+                f.write(group_var_role_template.render(
+                    role=role,
+                    pre_deployments=deployments['pre_deployments'],
+                    post_deployments=deployments['post_deployments']))
+
+        for role_name, role in six.iteritems(role_data):
+            role_path = os.path.join(tmp_path, role_name)
+
+            shutil.copyfile(
+                os.path.join(templates_path, 'deployments.yaml'),
+                os.path.join(role_path, 'deployments.yaml'))
+
         self.log.info("The TripleO configuration has been successfully "
                       "generated into: %s" % tmp_path)
         return tmp_path