From db23426b426a3299b8a2b041dcc5d8085c9966a1 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Fri, 15 Dec 2017 16:51:29 +0000 Subject: [PATCH] Inital import of tripleo ansible inventory code This is moved from tripleo-validations so we can make use of the same code in tripleoclient for undercloud deploy, commit copied: a1e9265 Merge "Fix include action DEPRECATION WARNING (Ansible >2.4)" Note this doesn't currently move the inventory script, as we'll need to adjust packaging and that's not strictly needed for the use of the code in tripleoclient. Change-Id: I03f609b8ff9f36c95a8382aa1705147d99cb973b --- tripleo_common/inventory.py | 228 +++++++++++++++++++ tripleo_common/tests/test_inventory.py | 294 +++++++++++++++++++++++++ 2 files changed, 522 insertions(+) create mode 100644 tripleo_common/inventory.py create mode 100644 tripleo_common/tests/test_inventory.py diff --git a/tripleo_common/inventory.py b/tripleo_common/inventory.py new file mode 100644 index 000000000..f0aff3da8 --- /dev/null +++ b/tripleo_common/inventory.py @@ -0,0 +1,228 @@ +#!/usr/bin/env python + +# Copyright 2017 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from collections import OrderedDict + +from heatclient.exc import HTTPNotFound + +HOST_NETWORK = 'ctlplane' + + +class StackOutputs(object): + """Item getter for stack outputs. + + It takes a long time to resolve stack outputs. This class ensures that + we only have to do it once and then reuse the results from that call in + subsequent lookups. It also lazy loads the outputs so we don't spend time + on unnecessary Heat calls. + """ + + def __init__(self, plan, hclient): + self.plan = plan + self.outputs = {} + self.hclient = hclient + self.stack = None + + def _load_outputs(self): + """Load outputs from the stack if necessary + + Retrieves the stack outputs if that has not already happened. If it + has then this is a noop. + + Sets the outputs to an empty dict if the stack is not found. + """ + if not self.stack: + try: + self.stack = self.hclient.stacks.get(self.plan) + except HTTPNotFound: + self.outputs = {} + return + self.outputs = {i['output_key']: i['output_value'] + for i in self.stack.outputs + } + + def __getitem__(self, key): + self._load_outputs() + return self.outputs[key] + + def __iter__(self): + self._load_outputs() + return iter(self.outputs.keys()) + + def get(self, key, default=None): + try: + self.__getitem__(key) + except KeyError: + pass + return self.outputs.get(key, default) + + +class TripleoInventory(object): + def __init__(self, configs, session, hclient): + self.configs = configs + self.session = session + self.hclient = hclient + self.stack_outputs = StackOutputs(self.configs.plan, self.hclient) + + @staticmethod + def get_roles_by_service(enabled_services): + # Flatten the lists of services for each role into a set + services = set( + [item for role_services in enabled_services.values() + for item in role_services]) + + roles_by_services = {} + for service in services: + roles_by_services[service] = [] + for role, val in enabled_services.items(): + if service in val: + roles_by_services[service].append(role) + roles_by_services[service] = sorted(roles_by_services[service]) + return roles_by_services + + def get_overcloud_environment(self): + try: + environment = self.hclient.stacks.environment(self.configs.plan) + return environment + except HTTPNotFound: + return {} + + UNDERCLOUD_SERVICES = [ + 'openstack-nova-compute', 'openstack-heat-engine', + 'openstack-ironic-conductor', 'openstack-swift-container', + 'openstack-swift-object', 'openstack-mistral-engine'] + + def get_undercloud_service_list(self): + """Return list of undercloud services - currently static + + Replace this when we have a better way - e.g. heat deploys undercloud + """ + return self.UNDERCLOUD_SERVICES + + def list(self): + ret = OrderedDict({ + 'undercloud': { + 'hosts': ['localhost'], + 'vars': { + 'ansible_connection': 'local', + 'auth_url': self.configs.auth_url, + 'cacert': self.configs.cacert, + 'os_auth_token': self.session.get_token(), + 'plan': self.configs.plan, + 'project_name': self.configs.project_name, + 'username': self.configs.username, + }, + } + }) + + swift_url = self.session.get_endpoint(service_type='object-store', + interface='public') + if swift_url: + ret['undercloud']['vars']['undercloud_swift_url'] = swift_url + + keystone_url = self.stack_outputs.get('KeystoneURL') + if keystone_url: + ret['undercloud']['vars']['overcloud_keystone_url'] = keystone_url + admin_password = self.get_overcloud_environment().get( + 'parameter_defaults', {}).get('AdminPassword') + if admin_password: + ret['undercloud']['vars']['overcloud_admin_password'] =\ + admin_password + endpoint_map = self.stack_outputs.get('EndpointMap') + + ret['undercloud']['vars']['undercloud_service_list'] = \ + self.get_undercloud_service_list() + + if endpoint_map: + horizon_endpoint = endpoint_map.get('HorizonPublic', {}).get('uri') + if horizon_endpoint: + ret['undercloud']['vars']['overcloud_horizon_url'] =\ + horizon_endpoint + + role_net_ip_map = self.stack_outputs.get('RoleNetIpMap', {}) + role_node_id_map = self.stack_outputs.get('ServerIdData', {}) + networks = set() + role_net_hostname_map = self.stack_outputs.get( + 'RoleNetHostnameMap', {}) + children = [] + for role, hostnames in role_net_hostname_map.items(): + if hostnames: + names = hostnames.get(HOST_NETWORK) or [] + shortnames = [n.split(".%s." % HOST_NETWORK)[0] for n in names] + # Create a group per hostname to map hostname to IP + ips = role_net_ip_map[role][HOST_NETWORK] + for idx, name in enumerate(shortnames): + ret[name] = {'hosts': [ips[idx]]} + if 'server_ids' in role_node_id_map: + ret[name]['vars'] = { + 'deploy_server_id': role_node_id_map[ + 'server_ids'][role][idx]} + # Add variable for listing enabled networks in the node + ret[name]['vars']['enabled_networks'] = \ + [str(net) for net in role_net_ip_map[role]] + # Add variable for IP on each network + for net in role_net_ip_map[role]: + ret[name]['vars']["%s_ip" % net] = \ + role_net_ip_map[role][net][idx] + networks.update(ret[name]['vars']['enabled_networks']) + + children.append(role) + ret[role] = { + 'children': sorted(shortnames), + 'vars': { + 'ansible_ssh_user': self.configs.ansible_ssh_user, + 'bootstrap_server_id': role_node_id_map.get( + 'bootstrap_server_id'), + 'role_name': role, + } + } + + if children: + vip_map = self.stack_outputs.get('VipMap', {}) + vips = {(vip_name + "_vip"): vip + for vip_name, vip in vip_map.items() + if vip and (vip_name in networks or vip_name == 'redis')} + ret['overcloud'] = { + 'children': sorted(children), + 'vars': vips + } + + # Associate services with roles + roles_by_service = self.get_roles_by_service( + self.stack_outputs.get('EnabledServices', {})) + for service, roles in roles_by_service.items(): + service_children = [role for role in roles + if ret.get(role) is not None] + if service_children: + ret[service.lower()] = { + 'children': service_children, + 'vars': { + 'ansible_ssh_user': self.configs.ansible_ssh_user + } + } + + # Prevent Ansible from repeatedly calling us to get empty host details + ret['_meta'] = {'hostvars': {}} + + return ret + + def host(self): + # NOTE(mandre) + # Dynamic inventory scripts must return empty json if they don't + # provide detailed info for hosts: + # http://docs.ansible.com/ansible/developing_inventory.html + return {} diff --git a/tripleo_common/tests/test_inventory.py b/tripleo_common/tests/test_inventory.py new file mode 100644 index 000000000..c3ce25303 --- /dev/null +++ b/tripleo_common/tests/test_inventory.py @@ -0,0 +1,294 @@ +# -*- coding: utf-8 -*- + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from heatclient.exc import HTTPNotFound +from mock import MagicMock + +from tripleo_common.inventory import StackOutputs +from tripleo_common.inventory import TripleoInventory +from tripleo_common.tests import base + + +MOCK_ENABLED_SERVICES = { + "ObjectStorage": [ + "kernel", + "swift_storage", + "tripleo_packages" + ], + "Controller": [ + "kernel", + "keystone", + "tripleo_packages" + ], + "Compute": [ + "nova_compute", + "kernel", + "tripleo_packages" + ], + "CephStorage": [ + "kernel", + "tripleo_packages" + ], + "BlockStorage": [ + "cinder_volume", + "kernel", + "tripleo_packages" + ] +} + + +class TestInventory(base.TestCase): + def setUp(self): + super(TestInventory, self).setUp() + self.outputs_data = {'outputs': [ + {'output_key': 'EnabledServices', + 'output_value': { + 'Controller': ['a', 'b', 'c'], + 'Compute': ['d', 'e', 'f'], + 'CustomRole': ['g', 'h', 'i']}}, + {'output_key': 'KeystoneURL', + 'output_value': 'xyz://keystone'}, + {'output_key': 'ServerIdData', + 'output_value': { + 'server_ids': { + 'Controller': ['a', 'b', 'c'], + 'Compute': ['d'], + 'CustomRole': ['e']}, + 'bootstrap_server_id': 'a'}}, + {'output_key': 'RoleNetHostnameMap', + 'output_value': { + 'Controller': { + 'ctlplane': ['c-0.ctlplane.localdomain', + 'c-1.ctlplane.localdomain', + 'c-2.ctlplane.localdomain']}, + 'Compute': { + 'ctlplane': ['cp-0.ctlplane.localdomain']}, + 'CustomRole': { + 'ctlplane': ['cs-0.ctlplane.localdomain']}}}, + {'output_key': 'RoleNetIpMap', + 'output_value': { + 'Controller': { + 'ctlplane': ['x.x.x.1', + 'x.x.x.2', + 'x.x.x.3']}, + 'Compute': { + 'ctlplane': ['y.y.y.1']}, + 'CustomRole': { + 'ctlplane': ['z.z.z.1']}}}, + {'output_key': 'VipMap', + 'output_value': { + 'ctlplane': 'x.x.x.4', + 'redis': 'x.x.x.6'}}]} + self.plan_name = 'overcloud' + + self.hclient = MagicMock() + self.hclient.stacks.environment.return_value = { + 'parameter_defaults': {'AdminPassword': 'theadminpw'}} + self.mock_stack = MagicMock() + self.mock_stack.outputs = self.outputs_data['outputs'] + self.hclient.stacks.get.return_value = self.mock_stack + + self.configs = MagicMock() + self.configs.plan = self.plan_name + self.configs.auth_url = 'xyz://keystone.local' + self.configs.cacert = 'acacert' + self.configs.project_name = 'admin' + self.configs.username = 'admin' + self.configs.ansible_ssh_user = 'heat-admin' + + self.session = MagicMock() + self.session.get_token.return_value = 'atoken' + self.session.get_endpoint.return_value = 'anendpoint' + + self.outputs = StackOutputs('overcloud', self.hclient) + self.inventory = TripleoInventory( + self.configs, self.session, self.hclient) + self.inventory.stack_outputs = self.outputs + + def test_get_roles_by_service(self): + services = TripleoInventory.get_roles_by_service( + MOCK_ENABLED_SERVICES) + expected = { + 'kernel': ['BlockStorage', 'CephStorage', 'Compute', 'Controller', + 'ObjectStorage'], + 'swift_storage': ['ObjectStorage'], + 'tripleo_packages': ['BlockStorage', 'CephStorage', 'Compute', + 'Controller', 'ObjectStorage'], + 'keystone': ['Controller'], + 'nova_compute': ['Compute'], + 'cinder_volume': ['BlockStorage'], + } + self.assertDictEqual(services, expected) + + def test_outputs_are_empty_if_stack_doesnt_exist(self): + self.hclient.stacks.get.side_effect = HTTPNotFound('not found') + stack_outputs = StackOutputs('no-plan', self.hclient) + self.assertEqual(list(stack_outputs), []) + + def test_outputs_valid_key_calls_api(self): + expected = 'xyz://keystone' + self.hclient.stacks.output_show.return_value = dict(output=dict( + output_value=expected)) + self.assertEqual(expected, self.outputs['KeystoneURL']) + # This should also support the get method + self.assertEqual(expected, self.outputs.get('KeystoneURL')) + self.assertTrue(self.hclient.called_once_with('overcloud', + 'KeystoneURL')) + + def test_outputs_invalid_key_raises_keyerror(self): + self.assertRaises(KeyError, lambda: self.outputs['Invalid']) + + def test_outputs_get_method_returns_default(self): + default = 'default value' + self.assertEqual(default, self.outputs.get('Invalid', default)) + + def test_outputs_iterating_returns_list_of_output_keys(self): + self.assertEqual( + {'EnabledServices', 'KeystoneURL', 'ServerIdData', + 'RoleNetHostnameMap', 'RoleNetIpMap', 'VipMap'}, + set([o for o in self.outputs])) + + def test_inventory_list(self): + expected = {'c-0': {'hosts': ['x.x.x.1'], + 'vars': {'deploy_server_id': 'a', + 'ctlplane_ip': 'x.x.x.1', + 'enabled_networks': ['ctlplane']}}, + 'c-1': {'hosts': ['x.x.x.2'], + 'vars': {'deploy_server_id': 'b', + 'ctlplane_ip': 'x.x.x.2', + 'enabled_networks': ['ctlplane']}}, + 'c-2': {'hosts': ['x.x.x.3'], + 'vars': {'deploy_server_id': 'c', + 'ctlplane_ip': 'x.x.x.3', + 'enabled_networks': ['ctlplane']}}, + 'Compute': { + 'children': ['cp-0'], + 'vars': {'ansible_ssh_user': 'heat-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'Compute'}}, + 'Controller': { + 'children': ['c-0', 'c-1', 'c-2'], + 'vars': {'ansible_ssh_user': 'heat-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'Controller'}}, + 'cp-0': {'hosts': ['y.y.y.1'], + 'vars': {'deploy_server_id': 'd', + 'ctlplane_ip': 'y.y.y.1', + 'enabled_networks': ['ctlplane']}}, + 'cs-0': {'hosts': ['z.z.z.1'], + 'vars': {'deploy_server_id': 'e', + 'ctlplane_ip': 'z.z.z.1', + 'enabled_networks': ['ctlplane']}}, + 'CustomRole': { + 'children': ['cs-0'], + 'vars': {'ansible_ssh_user': 'heat-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'CustomRole'}}, + 'overcloud': { + 'children': ['Compute', 'Controller', 'CustomRole'], + 'vars': { + 'ctlplane_vip': 'x.x.x.4', + 'redis_vip': 'x.x.x.6'}}, + 'undercloud': { + 'hosts': ['localhost'], + 'vars': {'ansible_connection': 'local', + 'auth_url': 'xyz://keystone.local', + 'cacert': 'acacert', + 'os_auth_token': 'atoken', + 'overcloud_keystone_url': 'xyz://keystone', + 'overcloud_admin_password': 'theadminpw', + 'plan': 'overcloud', + 'project_name': 'admin', + 'undercloud_service_list': [ + 'openstack-nova-compute', + 'openstack-heat-engine', + 'openstack-ironic-conductor', + 'openstack-swift-container', + 'openstack-swift-object', + 'openstack-mistral-engine'], + 'undercloud_swift_url': 'anendpoint', + 'username': 'admin'}}} + inv_list = self.inventory.list() + for k in expected: + self.assertEqual(expected[k], inv_list[k]) + + def test_ansible_ssh_user(self): + self.configs.ansible_ssh_user = 'my-custom-admin' + self.inventory = TripleoInventory( + self.configs, self.session, self.hclient) + self.inventory.stack_outputs = self.outputs + + expected = {'c-0': {'hosts': ['x.x.x.1'], + 'vars': {'deploy_server_id': 'a', + 'ctlplane_ip': 'x.x.x.1', + 'enabled_networks': ['ctlplane']}}, + 'c-1': {'hosts': ['x.x.x.2'], + 'vars': {'deploy_server_id': 'b', + 'ctlplane_ip': 'x.x.x.2', + 'enabled_networks': ['ctlplane']}}, + 'c-2': {'hosts': ['x.x.x.3'], + 'vars': {'deploy_server_id': 'c', + 'ctlplane_ip': 'x.x.x.3', + 'enabled_networks': ['ctlplane']}}, + 'Compute': { + 'children': ['cp-0'], + 'vars': {'ansible_ssh_user': 'my-custom-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'Compute'}}, + 'Controller': { + 'children': ['c-0', 'c-1', 'c-2'], + 'vars': {'ansible_ssh_user': 'my-custom-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'Controller'}}, + 'cp-0': {'hosts': ['y.y.y.1'], + 'vars': {'deploy_server_id': 'd', + 'ctlplane_ip': 'y.y.y.1', + 'enabled_networks': ['ctlplane']}}, + 'cs-0': {'hosts': ['z.z.z.1'], + 'vars': {'deploy_server_id': 'e', + 'ctlplane_ip': 'z.z.z.1', + 'enabled_networks': ['ctlplane']}}, + 'CustomRole': { + 'children': ['cs-0'], + 'vars': {'ansible_ssh_user': 'my-custom-admin', + 'bootstrap_server_id': 'a', + 'role_name': 'CustomRole'}}, + 'overcloud': { + 'children': ['Compute', 'Controller', 'CustomRole'], + 'vars': { + 'ctlplane_vip': 'x.x.x.4', + 'redis_vip': 'x.x.x.6'}}, + 'undercloud': { + 'hosts': ['localhost'], + 'vars': {'ansible_connection': 'local', + 'auth_url': 'xyz://keystone.local', + 'cacert': 'acacert', + 'os_auth_token': 'atoken', + 'overcloud_keystone_url': 'xyz://keystone', + 'overcloud_admin_password': 'theadminpw', + 'plan': 'overcloud', + 'project_name': 'admin', + 'undercloud_service_list': [ + 'openstack-nova-compute', + 'openstack-heat-engine', + 'openstack-ironic-conductor', + 'openstack-swift-container', + 'openstack-swift-object', + 'openstack-mistral-engine'], + 'undercloud_swift_url': 'anendpoint', + 'username': 'admin'}}} + + inv_list = self.inventory.list() + for k in expected: + self.assertEqual(expected[k], inv_list[k])