openstack
/
tripleo-ansible
Code Issues Proposed changes

Generalize ceph_spec_bootstrap inventory parsing 

Create a generic helper class in module_utils to parse
inventory yaml. Similar to the ansible-core InventoryManager
but usable from ansible modules with minimal dependencies
(just python stdlib and yaml).

This should be compatible with any current/past tripleo inventory
structures and potentially future inventory structure changes (within
reason).

Related-bug: #1998649
Change-Id: Id2fb39054bd5f81deb967ed9938c613a2ca6ea2c
This commit is contained in:
Oliver Walsh 2022-12-01 23:18:10 +00:00
parent 45b0fc27cb
commit 92c59b3f99
6 changed files with 433 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

191
tripleo_ansible/ansible_plugins/module_utils/tripleo_inventory_manager.py Normal file
View File

@ -0,0 +1,191 @@
#!/usr/bin/env python
# Copyright (c) 2022 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import deque
import copy
import weakref
import yaml
class TripleoInventoryHost:
def __init__(self, name):
self.name = name
self.vars = {}
self._groups = {}
def add_group(self, group):
self._groups[group.name] = weakref.ref(group)
def __str__(self):
return self.name
def __repr__(self):
return "{}('{}')".format(
self.__class__.__name__,
self.name
)
@property
def groups(self):
return {n: r() for n, r in self._groups.items()}
def resolve_vars(self):
"""Returns the resulting vars for this host including group vars"""
vars = copy.deepcopy(self.vars)
for group in self.groups.values():
group_vars = copy.deepcopy(group.vars)
group_vars.update(vars)
vars = group_vars
return vars
class TripleoInventoryGroup:
def __init__(self, name):
self.name = name
self._hosts = {}
self.vars = {}
self._children = {}
self._parents = {}
def add_parent(self, group):
if self in group.get_ancestors():
raise RuntimeError(
"Adding group '{}' as parent of '{}' creates a recursive dependency loop".format(
group.name, self.name
)
)
self._parents[group.name] = weakref.ref(group)
def add_child(self, group):
if self in group.get_descendants():
raise RuntimeError(
"Adding group '{}' as child of '{}' creates a recursive dependency loop".format(
group.name, self.name
)
)
self._children[group.name] = weakref.ref(group)
def add_host(self, host):
self._hosts[host.name] = weakref.ref(host)
def get_ancestors(self):
ancestors = set(self.parents.values())
parents_todo = deque(self.parents.values())
while parents_todo:
parent = parents_todo.popleft()
parent_ancestors = parent.parents.values()
ancestors.update(parent_ancestors)
parents_todo.extend(parent_ancestors)
return list(ancestors)
def get_descendants(self):
descendants = set(self.children.values())
children_todo = deque(self.children.values())
while children_todo:
child = children_todo.popleft()
child_descendents = child.children.values()
descendants.update(child_descendents)
children_todo.extend(child_descendents)
return list(descendants)
def __str__(self):
return self.name
def __repr__(self):
return "{}('{}')".format(
self.__class__.__name__,
self.name
)
@property
def hosts(self):
return {n: r() for n, r in self._hosts.items()}
@property
def children(self):
return {n: r() for n, r in self._children.items()}
@property
def parents(self):
return {n: r() for n, r in self._parents.items()}
class TripleoInventoryManager:
"""Container class for a tree of inventory hosts/groups"""
def __init__(self, inventory_file=None):
self.host = {}
self.groups = {}
if inventory_file is not None:
self.parse_inventory_file(inventory_file)
def parse_inventory_file(self, inventory_file):
"""Parse a yaml ansible inventory file"""
self.hosts, self.groups = self._read_yaml_inventory(inventory_file)
def get_hosts(self, groupname):
"""Return a list of host objects for the given group"""
hosts = []
groups = deque([])
if groupname in self.groups:
groups.append(self.groups[groupname])
while groups:
group = groups.popleft()
hosts += group.hosts.values()
groups.extend(group.children.values())
return hosts
def _read_yaml_inventory(self, inventory_file):
with open(inventory_file) as in_file:
data = yaml.safe_load(in_file)
return self._parse_inventory(data)
def _parse_inventory(self, data):
groups = {}
hosts = {}
# Initialize groups_to_parse
# As child groups are encountered they will be appended
groups_to_parse = deque([
# (name, parent, data)
(group_name, None, group_data) for group_name, group_data in data.items()
])
# Build the tree.
# InventoryManager holds a reference to all object so can just use weak-refs
# for the cyclic parent/child relationships
while groups_to_parse:
group_name, parent_group, group_data = groups_to_parse.popleft()
group = groups.setdefault(group_name, TripleoInventoryGroup(group_name))
if parent_group is not None:
group.add_parent(parent_group)
parent_group.add_child(group)
group_hosts = group_data.get('hosts', {})
if isinstance(group_hosts, str):
host_name = group_hosts
host = hosts.setdefault(host_name, TripleoInventoryHost(host_name))
host.add_group(group)
group.add_host(host)
else:
for host_name, host_data in group_hosts.items():
host = hosts.setdefault(host_name, TripleoInventoryHost(host_name))
if host_data is not None:
host.vars.update(host_data)
host.add_group(group)
group.add_host(host)
group.vars.update(group_data.get('vars', {}))
for child_name, child_data in group_data.get('children', {}).items():
groups_to_parse.append((child_name, group, child_data))
return hosts, groups

75
tripleo_ansible/ansible_plugins/modules/ceph_spec_bootstrap.py
View File

@ -24,6 +24,10 @@ try:
from ansible.module_utils import ceph_spec
except ImportError:
from tripleo_ansible.ansible_plugins.module_utils import ceph_spec
try:
from ansible.module_utils import tripleo_inventory_manager
except ImportError:
from tripleo_ansible.ansible_plugins.module_utils import tripleo_inventory_manager
ANSIBLE_METADATA = {
@ -187,26 +191,19 @@ def get_inventory_hosts_to_ips(inventory, roles, tld, fqdn=False):
Then the hosts of the CephStorage group (from the roles list)
are ['ceph-0'] because overcloud_CephStorage is a child group.
Does not handle if one group has both children and hosts, but only
needs to handle the types of groups generated in tripleo inventory.
"""
hosts_to_ips = {}
for key in inventory:
if key in roles:
if 'children' in inventory[key] and 'hosts' not in inventory[key]:
# e.g. if CephStorage has children (e.g. overcloud_CephStorage)
# then set the key to overcloud_CephStorage so we get its hosts
key = [k for k, v in inventory[key]['children'].items()][0]
if 'hosts' in inventory[key]:
for host in inventory[key]['hosts']:
ip = inventory[key]['hosts'][host]['ansible_host']
if fqdn:
hostname = inventory[key]['hosts'][host]['canonical_hostname']
else:
hostname = host
if tld:
hostname += "." + tld
hosts_to_ips[hostname] = ip
for role in roles:
role_hosts = inventory.get_hosts(groupname=role)
for host in role_hosts:
ip = host.vars['ansible_host']
if fqdn:
hostname = host.vars['canonical_hostname']
else:
hostname = host.name
if tld:
hostname += "." + tld
hosts_to_ips[hostname] = ip
return hosts_to_ips
@ -255,17 +252,18 @@ def get_inventory_roles_to_hosts(inventory, roles, tld, fqdn=False):
Uses ansible inventory as source
"""
roles_to_hosts = {}
for key in inventory:
if key in roles:
roles_to_hosts[key] = []
for host in inventory[key]['hosts']:
if fqdn:
hostname = inventory[key]['hosts'][host]['canonical_hostname']
else:
hostname = host
if tld:
hostname += "." + tld
roles_to_hosts[key].append(hostname)
for role in roles:
roles_to_hosts[role] = []
role_hosts = inventory.get_hosts(groupname=role)
for host in role_hosts:
if fqdn:
hostname = host.vars['canonical_hostname']
else:
hostname = host.name
if tld:
hostname += "." + tld
roles_to_hosts[role].append(hostname)
return roles_to_hosts
@ -322,10 +320,13 @@ def get_roles_to_svcs_from_inventory(inventory):
for ceph_name in ceph_list:
ceph_services.append(ceph_name)
inverse_service_map[ceph_name] = tripleo_name
for key in inventory:
key_rename = key.replace('ceph_', '')
for group in inventory.groups.values():
key_rename = group.name.replace('ceph_', '')
if key_rename in ceph_services:
for role in inventory[key]['children'].keys():
for child_group in group.get_descendants():
role = child_group.vars.get('tripleo_role_name')
if role is None:
continue
if role in roles_to_services.keys():
roles_to_services[role].append(inverse_service_map[key_rename])
else:
@ -575,8 +576,9 @@ def main():
roles_to_svcs.keys(), tld)
hosts_to_ips = get_deployed_hosts_to_ips(deployed_metalsmith, tld)
elif method == 'tripleo_ansible_inventory':
with open(tripleo_ansible_inventory, 'r') as stream:
inventory = yaml.safe_load(stream)
inventory = tripleo_inventory_manager.TripleoInventoryManager(
tripleo_ansible_inventory
)
roles_to_svcs = get_roles_to_svcs_from_inventory(inventory)
roles_to_hosts = get_inventory_roles_to_hosts(inventory,
roles_to_svcs.keys(),
@ -588,8 +590,9 @@ def main():
roles_to_svcs = get_roles_to_svcs_from_roles(tripleo_roles)
roles_to_hosts = get_deployed_roles_to_hosts(deployed_metalsmith,
roles_to_svcs.keys(), tld)
with open(tripleo_ansible_inventory, 'r') as stream:
inventory = yaml.safe_load(stream)
inventory = tripleo_inventory_manager.TripleoInventoryManager(
tripleo_ansible_inventory
)
hosts_to_ips = get_inventory_hosts_to_ips(inventory,
roles_to_svcs.keys(),
tld, fqdn)

2
tripleo_ansible/roles/tripleo_cephadm/molecule/default/mock/mock_inventory.yml
View File

@ -6,6 +6,8 @@ Standalone:
canonical_hostname: standalone.localdomain
ctlplane_hostname: standalone.ctlplane.localdomain
ctlplane_ip: 192.168.24.1
vars:
tripleo_role_name: Standalone
ceph_osd:
children:
Standalone: {}

0
tripleo_ansible/tests/module_utils/__init__.py Normal file
View File

191
tripleo_ansible/tests/module_utils/test_tripleo_inventory_manager.py Normal file
View File

@ -0,0 +1,191 @@
# Copyright 2022 Red Hat, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Test tripleo_inventory_manager module"""
import os
import tripleo_common.tests
from tripleo_ansible.ansible_plugins.module_utils import tripleo_inventory_manager
from tripleo_ansible.tests import base as tests_base
class TestTripleoInventoryHost(tests_base.TestCase):
"""Test the TripleoInventoryHost class"""
def test_basic(self):
"""Basic construction test"""
h = tripleo_inventory_manager.TripleoInventoryHost('foobar')
self.assertEqual('foobar', str(h))
self.assertEqual("TripleoInventoryHost('foobar')", repr(h))
self.assertEqual({}, h.groups)
self.assertEqual({}, h.vars)
def test_add_group(self):
"""Test adding a host to a group creates a weak reference"""
h = tripleo_inventory_manager.TripleoInventoryHost('foobar')
class FakeGroup:
name = 'bar'
g = FakeGroup()
h.add_group(g)
self.assertEqual({'bar': g}, h.groups)
del g
self.assertEqual({'bar': None}, h.groups)
class TestTripleoInventoryGroup(tests_base.TestCase):
"""Test the TripleoInventoryGroup class"""
def test_basic(self):
"""Basic construction test"""
g = tripleo_inventory_manager.TripleoInventoryGroup('foobar')
self.assertEqual('foobar', str(g))
self.assertEqual("TripleoInventoryGroup('foobar')", repr(g))
self.assertEqual({}, g.hosts)
self.assertEqual({}, g.children)
def test_add_host(self):
""""Test adding a host to a group creates a weak reference"""
class FakeHost:
name = 'bar'
h = FakeHost()
g = tripleo_inventory_manager.TripleoInventoryGroup('foobar')
g.add_host(h)
self.assertEqual({'bar': h}, g.hosts)
del h
self.assertEqual({'bar': None}, g.hosts)
def test_add_child(self):
"""Test adding a child to a group creates a weak reference"""
g1 = tripleo_inventory_manager.TripleoInventoryGroup('foo')
g2 = tripleo_inventory_manager.TripleoInventoryGroup('bar')
g1.add_child(g2)
g2.add_parent(g1)
self.assertEqual({'bar': g2}, g1.children)
del g2
self.assertEqual({'bar': None}, g1.children)
def test_add_parent(self):
"""Test adding a partent to a group creates a weak reference"""
g1 = tripleo_inventory_manager.TripleoInventoryGroup('foo')
g2 = tripleo_inventory_manager.TripleoInventoryGroup('bar')
g2.add_child(g1)
g1.add_parent(g2)
self.assertEqual({'bar': g2}, g1.parents)
del g2
self.assertEqual({'bar': None}, g1.parents)
def test_get_descendents(self):
"""Test get_descendents removes duplicates"""
# Create a diamond dependancy graph
g1 = tripleo_inventory_manager.TripleoInventoryGroup('foo')
g2 = tripleo_inventory_manager.TripleoInventoryGroup('bar')
g3 = tripleo_inventory_manager.TripleoInventoryGroup('baz')
g4 = tripleo_inventory_manager.TripleoInventoryGroup('bez')
g1.add_child(g2)
g2.add_parent(g1)
g1.add_child(g3)
g3.add_parent(g1)
g2.add_child(g4)
g4.add_parent(g2)
g3.add_child(g4)
g4.add_parent(g3)
self.assertEqual(set([g2, g3, g4]), set(g1.get_descendants()))
def test_add_child_cyclic_dependency(self):
"""Test add_child cyclic dependency"""
g1 = tripleo_inventory_manager.TripleoInventoryGroup('foo')
g2 = tripleo_inventory_manager.TripleoInventoryGroup('bar')
g1.add_child(g2)
g2.add_parent(g1)
self.assertRaises(RuntimeError, g2.add_child, g1)
def test_add_parent_cyclic_dependency(self):
"""Test add_child cyclic dependency"""
g1 = tripleo_inventory_manager.TripleoInventoryGroup('foo')
g2 = tripleo_inventory_manager.TripleoInventoryGroup('bar')
g1.add_child(g2)
g2.add_parent(g1)
self.assertRaises(RuntimeError, g1.add_parent, g2)
class TestTripleoInventoryManager(tests_base.TestCase):
"""Test the TripleoInventoryManager class"""
def test_parse_simple_inventory(self):
inv_data = {
'all': {
'children': {
'more': {
'hosts': {
'host1': {},
'host2': {'foo': 'bar'}
}
},
'and_more': {
'hosts': {
'host2': {}
},
'vars': {
'foo': 'two',
'bar': 'three'
}
}
},
'vars': {
'bar': 'baz'
},
'hosts': 'host3' # ansible accepts this format too
}
}
i = tripleo_inventory_manager.TripleoInventoryManager()
h, g = i._parse_inventory(inv_data)
self.assertEqual(set(['host1', 'host2', 'host3']), set(h.keys()))
self.assertEqual({'bar': 'three', 'foo': 'bar'}, h['host2'].resolve_vars())
def _get_tripleo_common_test_inv_path(self, filename):
return os.path.join(os.path.dirname(tripleo_common.tests.__file__), 'inventory_data', filename)
def test_real_inventory_old_style(self):
i = tripleo_inventory_manager.TripleoInventoryManager(self._get_tripleo_common_test_inv_path('overcloud_static.yaml'))
self.assertEqual(set(['overcloud-controller-0', 'overcloud-novacompute-0', 'undercloud']), set(i.hosts.keys()))
self.assertEqual(set(['overcloud-controller-0', 'overcloud-novacompute-0']), set([h.name for h in i.get_hosts('tuned')]))
self.assertEqual(set(['overcloud-controller-0']), set([h.name for h in i.get_hosts('nova_scheduler')]))
self.assertEqual(set(['overcloud-novacompute-0']), set([h.name for h in i.get_hosts('nova_libvirt')]))
self.assertEqual('Controller', i.hosts['overcloud-controller-0'].resolve_vars().get('tripleo_role_name'))
self.assertEqual('Compute', i.hosts['overcloud-novacompute-0'].resolve_vars().get('tripleo_role_name'))
def test_real_inventory_multistack_style(self):
i = tripleo_inventory_manager.TripleoInventoryManager(self._get_tripleo_common_test_inv_path('merged_static.yaml'))
self.assertEqual(
set(['overcloud-controller-0', 'overcloud-novacompute-0', 'cell1-cellcontrol-0', 'cell1-compute-0', 'undercloud']),
set(i.hosts.keys())
)
self.assertEqual(
set(['overcloud-controller-0', 'overcloud-novacompute-0', 'cell1-cellcontrol-0', 'cell1-compute-0']),
set([h.name for h in i.get_hosts('tuned')])
)
self.assertEqual(set(['overcloud-controller-0', 'overcloud-novacompute-0']), set([h.name for h in i.get_hosts('overcloud')]))
self.assertEqual(set(['cell1-cellcontrol-0', 'cell1-compute-0']), set([h.name for h in i.get_hosts('cell1')]))
self.assertEqual(set(['overcloud-controller-0']), set([h.name for h in i.get_hosts('nova_scheduler')]))
self.assertEqual(set(['overcloud-controller-0', 'cell1-cellcontrol-0']), set([h.name for h in i.get_hosts('nova_conductor')]))
self.assertEqual(set(['overcloud-novacompute-0', 'cell1-compute-0']), set([h.name for h in i.get_hosts('nova_libvirt')]))
self.assertEqual('Controller', i.hosts['overcloud-controller-0'].resolve_vars().get('tripleo_role_name'))
self.assertEqual('Compute', i.hosts['overcloud-novacompute-0'].resolve_vars().get('tripleo_role_name'))
self.assertEqual('CellController', i.hosts['cell1-cellcontrol-0'].resolve_vars().get('tripleo_role_name'))
self.assertEqual('Compute', i.hosts['cell1-compute-0'].resolve_vars().get('tripleo_role_name'))

14
tripleo_ansible/tests/modules/test_ceph_spec_bootstrap.py
View File

@ -19,6 +19,10 @@ import socket
import tempfile
import yaml
try:
from ansible.module_utils import tripleo_inventory_manager
except ImportError:
from tripleo_ansible.ansible_plugins.module_utils import tripleo_inventory_manager
from tripleo_ansible.ansible_plugins.modules import ceph_spec_bootstrap
from tripleo_ansible.tests import base as tests_base
@ -217,8 +221,9 @@ class TestCephSpecBootstrap(tests_base.TestCase):
ceph_service_types = ['mon', 'mgr', 'osd']
inventory_file = "roles/tripleo_cephadm/molecule/default/mock/mock_inventory.yml"
tld = ""
with open(inventory_file, 'r') as stream:
inventory = yaml.safe_load(stream)
inventory = tripleo_inventory_manager.TripleoInventoryManager(
inventory_file
)
roles_to_svcs = ceph_spec_bootstrap.get_roles_to_svcs_from_inventory(inventory)
expected = {'Standalone': ['CephOSD', 'CephMgr', 'CephMon']}
self.assertEqual(roles_to_svcs, expected)
@ -272,8 +277,9 @@ class TestCephSpecBootstrap(tests_base.TestCase):
ceph_service_types = ['mon', 'mgr', 'osd']
inventory_file = "roles/tripleo_cephadm/molecule/default/mock/mock_inventory.yml"
tld = "abc.local"
with open(inventory_file, 'r') as stream:
inventory = yaml.safe_load(stream)
inventory = tripleo_inventory_manager.TripleoInventoryManager(
inventory_file
)
roles_to_svcs = ceph_spec_bootstrap.get_roles_to_svcs_from_inventory(inventory)
expected = {'Standalone': ['CephOSD', 'CephMgr', 'CephMon']}
self.assertEqual(roles_to_svcs, expected)