From 298993d06a89b7835afc7187b63fd1a5402024bc Mon Sep 17 00:00:00 2001 From: Federico Ressi Date: Tue, 1 Oct 2019 17:16:59 +0200 Subject: [PATCH] Create OpenStack topology API to interact with cloud nodes Change-Id: I0e1e071c7b4fc70793d44eddf9b1eb1e094a0912 --- tobiko/config.py | 1 + tobiko/openstack/topology/__init__.py | 24 ++ tobiko/openstack/topology/_exception.py | 24 ++ tobiko/openstack/topology/_topology.py | 260 ++++++++++++++++++ tobiko/openstack/topology/config.py | 42 +++ tobiko/shell/ping/_statistics.py | 22 +- .../functional/openstack/test_topology.py | 65 +++++ 7 files changed, 431 insertions(+), 7 deletions(-) create mode 100644 tobiko/openstack/topology/__init__.py create mode 100644 tobiko/openstack/topology/_exception.py create mode 100644 tobiko/openstack/topology/_topology.py create mode 100644 tobiko/openstack/topology/config.py create mode 100644 tobiko/tests/functional/openstack/test_topology.py diff --git a/tobiko/config.py b/tobiko/config.py index fb66d7078..44215e6e1 100644 --- a/tobiko/config.py +++ b/tobiko/config.py @@ -34,6 +34,7 @@ CONFIG_MODULES = ['tobiko.openstack.glance.config', 'tobiko.openstack.neutron.config', 'tobiko.openstack.nova.config', 'tobiko.openstack.os_faults.config', + 'tobiko.openstack.topology.config', 'tobiko.shell.ssh.config', 'tobiko.shell.ping.config', 'tobiko.shell.sh.config', diff --git a/tobiko/openstack/topology/__init__.py b/tobiko/openstack/topology/__init__.py new file mode 100644 index 000000000..3e61c1e4f --- /dev/null +++ b/tobiko/openstack/topology/__init__.py @@ -0,0 +1,24 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +from tobiko.openstack.topology import _exception +from tobiko.openstack.topology import _topology + + +NoSuchOpenStackTopologyGroup = _exception.NoSuchOpenStackTopologyGroup +NoSuchOpenStackTopologyNode = _exception.NoSuchOpenStackTopologyNode + +get_openstack_topology = _topology.get_openstack_topology +OpenStackTopology = _topology.OpenStackTopology diff --git a/tobiko/openstack/topology/_exception.py b/tobiko/openstack/topology/_exception.py new file mode 100644 index 000000000..4605fb28e --- /dev/null +++ b/tobiko/openstack/topology/_exception.py @@ -0,0 +1,24 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import tobiko + + +class NoSuchOpenStackTopologyNode(tobiko.TobikoException): + message = "No such topology node: {name!r}" + + +class NoSuchOpenStackTopologyGroup(tobiko.TobikoException): + message = "No such topology group: {name!r}" diff --git a/tobiko/openstack/topology/_topology.py b/tobiko/openstack/topology/_topology.py new file mode 100644 index 000000000..9446a98c7 --- /dev/null +++ b/tobiko/openstack/topology/_topology.py @@ -0,0 +1,260 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import weakref +import socket + +import netaddr +from oslo_log import log + +import tobiko +from tobiko.shell import ping +from tobiko.shell import sh +from tobiko.shell import ssh +from tobiko.openstack import nova +from tobiko.openstack.topology import _exception + +LOG = log.getLogger(__name__) + + +def get_openstack_topology(topology_class=None): + topology_class = topology_class or DEFAULT_TOPOLOGY_CLASS + return tobiko.setup_fixture(DEFAULT_TOPOLOGY_CLASS) + + +def list_openstack_nodes(): + return get_openstack_topology().nodes + + +def list_openstack_node_groups(): + return get_openstack_topology().groups + + +def get_default_openstack_topology_class(): + return DEFAULT_TOPOLOGY_CLASS + + +def set_default_openstack_topology_class(topology_class): + # pylint: disable=global-statement + global DEFAULT_TOPOLOGY_CLASS + if not issubclass(topology_class, OpenStackTopology): + message = "{!r} is not subclass of OpenStackTopology".format( + topology_class) + raise TypeError(message) + DEFAULT_TOPOLOGY_CLASS = topology_class + + +class OpenStackTopologyElement(object): + + def __init__(self, topology, name): + self._topology = weakref.ref(topology) + self.name = name + + @property + def topology(self): + return self._topology() + + def __repr__(self): + return "{cls!s}(topology={topology!r}, name={name!r})".format( + cls=type(self).__name__, + topology=self.topology, + name=self.name) + + +class OpenStackTopologyNode(OpenStackTopologyElement): + + def __init__(self, topology, name, ssh_client=None): + super(OpenStackTopologyNode, self).__init__(topology=topology, + name=name) + self._addresses = list() + self._groups = set() + self._ssh_client = ssh_client + + def add_group(self, name): + self._groups.add(name) + + @property + def groups(self): + return tobiko.select(self.topology.get_group(name) + for name in sorted(self._groups)) + + def add_address(self, address): + self._addresses.append(netaddr.IPAddress(address)) + + @property + def addresses(self): + return tobiko.select(address + for address in self._addresses) + + @property + def ssh_client(self): + ssh_client = self._ssh_client + if ssh_client is None: + self._ssh_client = ssh_client = self.topology.get_ssh_client( + host=self.addresses.first) + return ssh_client + + +class OpenStackTopologyNodeGroup(OpenStackTopologyElement): + + def __init__(self, topology, name): + super(OpenStackTopologyNodeGroup, self).__init__(topology=topology, + name=name) + self._nodes = set() + + def add_node(self, name): + self._nodes.add(name) + + @property + def nodes(self): + return tobiko.select(self.topology.get_node(name) + for name in sorted(self._nodes)) + + +class OpenStackTopologyConfig(tobiko.SharedFixture): + + conf = None + + def setup_fixture(self): + from tobiko import config + CONF = config.CONF + self.conf = CONF.tobiko.topology + + +class OpenStackTopology(tobiko.SharedFixture): + + _nodes = None + _groups = None + config = tobiko.required_setup_fixture(OpenStackTopologyConfig) + + def setup_fixture(self): + self.clear_nodes() + self.discover_nodes() + + def clear_nodes(self): + self._nodes = {} + self._groups = {} + + def discover_nodes(self): + self.discover_configured_nodes() + self.discover_compute_nodes() + + def discover_configured_nodes(self): + for host in self.config.conf.nodes or []: + self.add_node(address=host) + + def discover_compute_nodes(self): + for hypervisor in nova.list_hypervisors(): + address = hypervisor.host_ip + if not ping.ping(address).received: + LOG.warning("Cannot reach hypervisor IP address %r from host " + "%r", hypervisor.host_ip, socket.gethostname()) + address = None + self.add_node(address=address, + hostname=node_name_from_hostname( + hypervisor.hypervisor_hostname), + group_names=['compute']) + + def add_node(self, name=None, address=None, hostname=None, + group_names=None, ssh_client=None): + name = name or self.get_node_name(hostname=hostname, + address=address, + ssh_client=ssh_client) + node = self.create_node(name=name, ssh_client=ssh_client) + if address: + node.add_address(address=address) + + for group_name in group_names or []: + self.add_group(name=group_name, node_names=[name]) + return node + + def create_node(self, name, **kwargs): + try: + return self.get_node(name=name) + except _exception.NoSuchOpenStackTopologyNode: + self._nodes[name] = node = self.new_node(name=name, **kwargs) + return node + + def get_node(self, name): + try: + return self._nodes[name] + except KeyError: + raise _exception.NoSuchOpenStackTopologyNode(name=name) + + def new_node(self, name, **kwargs): + return OpenStackTopologyNode(topology=self, name=name, **kwargs) + + @property + def nodes(self): + return tobiko.select(self.get_node(name) + for name in sorted(self._nodes)) + + def get_ssh_client(self, host, username=None, port=None, key_filename=None, + **ssh_parameters): + if not host: + message = "Invalid host address: {!r}".format(host) + raise ValueError(message) + conf = self.config.conf + return ssh.ssh_client(host=str(host), + username=(username or conf.username), + port=(port or conf.port), + key_filename=(key_filename or conf.key_file), + **ssh_parameters) + + def add_group(self, name, node_names=None): + group = self.create_group(name=name) + for node_name in node_names or []: + group.add_node(name=node_name) + self.get_node(name=node_name).add_group(name=name) + + def create_group(self, name): + try: + return self.get_group(name=name) + except _exception.NoSuchOpenStackTopologyGroup: + self._groups[name] = group = self.new_group(name=name) + return group + + def get_group(self, name): + try: + return self._groups[name] + except KeyError: + raise _exception.NoSuchOpenStackTopologyGroup(name=name) + + def new_group(self, name): + return OpenStackTopologyNodeGroup(topology=self, name=name) + + @property + def groups(self): + return tobiko.select(self.get_group(name) + for name in sorted(self._groups)) + + def get_node_name(self, hostname=None, address=None, ssh_client=None): + if address and not hostname: + ssh_client = ssh_client or self.get_ssh_client(host=address) + hostname = sh.get_hostname(ssh_client=ssh_client) + if hostname: + return node_name_from_hostname(hostname=hostname) + + message = ("Unable to get node name: hostname={!r}, " + "address={!r}").format( + hostname, address) + raise ValueError(message) + + +DEFAULT_TOPOLOGY_CLASS = OpenStackTopology + + +def node_name_from_hostname(hostname): + return hostname.split('.', 1)[0].lower() diff --git a/tobiko/openstack/topology/config.py b/tobiko/openstack/topology/config.py new file mode 100644 index 000000000..db22cbd0b --- /dev/null +++ b/tobiko/openstack/topology/config.py @@ -0,0 +1,42 @@ +# Copyright 2019 Red Hat +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import itertools + +from oslo_config import cfg + +GROUP_NAME = "topology" +OPTIONS = [ + cfg.ListOpt('nodes', + default=None, + help="List of hostname nodes"), + cfg.StrOpt('key_file', + default='~/.ssh/id_rsa', + help="Default SSH key to login to cloud nodes"), + cfg.StrOpt('username', + default=None, + help="Default username for SSH login"), + cfg.StrOpt('port', + default=None, + help="Default port for SSH login"), +] + + +def register_tobiko_options(conf): + conf.register_opts(group=cfg.OptGroup(GROUP_NAME), opts=OPTIONS) + + +def list_options(): + return [(GROUP_NAME, itertools.chain(OPTIONS))] diff --git a/tobiko/shell/ping/_statistics.py b/tobiko/shell/ping/_statistics.py index 1e698b1b2..fd9b7f0a2 100644 --- a/tobiko/shell/ping/_statistics.py +++ b/tobiko/shell/ping/_statistics.py @@ -173,20 +173,28 @@ class PingStatistics(object): def assert_transmitted(self): if not self.transmitted: - tobiko.fail("Any package has been transmitted to %(destination)r", + tobiko.fail("{transmitted!r} package(s) has been transmitted " + "to {destination!r}", + transmitted=self.transmitted, destination=self.destination) def assert_not_transmitted(self): if self.transmitted: - tobiko.fail("Some packages has been transmitted to " - "%(destination)r", destination=self.destination) + tobiko.fail("{transmitted!r} package(s) has been transmitted to " + "{destination!r}", + transmitted=self.transmitted, + destination=self.destination) def assert_replied(self): if not self.received: - tobiko.fail("Any reply package has been received from " - "%(destination)r", destination=self.destination) + tobiko.fail("{received!r} reply package(s) has been received from " + "{destination!r}", + received=self.received, + destination=self.destination) def assert_not_replied(self): if self.received: - tobiko.fail("Some reply packages has been received from " - "%(destination)r", destination=self.destination) + tobiko.fail("{received!r} reply package(s) has been received from " + "{destination!r}", + received=self.received, + destination=self.destination) diff --git a/tobiko/tests/functional/openstack/test_topology.py b/tobiko/tests/functional/openstack/test_topology.py new file mode 100644 index 000000000..17180573b --- /dev/null +++ b/tobiko/tests/functional/openstack/test_topology.py @@ -0,0 +1,65 @@ +# Copyright (c) 2019 Red Hat, Inc. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import absolute_import + +import netaddr + +import testtools + +import tobiko +from tobiko.openstack import nova +from tobiko.openstack import topology +from tobiko.shell import ping +from tobiko.shell import sh + + +class OpenStackTopologyTest(testtools.TestCase): + + topology = tobiko.required_setup_fixture(topology.OpenStackTopology) + + def test_get_openstack_topology(self): + topology_class = type(self.topology) + topo = topology.get_openstack_topology(topology_class=topology_class) + self.assertIsInstance(topo, topology.OpenStackTopology) + self.assertNotEqual([], list(topo.nodes)) + self.assertNotEqual([], list(topo.groups)) + + def test_ping_node(self): + topo = topology.get_openstack_topology() + for node in topo.nodes: + for address in node.addresses: + ping.ping(address, count=1).assert_replied() + + def test_ssh_client(self): + for node in self.topology.nodes: + hostname = sh.get_hostname( + ssh_client=node.ssh_client).split('.')[0] + self.assertEqual(node.name, hostname) + + def test_compute_group(self): + group = self.topology.get_group('compute') + nodes = group.nodes + hypervisors = {hypervisor.hypervisor_hostname.split('.')[0]: hypervisor + for hypervisor in nova.list_hypervisors()} + self.assertEqual(sorted(hypervisors), + sorted([node.name for node in nodes])) + for name, hypervisor in hypervisors.items(): + node = self.topology.get_node(name) + self.assertEqual(name, node.name) + if ping.ping(hypervisor.host_ip).received: + self.assertIn(netaddr.IPAddress(hypervisor.host_ip), + node.addresses) + self.assertIn(group, node.groups)