420 lines
14 KiB
Python
420 lines
14 KiB
Python
# Copyright 2019 Red Hat
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
from __future__ import absolute_import
|
|
|
|
import collections
|
|
import socket
|
|
import typing # noqa
|
|
import weakref
|
|
|
|
|
|
import netaddr
|
|
from oslo_log import log
|
|
import six
|
|
from six.moves.urllib import parse
|
|
|
|
import tobiko
|
|
from tobiko import docker
|
|
from tobiko import podman
|
|
from tobiko.shell import ip
|
|
from tobiko.shell import ping
|
|
from tobiko.shell import sh
|
|
from tobiko.shell import ssh
|
|
from tobiko.openstack import nova
|
|
from tobiko.openstack import keystone
|
|
from tobiko.openstack.topology import _exception
|
|
|
|
|
|
LOG = log.getLogger(__name__)
|
|
|
|
|
|
DEFAULT_TOPOLOGY_CLASS = (
|
|
'tobiko.openstack.topology._topology.OpenStackTopology')
|
|
|
|
|
|
def get_openstack_topology(topology_class=None):
|
|
# type: (typing.Any) -> OpenStackTopology
|
|
topology_class = topology_class or get_default_openstack_topology_class()
|
|
return tobiko.setup_fixture(topology_class)
|
|
|
|
|
|
def list_openstack_nodes(topology=None, group=None, hostnames=None, **kwargs):
|
|
topology = topology or get_openstack_topology()
|
|
if group:
|
|
nodes = topology.get_group(group=group)
|
|
else:
|
|
nodes = topology.nodes
|
|
if hostnames:
|
|
names = {node_name_from_hostname(hostname)
|
|
for hostname in hostnames}
|
|
nodes = [node
|
|
for node in nodes
|
|
if node.name in names]
|
|
if kwargs:
|
|
nodes = nodes.with_attributes(**kwargs)
|
|
return nodes
|
|
|
|
|
|
def find_openstack_node(topology=None, unique=False, **kwargs):
|
|
nodes = list_openstack_nodes(topology=topology, **kwargs)
|
|
if unique:
|
|
return nodes.unique
|
|
else:
|
|
return nodes.first
|
|
|
|
|
|
def get_openstack_node(hostname, address=None, topology=None):
|
|
topology = topology or get_openstack_topology()
|
|
return topology.get_node(hostname=hostname, address=address)
|
|
|
|
|
|
def list_openstack_node_groups(topology=None):
|
|
topology = topology or get_openstack_topology()
|
|
return topology.groups
|
|
|
|
|
|
def get_default_openstack_topology_class():
|
|
# type: () -> typing.Any
|
|
return DEFAULT_TOPOLOGY_CLASS
|
|
|
|
|
|
def set_default_openstack_topology_class(topology_class):
|
|
# pylint: disable=global-statement
|
|
global DEFAULT_TOPOLOGY_CLASS
|
|
DEFAULT_TOPOLOGY_CLASS = topology_class
|
|
|
|
|
|
class OpenStackTopologyNode(object):
|
|
|
|
_docker_client = None
|
|
_podman_client = None
|
|
|
|
def __init__(self, topology, name, public_ip, ssh_client):
|
|
self._topology = weakref.ref(topology)
|
|
self.name = name
|
|
self.public_ip = public_ip
|
|
self.ssh_client = ssh_client
|
|
self.groups = set()
|
|
|
|
@property
|
|
def topology(self):
|
|
return self._topology()
|
|
|
|
def add_group(self, group):
|
|
self.groups.add(group)
|
|
|
|
@property
|
|
def ssh_parameters(self):
|
|
return self.ssh_client.setup_connect_parameters()
|
|
|
|
@property
|
|
def docker_client(self):
|
|
docker_client = self._docker_client
|
|
if not docker_client:
|
|
self._docker_client = docker_client = docker.get_docker_client(
|
|
ssh_client=self.ssh_client)
|
|
return docker_client
|
|
|
|
@property
|
|
def podman_client(self):
|
|
podman_client = self._podman_client
|
|
if not podman_client:
|
|
self._podman_client = podman_client = podman.get_podman_client(
|
|
ssh_client=self.ssh_client)
|
|
return podman_client
|
|
|
|
def __repr__(self):
|
|
return "{cls!s}<name={name!r}>".format(cls=type(self).__name__,
|
|
name=self.name)
|
|
|
|
|
|
class OpenStackTopologyConfig(tobiko.SharedFixture):
|
|
|
|
conf = None
|
|
|
|
def setup_fixture(self):
|
|
from tobiko import config
|
|
CONF = config.CONF
|
|
self.conf = CONF.tobiko.topology
|
|
|
|
|
|
class OpenStackTopology(tobiko.SharedFixture):
|
|
|
|
config = tobiko.required_setup_fixture(OpenStackTopologyConfig)
|
|
|
|
def __init__(self):
|
|
super(OpenStackTopology, self).__init__()
|
|
self._reachable_ips = set()
|
|
self._unreachable_ips = set()
|
|
self._nodes_by_name = collections.OrderedDict()
|
|
self._nodes_by_ips = collections.OrderedDict()
|
|
self._nodes_by_group = collections.OrderedDict()
|
|
|
|
def setup_fixture(self):
|
|
self.discover_nodes()
|
|
|
|
def cleanup_fixture(self):
|
|
self._reachable_ips.clear()
|
|
self._unreachable_ips.clear()
|
|
self._nodes_by_name.clear()
|
|
self._nodes_by_ips.clear()
|
|
self._nodes_by_group.clear()
|
|
|
|
def discover_nodes(self):
|
|
self.discover_configured_nodes()
|
|
self.discover_controller_nodes()
|
|
self.discover_compute_nodes()
|
|
|
|
def discover_configured_nodes(self):
|
|
for address in self.config.conf.nodes or []:
|
|
self.add_node(address=address)
|
|
|
|
def discover_controller_nodes(self):
|
|
endpoints = keystone.list_endpoints(interface='public')
|
|
addresses = set(parse.urlparse(endpoint.url).hostname
|
|
for endpoint in endpoints)
|
|
for address in addresses:
|
|
self.add_node(address=address, group='controller')
|
|
|
|
def discover_compute_nodes(self):
|
|
for hypervisor in nova.list_hypervisors():
|
|
self.add_node(hostname=hypervisor.hypervisor_hostname,
|
|
address=hypervisor.host_ip,
|
|
group='compute')
|
|
|
|
def add_node(self, hostname=None, address=None, group=None,
|
|
ssh_client=None):
|
|
name = hostname and node_name_from_hostname(hostname) or None
|
|
ips = set()
|
|
if address:
|
|
ips.update(self._ips(address))
|
|
if hostname:
|
|
ips.update(self._ips(hostname))
|
|
ips = tobiko.select(ips)
|
|
|
|
try:
|
|
node = self.get_node(name=name, address=ips)
|
|
except _exception.NoSuchOpenStackTopologyNode:
|
|
node = self._add_node(hostname=hostname, ips=ips,
|
|
ssh_client=ssh_client)
|
|
|
|
if node and group:
|
|
self.add_group(group=group).append(node)
|
|
node.add_group(group=group)
|
|
return node
|
|
|
|
def _add_node(self, ips, hostname=None, ssh_client=None):
|
|
public_ip = self._public_ip(ips, ssh_client=ssh_client)
|
|
if public_ip is None:
|
|
LOG.debug("Unable to SSH connect to any node IP address: %s"
|
|
','.join(str(ip_address) for ip_address in ips))
|
|
return None
|
|
|
|
# I need to get a name for the new node
|
|
ssh_client = ssh_client or self._ssh_client(public_ip)
|
|
hostname = hostname or sh.get_hostname(ssh_client=ssh_client)
|
|
name = node_name_from_hostname(hostname)
|
|
try:
|
|
node = self._nodes_by_name[name]
|
|
except KeyError:
|
|
self._nodes_by_name[name] = node = self.create_node(
|
|
name=name, public_ip=public_ip, ssh_client=ssh_client)
|
|
other = self._nodes_by_ips.setdefault(public_ip, node)
|
|
if node is not other:
|
|
LOG.error("Two nodes have the same IP address (%s): %r, %r",
|
|
public_ip, node.name, other.name)
|
|
return node
|
|
|
|
def get_node(self, name=None, hostname=None, address=None):
|
|
name = name or (hostname and node_name_from_hostname(hostname))
|
|
details = {}
|
|
if name:
|
|
tobiko.check_valid_type(name, six.string_types)
|
|
details['name'] = name
|
|
try:
|
|
return self._nodes_by_name[name]
|
|
except KeyError:
|
|
pass
|
|
if address:
|
|
details['address'] = address
|
|
for ip_address in self._ips(address):
|
|
try:
|
|
return self._nodes_by_ips[ip_address]
|
|
except KeyError:
|
|
pass
|
|
raise _exception.NoSuchOpenStackTopologyNode(details=details)
|
|
|
|
def create_node(self, name, public_ip, ssh_client, **kwargs):
|
|
return OpenStackTopologyNode(topology=self, name=name,
|
|
public_ip=public_ip,
|
|
ssh_client=ssh_client, **kwargs)
|
|
|
|
@property
|
|
def nodes(self):
|
|
return tobiko.select(self.get_node(name)
|
|
for name in self._nodes_by_name)
|
|
|
|
def add_group(self, group):
|
|
try:
|
|
return self._nodes_by_group[group]
|
|
except KeyError:
|
|
self._nodes_by_group[group] = nodes = self.create_group()
|
|
return nodes
|
|
|
|
def create_group(self):
|
|
return tobiko.Selection()
|
|
|
|
def get_group(self, group):
|
|
try:
|
|
return self._nodes_by_group[group]
|
|
except KeyError:
|
|
raise _exception.NoSuchOpenStackTopologyNodeGroup(group=group)
|
|
|
|
@property
|
|
def groups(self):
|
|
return list(self._nodes_by_group)
|
|
|
|
def _ssh_client(self, address, username=None, port=None,
|
|
key_filename=None, **ssh_parameters):
|
|
username = username or self.config.conf.username
|
|
port = port or self.config.conf.port
|
|
key_filename = key_filename or self.config.conf.key_file
|
|
return ssh.ssh_client(host=str(address),
|
|
username=username,
|
|
key_filename=key_filename,
|
|
**ssh_parameters)
|
|
|
|
def _public_ip(self, ips, ssh_client=None):
|
|
reachable_ip = self._reachable_ip(ips)
|
|
if reachable_ip:
|
|
return reachable_ip
|
|
|
|
if not ssh_client:
|
|
# Try connecting via other nodes to get target node IP
|
|
# addresses
|
|
proxy_client = None
|
|
for proxy_node in self.nodes:
|
|
proxy_client = proxy_node.ssh_client
|
|
if proxy_client:
|
|
internal_ip = self._reachable_ip(ips,
|
|
proxy_client=proxy_client)
|
|
if internal_ip:
|
|
ssh_client = self._ssh_client(
|
|
internal_ip, proxy_client=proxy_client)
|
|
break
|
|
if ssh_client:
|
|
break
|
|
|
|
if ssh_client:
|
|
# Connect via SSH to to get target node IP addresses
|
|
ips = self._ips_from_host(ssh_client=ssh_client)
|
|
reachable_ip = self._reachable_ip(ips)
|
|
if reachable_ip:
|
|
return reachable_ip
|
|
|
|
LOG.warning('Unable to reach remote host via any IP address: %s',
|
|
', '.join(str(a) for a in ips))
|
|
return None
|
|
|
|
def _reachable_ip(self, ips, proxy_client=None, **kwargs):
|
|
reachable = None
|
|
if proxy_client:
|
|
untested_ips = ips
|
|
else:
|
|
# Exclude unreachable addresses
|
|
untested_ips = list()
|
|
for address in ips:
|
|
if address not in self._unreachable_ips:
|
|
if address in self._reachable_ips:
|
|
# Will take result from the first one of marked already
|
|
# marked as reachable
|
|
reachable = reachable or address
|
|
else:
|
|
# Will later search for results between the other IPs
|
|
untested_ips.append(address)
|
|
|
|
for address in untested_ips:
|
|
if reachable is None:
|
|
try:
|
|
received = ping.ping(address, count=1, timeout=5.,
|
|
ssh_client=proxy_client,
|
|
**kwargs).received
|
|
except ping.PingFailed:
|
|
pass
|
|
else:
|
|
if received:
|
|
reachable = address
|
|
# Mark IP as reachable
|
|
self._reachable_ips.add(address)
|
|
continue
|
|
|
|
# Mark IP as unreachable
|
|
self._unreachable_ips.add(address)
|
|
|
|
return reachable
|
|
|
|
@property
|
|
def ip_version(self):
|
|
ip_version = self.config.conf.ip_version
|
|
return ip_version and int(ip_version) or None
|
|
|
|
def _ips_from_host(self, **kwargs):
|
|
return ip.list_ip_addresses(ip_version=self.ip_version,
|
|
scope='global', **kwargs)
|
|
|
|
def _ips(self, obj):
|
|
if isinstance(obj, tobiko.Selection):
|
|
ips = obj
|
|
elif isinstance(obj, netaddr.IPAddress):
|
|
ips = tobiko.select([obj])
|
|
elif isinstance(obj, six.string_types):
|
|
try:
|
|
ips = tobiko.select([netaddr.IPAddress(obj)])
|
|
except (netaddr.AddrFormatError, ValueError):
|
|
ips = resolve_host_ips(obj)
|
|
else:
|
|
for item in iter(obj):
|
|
tobiko.check_valid_type(item, netaddr.IPAddress)
|
|
ips = tobiko.select(obj)
|
|
|
|
if ips and self.ip_version:
|
|
ips = ips.with_attributes(version=self.ip_version)
|
|
return ips
|
|
|
|
|
|
def resolve_host_ips(host, port=0):
|
|
tobiko.check_valid_type(host, six.string_types)
|
|
LOG.debug('Calling getaddrinfo with host %r', host)
|
|
ips = tobiko.Selection()
|
|
try:
|
|
addrinfo = socket.getaddrinfo(host, port, 0, 0,
|
|
socket.AI_CANONNAME | socket.IPPROTO_TCP)
|
|
except socket.gaierror:
|
|
LOG.exception('Error calling getaddrinfo for host %r', host)
|
|
else:
|
|
for _, _, _, canonical_name, sockaddr in addrinfo:
|
|
try:
|
|
ips.append(netaddr.IPAddress(sockaddr[0]))
|
|
except netaddr.AddrFormatError as ex:
|
|
LOG.error("Invalid sockaddr for host %r: %r -> %r (%s)",
|
|
host, canonical_name, sockaddr, ex)
|
|
else:
|
|
LOG.debug("IP address for host %r: %r -> %r",
|
|
host, canonical_name, sockaddr)
|
|
return ips
|
|
|
|
|
|
def node_name_from_hostname(hostname):
|
|
return hostname.split('.', 1)[0].lower()
|