Rework the test framework & the clustering test

* Remove the dead code;
* Rework the test types;
* Restore the instance connectivity check;
* Rework the clustering test to support the new node addition workflow;
* Check whether a machine where MicroStack is installed has hardware
  virtualization capabilities for different architectures. If not, use
  software emulation;
  * the host model is used with KVM since the default QEMU CPU models on
    x86_64 are subject to vulnerabilities without certain CPU-specific
    features. This conflicts with being able to use live migration
    reliably across hosts with different CPUs.
* Add a default-source-ip init argument to allow controlling the source
  IP of the installation host that will be used as a control ip or
  compute ip locally.
  * used in the clustering test so that the local host IP on the
    multipass network is used as a control IP instead of the IP
    through which the default gateway is available;
  * the IP through which the default gateway is accessible is
    used as a fallback for default-source-ip;
* Given upstream CI has a low amount of resources allocated per machine
  use LXD to set up a dummy compute node;
  * Set RLIMIT_MEMLOCK to 'unlimited' in the LXD container profile
    (see the discussion in LP: #1906280);
  * set remember_owner to 0 in qemu.conf for libvirt to avoid the
    uses of XATTRS (the root user is used anyway so there is no
    need to remember a file owner), otherwise libvirt errors out
    in an unprivileged LXD container.
* Use numeric versions of OpenStack packages in the python-packages
  section of the openstack-projects part since the resolver change in
  recent versions of pip disallows for constraints dependencies of
  packages that come from a URL or a path.
  https://github.com/pypa/pip/issues/8210
  * The newest released version of pip is always used during builds
    since snapcraft uses venv to set up virtual environments and the
    ensurepip package is invoked such that a pip version shipped with
    the distro version of python is upgraded:
    https://github.com/python/cpython/blob/3.8/Lib/venv/__init__.py#L282-L289
            cmd = [context.env_exe, '-Im', 'ensurepip', '--upgrade',
                                                    '--default-pip']
  * Environment variables are ignored when pip is installed in the venv:
    https://docs.python.org/3/using/cmdline.html#id2 (-I option)
    So there is no way to use the old pip version resolver.

Minor clustering client and add-compute changes:

* use stderr for diagnostic messages;
* use stdout to output the connection string so that it can be easily
  picked up by CLI tools without parsing.

Change-Id: I5cb3872c5d142c34da2c8b073652c67021d9ef55

snap-overlay/bin/load-modules

@@ -2,4 +2,11 @@
 
 set -ex
 
-modprobe -a vhost vhost-net vhost-scsi vhost-vsock vfio nbd dm-mod dm-thin-pool dm-snapshot iscsi-tcp target-core-mod
+# If we are not running in any type of a container, attempt to load the necessary kernel modules and
+# expect them to be present based on external arrangements (e.g. they could be specified in a LXD profile).
+# TODO: this file will go away when strict confinement gets implemented as snapd will load modules
+# that are specified in the microstack-support interface.
+if [[ `systemd-detect-virt --container` == 'none' ]]
+then
+    modprobe -a vhost vhost-net vhost-scsi vhost-vsock vfio nbd dm-mod dm-thin-pool dm-snapshot iscsi-tcp target-core-mod
+fi

snap-overlay/bin/set-default-config.py

@@ -64,7 +64,16 @@ def _get_default_config():
         'config.monitoring.ipmi': '',
         'config.services.extra.telegraf': False,
         'config.monitoring.custom-config': f'{snap_common}/etc/telegraf'
-                                           '/telegraf-microstack.conf'
+                                           '/telegraf-microstack.conf',
+
+        # Use emulation by default (with an option to override if KVM is
+        # supported).
+        'config.nova.virt-type': 'qemu',
+        # Use a host CPU model so that any CPU features enabled for
+        # vulnerability mitigation are enabled.
+        'config.nova.cpu-mode': 'host-model',
+        # Do not override cpu-models by default.
+        'config.nova.cpu-models': '',
     }
 
 

snap-overlay/etc/nova/nova.conf.d/hypervisor.conf

@@ -1,9 +0,0 @@
-[DEFAULT]
-compute_driver = libvirt.LibvirtDriver
-
-[workarounds]
-disable_rootwrap = True
-
-[libvirt]
-virt_type = kvm
-cpu_mode = host-passthrough

snap-overlay/snap-openstack.yaml

@@ -52,8 +52,9 @@ setup:
     horizon-snap.conf.j2: "{snap_common}/etc/horizon/horizon.conf.d/horizon-snap.conf"
     horizon-nginx.conf.j2: "{snap_common}/etc/nginx/snap/sites-enabled/horizon.conf"
     05_snap_tweaks.j2: "{snap_common}/etc/horizon/local_settings.d/_05_snap_tweaks.py"
-    libvirtd.conf.j2: "{snap_common}/libvirt/libvirtd.conf"
+    libvirtd.conf.j2: "{snap_common}/etc/libvirt/libvirtd.conf"
-    virtlogd.conf.j2: "{snap_common}/libvirt/virtlogd.conf"
+    qemu.conf.j2: "{snap_common}/etc/libvirt/qemu.conf"
+    virtlogd.conf.j2: "{snap_common}/etc/libvirt/virtlogd.conf"
     microstack.rc.j2: "{snap_common}/etc/microstack.rc"
     microstack.json.j2: "{snap_common}/etc/microstack.json"
     glance.conf.d.keystone.conf.j2: "{snap_common}/etc/glance/glance.conf.d/keystone.conf"
@@ -118,6 +119,9 @@ setup:
     ovn_metadata_proxy_shared_secret: 'config.credentials.ovn-metadata-proxy-shared-secret'
     setup_loop_based_cinder_lvm_backend: 'config.cinder.setup-loop-based-cinder-lvm-backend'
     lvm_backend_volume_group: 'config.cinder.lvm-backend-volume-group'
+    virt_type: 'config.nova.virt-type'
+    cpu_mode: 'config.nova.cpu-mode'
+    cpu_models: 'config.nova.cpu-models'
 entry_points:
   keystone-manage:
     binary: "{snap}/bin/keystone-manage"

snap-overlay/templates/nova-snap.conf.j2

@@ -1,4 +1,7 @@
 [DEFAULT]
+
+compute_driver = libvirt.LibvirtDriver
+
 # Set state path to writable directory
 state_path = {{ snap_common }}/lib
 # Log to systemd journal
@@ -9,6 +12,16 @@ use_journal = True
 host = {{ node_fqdn }}
 my_ip = {{ compute_ip }}
 
+[workarounds]
+disable_rootwrap = True
+
+[libvirt]
+virt_type = {{ virt_type }}
+cpu_mode = {{ cpu_mode }}
+{% if cpu_mode == 'custom' %}
+cpu_models = {{ cpu_models }}
+{% endif %}
+
 [oslo_concurrency]
 # Oslo Concurrency lock path
 lock_path = {{ snap_common }}/lock

snap-overlay/templates/qemu.conf.j2

@@ -0,0 +1,4 @@
+# Whether libvirt should remember and restore the original
+# ownership over files it is relabeling. Defaults to 1, set
+# to 0 to disable the feature.
+remember_owner = 0

snapcraft.yaml

@@ -882,7 +882,7 @@ parts:
       - -*
 
   openvswitch:
-    source: https://github.com/openvswitch/ovs/archive/v2.13.0.tar.gz
+    source: https://github.com/openvswitch/ovs/archive/v2.14.0.tar.gz
     plugin: autotools
     build-packages:
       - autoconf
@@ -1231,7 +1231,7 @@ parts:
     - --includedir=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/include
     - --oldincludedir=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/include
     - --localstatedir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common
-    - --sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common
+    - --sysconfdir=/var/snap/$SNAPCRAFT_PROJECT_NAME/common/etc/
     - DNSMASQ=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/sbin/dnsmasq
     - DMIDECODE=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/sbin/dmidecode
     - OVSVSCTL=/snap/$SNAPCRAFT_PROJECT_NAME/current/usr/local/bin/ovs-vsctl

test-requirements.txt

@@ -3,3 +3,5 @@ petname
 selenium
 stestr
 xvfbwrapper
+netifaces
+tenacity

tests/framework.py

@@ -1,55 +1,24 @@
 import logging
 import json
 import unittest
-import os
 import subprocess
-import time
+import yaml
-from typing import List
 
 import petname
+import tenacity
 from selenium import webdriver
 from selenium.webdriver.firefox.options import Options as FirefoxOptions
 from selenium.webdriver.common.by import By
 
 
 # Setup logging
-log = logging.getLogger("microstack_test")
+logger = logging.getLogger("microstack_test")
-log.setLevel(logging.DEBUG)
+logger.setLevel(logging.DEBUG)
 stream = logging.StreamHandler()
 formatter = logging.Formatter(
     '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
 stream.setFormatter(formatter)
-log.addHandler(stream)
+logger.addHandler(stream)
-
-
-def check(*args: List[str]) -> int:
-    """Execute a shell command, raising an error on failed excution.
-
-    :param args: strings to be composed into the bash call.
-
-    """
-    return subprocess.check_call(args)
-
-
-def check_output(*args: List[str]) -> str:
-    """Execute a shell command, returning the output of the command.
-
-    :param args: strings to be composed into the bash call.
-
-    Include our env; pass in any extra keyword args.
-    """
-    return subprocess.check_output(args, universal_newlines=True).strip()
-
-
-def call(*args: List[str]) -> bool:
-    """Execute a shell command.
-
-    Return True if the call executed successfully (returned 0), or
-    False if it returned with an error code (return > 0)
-
-    :param args: strings to be composed into the bash call.
-    """
-    return not subprocess.call(args)
 
 
 def gui_wrapper(func):
@@ -73,135 +42,351 @@ def gui_wrapper(func):
     return wrapper
 
 
-class Host():
+class TestHost:
-    """A host with MicroStack installed."""
 
     def __init__(self):
-        self.prefix = []
+        pass
-        self.dump_dir = '/tmp'
-        self.machine = ''
-        self.distro = os.environ.get('DISTRO') or 'bionic'
-        self.snap = os.environ.get('SNAP_FILE') or \
-            'microstack_ussuri_amd64.snap'
-        self.horizon_ip = '10.20.20.1'
-        self.host_type = 'localhost'
 
-        if os.environ.get('MULTIPASS'):
+    def destroy(self):
-            self.host_type = 'multipass'
+        raise NotImplementedError
-            print("Booting a Multipass VM ...")
-            self.multipass()
 
-        self.microstack_test()
+    def check_output(self, args, **kwargs):
+        raise NotImplementedError
 
-    def install(self, snap=None, channel='dangerous'):
+    def call(self, args, **kwargs):
-        if snap is None:
+        raise NotImplementedError
-            snap = self.snap
-        print("Installing {}".format(snap))
 
-        check(*self.prefix, 'sudo', 'snap', 'install',
+    def check_call(self, args, **kwargs):
-              '--{}'.format(channel), '--devmode', snap)
+        raise NotImplementedError
+
+    def install_snap(self, name, options):
+        self.check_output(['sudo', 'snap', 'install', name, *options])
+
+    def remove_snap(self, name, options):
+        self.check_output(['sudo', 'snap', 'remove', name, *options])
+
+    def snap_connect(self, snap_name, plug_name):
+        self.check_output(['sudo', 'snap', 'connect',
+                          f'{snap_name}:{plug_name}'])
+
+    def install_microstack(self, *, channel='edge', path=None):
+        """Install MicroStack at this host and connect relevant plugs.
+        """
+        if path is not None:
+            self.install_snap(path, ['--devmode'])
+        else:
+            self.install_snap('microstack', [f'--{channel}', '--devmode'])
 
         # TODO: add microstack-support once it is merged into snapd.
-        connections = [
+        plugs = [
-                'microstack:libvirt', 'microstack:netlink-audit',
+                'libvirt', 'netlink-audit',
-                'microstack:firewall-control', 'microstack:hardware-observe',
+                'firewall-control', 'hardware-observe',
-                'microstack:kernel-module-observe', 'microstack:kvm',
+                'kernel-module-observe', 'kvm',
-                'microstack:log-observe', 'microstack:mount-observe',
+                'log-observe', 'mount-observe',
-                'microstack:netlink-connector', 'microstack:network-observe',
+                'netlink-connector', 'network-observe',
-                'microstack:openvswitch-support', 'microstack:process-control',
+                'openvswitch-support', 'process-control',
-                'microstack:system-observe', 'microstack:network-control',
+                'system-observe', 'network-control',
-                'microstack:system-trace', 'microstack:block-devices',
+                'system-trace', 'block-devices',
-                'microstack:raw-usb'
+                'raw-usb'
         ]
-        for connection in connections:
+        for plug in plugs:
-            check('sudo', 'snap', 'connect', connection)
+            self.snap_connect('microstack', plug)
 
-    def init(self, args=['--auto']):
+    def init_microstack(self, args=['--auto']):
-        print(f"Initializing the snap with {args}")
+        self.check_call(['sudo', 'microstack', 'init', *args])
-        check(*self.prefix, 'sudo', 'microstack', 'init', *args)
 
-    def multipass(self):
+    def setup_tempest_verifier(self):
-        self.machine = petname.generate()
+        self.check_call(['sudo', 'snap', 'install', 'microstack-test'])
-        self.prefix = ['multipass', 'exec', self.machine, '--']
+        self.check_call(['sudo', 'mkdir', '-p',
+                        '/tmp/snap.microstack-test/tmp'])
+        self.check_call(['sudo', 'cp',
+                         '/var/snap/microstack/common/etc/microstack.json',
+                         '/tmp/snap.microstack-test/tmp/microstack.json'])
+        self.check_call(['microstack-test.rally', 'db', 'recreate'])
+        self.check_call([
+            'microstack-test.rally', 'deployment', 'create',
+            '--filename', '/tmp/microstack.json',
+            '--name', 'snap_generated'])
+        self.check_call(['microstack-test.tempest-init'])
 
-        check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass')
+    def run_verifications(self):
+        """Run a set of verification tests on MicroStack from this host."""
+        self.check_call([
+            'microstack-test.rally', 'verify', 'start',
+            '--load-list',
+            '/snap/microstack-test/current/2020.06-test-list.txt',
+            '--detailed', '--concurrency', '2'])
+        self.check_call([
+            'microstack-test.rally', 'verify', 'report',
+            '--type', 'json', '--to',
+            '/tmp/verification-report.json'])
+        report = json.loads(self.check_output([
+            'sudo', 'cat',
+            '/tmp/snap.microstack-test/tmp/verification-report.json']))
+        # Make sure there are no verification failures in the report.
+        failures = list(report['verifications'].values())[0]['failures']
+        return failures
 
-        check('multipass', 'launch', '--cpus', '2', '--mem', '8G', self.distro,
-              '--name', self.machine)
-        check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine))
 
-        # Figure out machine's ip
+class LocalTestHost(TestHost):
-        info = check_output('multipass', 'info', self.machine, '--format',
-                            'json')
-        info = json.loads(info)
-        self.horizon_ip = info['info'][self.machine]['ipv4'][0]
 
-    def microstack_test(self):
+    def __init__(self):
-        check('sudo', 'snap', 'install', 'microstack-test')
+        super().__init__()
+        self.install_snap('multipass', ['--stable'])
+        self.install_snap('lxd', ['--stable'])
+        self.check_call(['sudo', 'lxd', 'init', '--auto'])
 
-    def dump_logs(self):
+        try:
-        # TODO: make unique log name
+            self.run(['sudo', 'lxc', 'profile', 'show', 'microstack'],
-        if check_output('whoami') == 'zuul':
+                     stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-            self.dump_dir = "/home/zuul/zuul-output/logs"
+                     check=True)
+        except subprocess.CalledProcessError as e:
+            stderr = e.stderr.decode('utf-8')
+            if 'No such object' in stderr:
+                self._create_microstack_profile()
+                return
+            else:
+                raise RuntimeError(
+                    'An unexpected exception has occurred '
+                    f'while trying to query the profile, stderr: {stderr}'
+                ) from e
+        self.run(['sudo', 'lxc', 'profile', 'delete', 'microstack'],
+                 check=True)
+        self._create_microstack_profile()
 
-        check(*self.prefix,
+    def _create_microstack_profile(self):
-              'sudo', 'tar', 'cvzf',
+        self.run(['sudo', 'lxc', 'profile', 'create', 'microstack'],
-              '{}/dump.tar.gz'.format(self.dump_dir),
+                 stdout=subprocess.PIPE,
-              '/var/snap/microstack/common/log',
+                 stderr=subprocess.PIPE,
-              '/var/snap/microstack/common/etc',
+                 check=True)
-              '/var/log/syslog')
+        profile_conf = {
-        if 'multipass' in self.prefix:
+            'config': {'linux.kernel_modules':
-            check('multipass', 'copy-files',
+                       'iptable_nat, ip6table_nat, ebtables, openvswitch,'
-                  '{}:/tmp/dump.tar.gz'.format(self.machine), '.')
+                       'tap, vhost, vhost_net, vhost_scsi, vhost_vsock',
-        print('Saved dump.tar.gz to local working dir.')
+                       'security.nesting': 'true',
+                       'limits.kernel.memlock': 'unlimited'
+                       },
+            'devices':
+            {
+                'tun': {'path': '/dev/net/tun', 'type': 'unix-char'},
+                'vhost-net': {'path': '/dev/vhost-net', 'type': 'unix-char'},
+                'vhost-scsi': {'path': '/dev/vhost-scsi', 'type': 'unix-char'},
+                'vhost-vsock': {'path': '/dev/vhost-vsock',
+                                'type': 'unix-char'}
+            }
+        }
+        self.run(['sudo', 'lxc', 'profile', 'edit', 'microstack'],
+                 stdout=subprocess.PIPE,
+                 stderr=subprocess.PIPE,
+                 check=True,
+                 input=yaml.dump(profile_conf).encode('utf-8'))
 
-    def teardown(self):
+    def destroy(self):
-        if 'multipass' in self.prefix:
+        self.remove_snap('microstack', ['--purge'])
-            check('sudo', 'multipass', 'delete', self.machine)
+
-            check('sudo', 'multipass', 'purge')
+    def check_output(self, args, **kwargs):
-        else:
+        return subprocess.check_output(args, **kwargs).strip()
-            if call('snap', 'list', 'microstack'):
+
-                # Uninstall microstack if it is installed (it may not be).
+    def call(self, args, **kwargs):
-                check('sudo', 'snap', 'remove', '--purge', 'microstack')
+        return subprocess.call(args, **kwargs)
+
+    def check_call(self, args, **kwargs):
+        subprocess.check_call(args, **kwargs)
+
+    def run(self, args, **kwargs):
+        subprocess.run(args, **kwargs)
+
+
+class MultipassTestHost(TestHost):
+    """A virtual host set up via Multipass on a local machine."""
+
+    def __init__(self, distribution):
+        self.distribution = distribution
+        self.name = petname.generate()
+        self._launch()
+
+    def check_output(self, args, **kwargs):
+        prefix = ['sudo', 'multipass', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        return subprocess.check_output(cmd, **kwargs).strip()
+
+    def call(self, args, **kwargs):
+        prefix = ['sudo', 'multipass', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        return subprocess.call(cmd, **kwargs)
+
+    def check_call(self, args, **kwargs):
+        prefix = ['sudo', 'multipass', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        subprocess.check_call(cmd, **kwargs)
+
+    def run(self, args, **kwargs):
+        prefix = ['sudo', 'multipass', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        subprocess.run(cmd, **kwargs)
+
+    def _launch(self):
+        # Possible upstream CI resource allocation is documented here:
+        # https://docs.opendev.org/opendev/infra-manual/latest/testing.html
+        # >= 8GiB of RAM
+        # 40-80 GB of storage (possibly under /opt)
+        # Swap is not guaranteed.
+        # With m1.tiny flavor the compute node needs slightly less than 3G of
+        # RAM and 2.5G of disk space.
+        subprocess.check_call(['sudo', 'sync'])
+        subprocess.check_call(['sudo', 'sh', '-c',
+                               'echo 3 > /proc/sys/vm/drop_caches'])
+        subprocess.check_call(['sudo', 'multipass', 'launch', '--cpus', '2',
+                               '--mem', '3G', '--disk', '4G',
+                               self.distribution, '--name', self.name])
+
+        info = json.loads(subprocess.check_output(
+            ['sudo', 'multipass', 'info', self.name,
+             '--format', 'json']))
+        self.address = info['info'][self.name]['ipv4'][0]
+
+    def copy_to(self, source_path, target_path=''):
+        """Copy a file from the local machine to the Multipass VM.
+        """
+        subprocess.check_call(['sudo', 'multipass', 'copy-files', source_path,
+                               f'{self.name}:{target_path}'])
+
+    def copy_from(self, source_path, target_path):
+        """Copy a file from the Multipass VM to the local machine.
+        """
+        subprocess.check_call(['sudo', 'multipass', 'copy-files',
+                               f'{self.name}:{source_path}',
+                               target_path])
+
+    def destroy(self):
+        subprocess.check_call(['sudo', 'multipass', 'delete', self.name])
+
+
+class LXDTestHost(TestHost):
+    """A container test host set up via LXD on a local machine."""
+
+    def __init__(self, distribution):
+        self.distribution = distribution
+        self.name = petname.generate()
+        self._launch()
+
+    def check_output(self, args, **kwargs):
+        prefix = ['sudo', 'lxc', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        return subprocess.check_output(cmd, **kwargs).strip()
+
+    def call(self, args, **kwargs):
+        prefix = ['sudo', 'lxc', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        return subprocess.call(cmd, **kwargs)
+
+    def check_call(self, args, **kwargs):
+        prefix = ['sudo', 'lxc', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        subprocess.check_call(cmd, **kwargs)
+
+    def run(self, args, **kwargs):
+        prefix = ['sudo', 'lxc', 'exec', self.name, '--']
+        cmd = []
+        cmd.extend(prefix)
+        cmd.extend(args)
+        subprocess.check_call(cmd, **kwargs)
+
+    def _launch(self):
+        subprocess.check_call(['sudo', 'lxc', 'launch',
+                               f'ubuntu:{self.distribution}', self.name,
+                               '--profile', 'default',
+                               '--profile', 'microstack'])
+
+        @tenacity.retry(wait=tenacity.wait_fixed(3))
+        def fetch_addr_info():
+            info = json.loads(subprocess.check_output(
+                ['sudo', 'lxc', 'query', f'/1.0/instances/{self.name}/state']))
+            addrs = info['network']['eth0']['addresses']
+            addr_info = next(filter(lambda a: a['family'] == 'inet', addrs),
+                             None)
+            if addr_info is None:
+                raise RuntimeError('The container interface does'
+                                   ' not have an IPv4 address which'
+                                   ' is unexpected')
+            return addr_info
+        self.address = fetch_addr_info()['address']
+
+    def copy_to(self, source_path, target_path=''):
+        """Copy file or directory to the container.
+        """
+        subprocess.check_call(['sudo', 'lxc', 'file', 'push', source_path,
+                               f'{self.name}/{target_path}',
+                               '--recursive', '--create-dirs'])
+
+    def copy_from(self, source_path, target_path):
+        """Copy file or directory from the container.
+        """
+        subprocess.check_call(['sudo', 'lxc', 'file', 'pull'
+                               f'{self.name}/{source_path}', target_path,
+                               '--recursive', '--create-dirs'])
+
+    def destroy(self):
+        subprocess.check_call(['sudo', 'lxc', 'delete', self.name, '--force'])
 
 
 class Framework(unittest.TestCase):
 
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.HOSTS = []
+        self._test_hosts = []
 
-    def get_host(self):
+    def setUp(self):
-        if self.HOSTS:
+        self._localhost = LocalTestHost()
-            return self.HOSTS[0]
-        host = Host()
-        self.HOSTS.append(host)
-        return host
 
-    def add_host(self):
+    def tearDown(self):
-        host = Host()
+        for host in self._test_hosts:
-        self.HOSTS.append(host)
+            host.destroy()
-        return host
 
-    def verify_instance_networking(self, host, instance_name):
+    @property
+    def localhost(self):
+        return self._localhost
+
+    def add_multipass_host(self, distribution):
+        new_test_host = MultipassTestHost(distribution)
+        self._test_hosts.append(new_test_host)
+        return new_test_host
+
+    def add_lxd_host(self, distribution):
+        new_test_host = LXDTestHost(distribution)
+        self._test_hosts.append(new_test_host)
+        return new_test_host
+
+    def verify_instance_networking(self, test_host, instance_name):
         """Verify that we have networking on an instance
 
         We should be able to ping the instance.
 
         And we should be able to reach the Internet.
 
+        :param :class:`TestHost` test_host: The host to run the test from.
+        :param str instance_name: The name of the Nova instance to connect to.
         """
-        print("Skipping instance networking test due to bug #1852206")
+        logger.debug("Testing ping ...")
-        # TODO re-enable this test when we have fixed
-        # https://bugs.launchpad.net/microstack/+bug/1852206
-        return True
-        prefix = host.prefix
-
-        # Ping the instance
-        print("Testing ping ...")
         ip = None
-        servers = check_output(*prefix, '/snap/bin/microstack.openstack',
+        servers = test_host.check_output([
-                               'server', 'list', '--format', 'json')
+            '/snap/bin/microstack.openstack',
+            'server', 'list', '--format', 'json'
+        ])
         servers = json.loads(servers)
         for server in servers:
             if server['Name'] == instance_name:
@@ -210,56 +395,25 @@ class Framework(unittest.TestCase):
 
         self.assertTrue(ip)
 
-        pings = 1
+        test_host.call(['ping', '-i1', '-c10', '-w11', ip])
-        max_pings = 600  # ~10 minutes!
-        while not call(*prefix, 'ping', '-c1', '-w1', ip):
-            pings += 1
-            if pings > max_pings:
-                self.assertFalse(True, msg='Max pings reached!')
-
-        print("Testing instances' ability to connect to the Internet")
-        # Test Internet connectivity
-        attempts = 1
-        max_attempts = 300  # ~10 minutes!
-        username = check_output(*prefix, 'whoami')
-
-        while not call(
-                *prefix,
-                'ssh',
-                '-oStrictHostKeyChecking=no',
-                '-i', '/home/{}/.ssh/id_microstack'.format(username),
-                'cirros@{}'.format(ip),
-                '--', 'ping', '-c1', '91.189.94.250'):
-            attempts += 1
-            if attempts > max_attempts:
-                self.assertFalse(
-                    True,
-                    msg='Unable to access the Internet!')
-            time.sleep(1)
 
     @gui_wrapper
-    def verify_gui(self, host):
+    def verify_gui(self, test_host):
-        """Verify that Horizon Dashboard works
+        """Verify Horizon Dashboard operation by logging in."""
-
+        control_ip = test_host.check_output([
-        We should be able to reach the dashboard.
+            'sudo', 'snap', 'get', 'microstack', 'config.network.control-ip',
-
+        ]).decode('utf-8')
-        We should be able to login.
+        logger.debug('Verifying GUI for (IP: {})'.format(control_ip))
-
+        dashboard_port = test_host.check_output([
-        """
+            'sudo', 'snap', 'get',
-        # Test
-        print('Verifying GUI for (IP: {})'.format(host.horizon_ip))
-        # Verify that our GUI is working properly
-        dashboard_port = check_output(*host.prefix, 'sudo', 'snap', 'get',
-                                      'microstack',
-                                      'config.network.ports.dashboard')
-        keystone_password = check_output(
-            *host.prefix, 'sudo', 'snap', 'get',
             'microstack',
-            'config.credentials.keystone-password')
+            'config.network.ports.dashboard']).decode('utf-8')
-        self.driver.get("http://{}:{}/".format(
+        keystone_password = test_host.check_output([
-            host.horizon_ip,
+            'sudo', 'snap', 'get',
-            dashboard_port
+            'microstack',
-        ))
+            'config.credentials.keystone-password'
+        ]).decode('utf-8')
+        self.driver.get(f'http://{control_ip}:{dashboard_port}/')
         # Login to horizon!
         self.driver.find_element(By.ID, "id_username").click()
         self.driver.find_element(By.ID, "id_username").send_keys("admin")
@@ -269,26 +423,3 @@ class Framework(unittest.TestCase):
         # Verify that we can click something on the dashboard -- e.g.,
         # we're still not sitting at the login screen.
         self.driver.find_element(By.LINK_TEXT, "Images").click()
-
-    def setUp(self):
-        self.passed = False  # HACK: trigger (or skip) log dumps.
-
-    def tearDown(self):
-        """Clean hosts up, possibly leaving debug information behind."""
-
-        print("Tests complete. Cleaning up.")
-        while self.HOSTS:
-            host = self.HOSTS.pop()
-            if not self.passed:
-                print(
-                    "Tests failed. Leaving {} in place.".format(host.machine))
-                # Skipping log dump, due to
-                # https://bugs.launchpad.net/microstack/+bug/1860783
-                # host.dump_logs()
-                if os.environ.get('INTERACTIVE_DEBUG'):
-                    print('INTERACTIVE_DEBUG set. '
-                          'Opening a shell on test machine.')
-                    call('multipass', 'shell', host.machine)
-            else:
-                print("Tests complete. Cleaning up.")
-                host.teardown()

tests/test_basic.py

@@ -16,13 +16,11 @@ Web IDE.
 
 import os
 import sys
-import time
-import json
 import unittest
 
 sys.path.append(os.getcwd())
 
-from tests.framework import Framework, check, check_output, call  # noqa E402
+from tests.framework import Framework  # noqa E402
 
 
 class TestBasics(Framework):
@@ -34,23 +32,21 @@ class TestBasics(Framework):
         open the Horizon GUI.
 
         """
-        host = self.get_host()
+        self._localhost.install_microstack(path='microstack_ussuri_amd64.snap')
-        host.install()
+        self._localhost.init_microstack([
-        host.init([
             '--auto',
             '--control',
             '--setup-loop-based-cinder-lvm-backend',
             '--loop-device-file-size=24'
             ])
-        prefix = host.prefix
+        endpoints = self._localhost.check_output(
+            ['/snap/bin/microstack.openstack', 'endpoint', 'list']
+        ).decode('utf-8')
 
-        endpoints = check_output(
+        control_ip = self._localhost.check_output(
-            *prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list')
+            ['sudo', 'snap', 'get', 'microstack', 'config.network.control-ip'],
+        ).decode('utf-8')
 
-        control_ip = check_output(
-            *prefix, 'sudo', 'snap', 'get',
-            'microstack', 'config.network.control-ip'
-        )
         # Endpoints should contain the control IP.
         self.assertTrue(control_ip in endpoints)
 
@@ -58,64 +54,46 @@ class TestBasics(Framework):
         self.assertFalse("localhost" in endpoints)
 
         # We should be able to launch an instance
+        instance_name = 'test-instance'
         print("Testing microstack.launch ...")
-        check(*prefix, '/snap/bin/microstack.launch', 'cirros',
+        self._localhost.check_output(
-              '--name', 'breakfast', '--retry')
+            ['/snap/bin/microstack.launch', 'cirros',
-
+             '--name', instance_name, '--retry']
-        # ... and ping it
+        )
-        # Skip these tests in the gate, as they are not reliable there.
+        self.verify_instance_networking(self._localhost, instance_name)
-        # TODO: fix these in the gate!
-        if 'multipass' in prefix:
-            self.verify_instance_networking(host, 'breakfast')
-        else:
-            # Artificial wait, to allow for stuff to settle for the GUI test.
-            # TODO: get rid of this, when we drop the ping tests back int.
-            time.sleep(10)
 
         # The Horizon Dashboard should function
-        self.verify_gui(host)
+        self.verify_gui(self._localhost)
 
         # Verify that we can uninstall the snap cleanly, and that the
         # ovs bridge goes away.
 
         # Check to verify that our bridge is there.
-        self.assertTrue('br-ex' in check_output(*prefix, 'ip', 'a'))
+        self.assertTrue(
+            'br-ex' in self._localhost.check_output(
+                ['ip', 'a']).decode('utf-8'))
 
-        check(*prefix, 'sudo', 'mkdir', '-p', '/tmp/snap.microstack-test/tmp')
+        self._localhost.setup_tempest_verifier()
-        check(*prefix, 'sudo', 'cp',
-              '/var/snap/microstack/common/etc/microstack.json',
-              '/tmp/snap.microstack-test/tmp/microstack.json')
-        check(*prefix, 'microstack-test.rally', 'db', 'recreate')
-        check(*prefix, 'microstack-test.rally', 'deployment', 'create',
-              '--filename', '/tmp/microstack.json',
-              '--name', 'snap_generated')
-        check(*prefix, 'microstack-test.tempest-init')
-        check(*prefix, 'microstack-test.rally', 'verify', 'start',
-              '--load-list',
-              '/snap/microstack-test/current/2020.06-test-list.txt',
-              '--detailed', '--concurrency', '2')
-        check(*prefix, 'microstack-test.rally', 'verify', 'report',
-              '--type', 'json', '--to',
-              '/tmp/verification-report.json')
-        report = json.loads(check_output(
-            *prefix, 'sudo', 'cat',
-            '/tmp/snap.microstack-test/tmp/verification-report.json'))
         # Make sure there are no verification failures in the report.
-        failures = list(report['verifications'].values())[0]['failures']
+        failures = self._localhost.run_verifications()
         self.assertEqual(failures, 0, 'Verification tests had failure.')
 
         # Try to remove the snap without sudo.
-        self.assertFalse(
+        self.assertEqual(self._localhost.call([
-                call(*prefix, 'snap', 'remove', '--purge', 'microstack'))
+            'snap', 'remove', '--purge', 'microstack']), 1)
 
         # Retry with sudo (should succeed).
-        check(*prefix, 'sudo', 'snap', 'remove', '--purge', 'microstack')
+        self._localhost.check_call(
+            ['sudo', 'snap', 'remove', '--purge', 'microstack'])
 
         # Verify that MicroStack is gone.
-        self.assertFalse(call(*prefix, 'snap', 'list', 'microstack'))
+        self.assertEqual(self._localhost.call(
+            ['snap', 'list', 'microstack']), 1)
 
         # Verify that bridge is gone.
-        self.assertFalse('br-ex' in check_output(*prefix, 'ip', 'a'))
+        self.assertFalse(
+            'br-ex' in self._localhost.check_output(
+                ['ip', 'a']).decode('utf-8'))
 
         # We made it to the end. Set passed to True!
         self.passed = True

tests/test_cluster.py

@@ -14,104 +14,126 @@ import json
 import os
 import sys
 import unittest
+import netifaces
+import tenacity
+import logging
 
 sys.path.append(os.getcwd())
 
-from tests.framework import Framework, check, check_output, call  # noqa E402
+from tests.framework import Framework  # noqa E402
 
-
+logger = logging.getLogger(__name__)
-os.environ['MULTIPASS'] = 'true'  # TODO better way to do this.
+logger.setLevel(logging.DEBUG)
+stream = logging.StreamHandler()
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+stream.setFormatter(formatter)
+logger.addHandler(stream)
 
 
 class TestCluster(Framework):
 
     def test_cluster(self):
+        openstack_cmd = '/snap/bin/microstack.openstack'
+        control_host = self._localhost
+        control_host.install_microstack(path='microstack_ussuri_amd64.snap')
 
-        # After the setUp step, we should have a control node running
+        # Get an IP address on the lxdbr0 bridge and use it for the
-        # in a multipass vm. Let's look up its cluster password and ip
+        # control IP so that the tunnel ports of the compute node target the
-        # address.
+        # IP on the lxdbr0 subnet. Using netifaces only work for the
+        # localhost control node scenario.
+        ifaddrs = netifaces.ifaddresses('lxdbr0')[netifaces.AF_INET]
+        # We expect only one address on this interface, if multiple are present
+        # there is something wrong and we should fail the test and reassess.
+        self.assertEqual(len(ifaddrs), 1)
+        control_ip = ifaddrs[0]['addr']
 
-        openstack = '/snap/bin/microstack.openstack'
+        control_host.init_microstack(['--auto', '--control',
-        control_host = self.get_host()
+                                      f'--default-source-ip={control_ip}'])
-        control_host.install()
-        control_host.init(['--control'])
 
-        control_prefix = control_host.prefix
+        compute_host = self.add_lxd_host('focal')
-        cluster_password = check_output(*control_prefix, 'sudo', 'snap',
+        compute_host.copy_to('microstack_ussuri_amd64.snap', '/root/')
-                                        'get', 'microstack',
-                                        'config.cluster.password')
-        control_ip = check_output(*control_prefix, 'sudo', 'snap',
-                                  'get', 'microstack',
-                                  'config.network.control-ip')
 
-        self.assertTrue(cluster_password)
+        # snapd does not come up immediately in the container.
-        self.assertTrue(control_ip)
+        @tenacity.retry(wait=tenacity.wait_fixed(1),
+                        stop=tenacity.stop_after_attempt(10))
+        def wait_snapd():
+            compute_host.check_call(['sudo', 'snap', 'list'])
 
-        compute_host = self.add_host()
+        wait_snapd()
-        compute_host.install()
 
-        compute_machine = compute_host.machine
+        # wait for an IPv4 address to appear on the container interface
-        compute_prefix = compute_host.prefix
+        @tenacity.retry(wait=tenacity.wait_fixed(1),
+                        stop=tenacity.stop_after_attempt(10))
+        def wait_addr():
+            logger.debug('Checking for an eth0 interface addresses presence'
+                         ' in the container.')
+            cmd = ['ip', '-4', '-o', 'addr', 'show', 'eth0']
+            ip_out = compute_host.check_output(cmd).decode('utf-8')
+            logger.debug(f'{" ".join(cmd)} output:\n{ip_out}')
+
+        wait_addr()
+
+        compute_host.install_microstack(path='microstack_ussuri_amd64.snap')
 
         # TODO add the following to args for init
-        check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack',
+        compute_host.check_call([
-              'config.network.control-ip={}'.format(control_ip))
+            'sudo', 'snap', 'set', 'microstack',
+            f'config.network.control-ip={control_ip}'])
 
-        check(*compute_prefix, 'sudo', 'microstack.init', '--compute',
+        connection_string = control_host.check_output([
-              '--cluster-password', cluster_password, '--debug')
+            'sudo', 'microstack', 'add-compute'
+        ]).decode('utf-8')
+        self.assertTrue(connection_string)
+
+        compute_host.check_call([
+            'sudo', 'microstack.init', '--auto',
+            '--compute', '--join', connection_string, '--debug'
+        ])
 
         # Verify that our services look setup properly on compute node.
-        services = check_output(
+        services = compute_host.check_output([
-            *compute_prefix, 'systemctl', 'status', 'snap.microstack.*',
+            'systemctl', 'status', 'snap.microstack.*',
-            '--no-page')
+            '--no-page']).decode('utf-8')
 
         self.assertTrue('nova-compute' in services)
         self.assertFalse('keystone-' in services)
 
-        check(*compute_prefix, '/snap/bin/microstack.launch', 'cirros',
+        compute_fqdn = compute_host.check_output([
-              '--name', 'breakfast', '--retry',
+            'hostname', '-f']).decode('utf-8')
-              '--availability-zone', 'nova:{}'.format(compute_machine))
 
-        # TODO: verify horizon dashboard on control node.
+        instance_name = 'test-instance'
+        # Launch from the control host but schedule to the compute host.
+        control_host.check_call([
+            '/snap/bin/microstack.launch', 'cirros',
+            '--name', instance_name, '--retry',
+            '--availability-zone', f'nova:{compute_fqdn}'])
 
         # Verify endpoints
-        compute_ip = check_output(*compute_prefix, 'sudo', 'snap',
+        compute_ip = compute_host.check_output([
-                                  'get', 'microstack',
+            'sudo', 'snap',
-                                  'config.network.compute-ip')
+            'get', 'microstack',
+            'config.network.compute-ip'
+        ]).decode('utf-8')
         self.assertFalse(compute_ip == control_ip)
 
         # Ping the instance
         ip = None
-        servers = check_output(*compute_prefix, openstack,
+        servers = compute_host.check_output([
-                               'server', 'list', '--format', 'json')
+            openstack_cmd,
+            'server', 'list', '--format', 'json'
+        ]).decode('utf-8')
         servers = json.loads(servers)
         for server in servers:
-            if server['Name'] == 'breakfast':
+            if server['Name'] == instance_name:
                 ip = server['Networks'].split(",")[1].strip()
                 break
 
         self.assertTrue(ip)
 
-        pings = 1
+        control_host.check_call(['ping', '-c10', '-w11', ip])
-        max_pings = 60  # ~1 minutes
-        # Ping the machine from the control node (we don't have
-        # networking wired up for the other nodes).
-        while not call(*control_prefix, 'ping', '-c1', '-w1', ip):
-            pings += 1
-            if pings > max_pings:
-                self.assertFalse(
-                    True,
-                    msg='Max pings reached for instance on {}!'.format(
-                        compute_machine))
-
         self.passed = True
 
-        # Compute machine cleanup
-        check('sudo', 'multipass', 'delete', compute_machine)
-
 
 if __name__ == '__main__':
-    # Run our tests, ignoring deprecation warnings and warnings about
-    # unclosed sockets. (TODO: setup a selenium server so that we can
-    # move from PhantomJS, which is deprecated, to to Selenium headless.)
     unittest.main(warnings='ignore')

tools/cluster/cluster/add_compute.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import sys
 import uuid
 import secrets
 import argparse
@@ -106,7 +107,8 @@ def add_compute():
     # Print the connection string and an expiration notice to the user.
     print('Use the following connection string to add a new compute node'
           f' to the cluster (valid for {VALIDITY_PERIOD.minutes} minutes from'
-          f' this moment):\n{connection_string}')
+          f' this moment):', file=sys.stderr)
+    print(connection_string)
 
 
 def main():

tools/cluster/cluster/client.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python3
 
+import sys
 import urllib3
 import json
 
@@ -65,7 +66,8 @@ def join():
         print('An authorization failure has occurred while joining the'
               ' the cluster: please make sure the connection string'
               ' was entered as returned by the "add-compute" command'
-              ' and that it was used before its expiration time.')
+              ' and that it was used before its expiration time.',
+              file=sys.stderr)
         if response_data:
             message = json.loads(response_data)['message']
             raise UnauthorizedRequestError(message)

tools/init/init/main.py

@@ -33,6 +33,7 @@ import argparse
 import logging
 import sys
 import socket
+import ipaddress
 
 from functools import wraps
 
@@ -42,6 +43,7 @@ from init.shell import (
     check,
     check_output,
     config_set,
+    fallback_source_address,
 )
 
 from init import questions
@@ -68,6 +70,16 @@ def check_file_size_positive(value):
     return ival
 
 
+def check_source_ip_address_valid(value):
+    try:
+        addr = ipaddress.ip_address(value)
+    except ValueError as e:
+        raise argparse.ArgumentTypeError(
+            'Invalid source IP address provided in as an argument.'
+        ) from e
+    return addr
+
+
 def parse_init_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('--auto', '-a', action='store_true',
@@ -94,6 +106,13 @@ def parse_init_args():
             help=('File size in GB (10^9) of a file to be exposed as a loop'
                   ' device for the Cinder LVM backend.')
     )
+    parser.add_argument('--default-source-ip',
+                        dest='default_source_ip',
+                        type=check_source_ip_address_valid,
+                        default=fallback_source_address(),
+                        help='The IP address to be used by MicroStack'
+                             ' services as a source IP where possible. This'
+                             ' option can be useful for multi-homed nodes.')
     args = parser.parse_args()
     return args
 
@@ -124,6 +143,9 @@ def process_init_args(args):
         config_set(**{
             'config.cluster.connection-string.raw': args.connection_string})
 
+    config_set(**{
+        'config.network.default-source-ip': args.default_source_ip})
+
     if args.auto and not args.control and not args.connection_string:
         raise ValueError('The connection string parameter must be specified'
                          ' for compute nodes.')
@@ -157,7 +179,6 @@ def init() -> None:
         questions.DnsDomain(),
         questions.NetworkSettings(),
         questions.OsPassword(),  # TODO: turn this off if COMPUTE.
-        questions.ForceQemu(),
         # The following are not yet implemented:
         # questions.VmSwappiness(),
         # questions.FileHandleLimits(),

tools/init/init/questions/__init__.py

@@ -257,38 +257,6 @@ class OsPassword(ConfigQuestion):
         _env['keystone_password'] = answer
 
 
-class ForceQemu(Question):
-    _type = 'boolean'
-    config_key = 'config.host.check-qemu'
-
-    def yes(self, answer: str) -> None:
-        """Possibly force us to use qemu emulation rather than kvm."""
-
-        cpuinfo = check_output('cat', '/proc/cpuinfo')
-        if 'vmx' in cpuinfo or 'svm' in cpuinfo:
-            # We have processor extensions installed. No need to Force
-            # Qemu emulation.
-            return
-
-        _path = '{SNAP_COMMON}/etc/nova/nova.conf.d/hypervisor.conf'.format(
-            **_env)
-
-        with open(_path, 'w') as _file:
-            _file.write("""\
-[DEFAULT]
-compute_driver = libvirt.LibvirtDriver
-
-[workarounds]
-disable_rootwrap = True
-
-[libvirt]
-virt_type = qemu
-cpu_mode = host-model
-""")
-
-        # TODO: restart nova services when re-running this after init.
-
-
 class VmSwappiness(Question):
 
     _type = 'boolean'
@@ -454,6 +422,7 @@ class NovaHypervisor(Question):
 
     def yes(self, answer):
         log.info('Configuring nova compute hypervisor ...')
+        self._maybe_enable_emulation()
         enable('libvirtd')
         enable('virtlogd')
         enable('nova-compute')
@@ -464,6 +433,66 @@ class NovaHypervisor(Question):
         disable('virtlogd')
         disable('nova-compute')
 
+    def _maybe_enable_emulation(self):
+        log.info('Checking virtualization extensions presence on the host')
+        # Use KVM if it is supported, alternatively fall back to software
+        # emulation.
+        if self._is_hw_virt_supported():
+            log.info('Hardware virtualization is supported - KVM will be used'
+                     ' for Nova instances')
+            shell.config_set(**{'config.nova.virt-type': 'kvm'})
+            shell.config_set(**{'config.nova.cpu-mode': 'host-passthrough'})
+        else:
+            log.warning('Hardware virtualization is not supported - software'
+                        ' emulation will be used for Nova instances')
+            shell.config_set(**{'config.nova.virt-type': 'qemu'})
+            shell.config_set(**{'config.nova.cpu-mode': 'host-passthrough'})
+
+    @staticmethod
+    def _is_hw_virt_supported():
+        # Sample lscpu outputs: util-linux/tests/expected/lscpu/
+        cpu_info = json.loads(check_output('lscpu', '-J'))['lscpu']
+        architecture = next(filter(lambda x: x['field'] == 'Architecture:',
+                                   cpu_info), None)['data'].split()
+        flags = next(filter(lambda x: x['field'] == 'Flags:', cpu_info),
+                     None)
+        if flags is not None:
+            flags = flags['data'].split()
+
+        vendor_id = next(filter(lambda x: x['field'] == 'Vendor ID:',
+                                cpu_info), None)
+        if vendor_id is not None:
+            vendor_id = vendor_id['data']
+
+        # Mimic virt-host-validate code (from libvirt) and assume nested
+        # support on ppc64 LE or BE.
+        if architecture in ['ppc64', 'ppc64le']:
+            return True
+        elif vendor_id is not None and flags is not None:
+            if vendor_id == 'AuthenticAMD' and 'svm' in flags:
+                return True
+            elif vendor_id == 'GenuineIntel' and 'vmx' in flags:
+                return True
+            elif vendor_id == 'IBM/S390' and 'sie' in flags:
+                return True
+            elif vendor_id == 'ARM':
+                # ARM 8.3-A added nested virtualization support but it is yet
+                # to land upstream https://lwn.net/Articles/812280/ at the time
+                # of writing (Nov 2020).
+                log.warning('Nested virtualization is not supported on ARM'
+                            ' - will use emulation')
+                return False
+            else:
+                log.warning('Unable to determine hardware virtualization'
+                            f' support by CPU vendor id "{vendor_id}":'
+                            ' assuming it is not supported.')
+                return False
+        else:
+            log.warning('Unable to determine hardware virtualization support'
+                        ' by the output of lscpu: assuming it is not'
+                        ' supported')
+            return False
+
 
 class NovaSpiceConsoleSetup(Question):
     """Run the Spice HTML5 console proxy service"""

tools/init/init/questions/clustering.py

@@ -2,13 +2,14 @@ import logging
 import msgpack
 import re
 import netaddr
+import sys
 
 from cryptography.hazmat.primitives import hashes
 from typing import Tuple
 
 from init.questions.question import Question, InvalidAnswer
 from init.shell import (
-    fetch_ip_address,
+    default_source_address,
     config_get,
     config_set,
 )
@@ -39,7 +40,8 @@ class Role(Question):
             if role in self._valid_roles:
                 return role
 
-            print('The role must be either "control" or "compute".')
+            print('The role must be either "control" or "compute".',
+                  file=sys.stderr)
 
         raise InvalidAnswer('Too many failed attempts.')
 
@@ -58,7 +60,8 @@ class ConnectionString(Question):
         except TypeError:
             print('The connection string contains non-ASCII'
                   ' characters please make sure you entered'
-                  ' it as returned by the add-compute command.')
+                  ' it as returned by the add-compute command.',
+                  file=sys.stderr)
             return answer, False
 
         try:
@@ -66,24 +69,28 @@ class ConnectionString(Question):
         except msgpack.exceptions.ExtraData:
             print('The connection string contains extra data'
                   ' characters please make sure you entered'
-                  ' it as returned by the add-compute command.')
+                  ' it as returned by the add-compute command.',
+                  file=sys.stderr)
             return answer, False
         except ValueError:
             print('The connection string contains extra data'
                   ' characters please make sure you entered'
-                  ' it as returned by the add-compute command.')
+                  ' it as returned by the add-compute command.',
+                  file=sys.stderr)
             return answer, False
         except msgpack.exceptions.FormatError:
             print('The connection string format is invalid'
                   ' please make sure you entered'
-                  ' it as returned by the add-compute command.')
+                  ' it as returned by the add-compute command.',
+                  file=sys.stderr)
             return answer, False
         except Exception:
             print('An unexpeted error has occured while trying'
                   ' to decode the connection string. Please'
                   ' make sure you entered it as returned by'
                   ' the add-compute command and raise an'
-                  ' issue if the error persists')
+                  ' issue if the error persists',
+                  file=sys.stderr)
             return answer, False
 
         # Perform token field validation as well so that the rest of
@@ -103,7 +110,7 @@ class ConnectionString(Question):
                 self._validate_hostname(hostname)
             except ValueError as e:
                 print(f'The hostname {hostname} provided in the connection'
-                      f' string is invalid: {str(e)}')
+                      f' string is invalid: {str(e)}', file=sys.stderr)
                 return answer, False
 
         fingerprint = conn_info.get('fingerprint')
@@ -111,7 +118,8 @@ class ConnectionString(Question):
             self._validate_fingerprint(fingerprint)
         except ValueError as e:
             print('The clustering service TLS certificate fingerprint provided'
-                  f' in the connection string is invalid: {str(e)}')
+                  f' in the connection string is invalid: {str(e)}',
+                  file=sys.stderr)
             return answer, False
 
         credential_id = conn_info.get('id')
@@ -119,7 +127,7 @@ class ConnectionString(Question):
             self._validate_credential_id(credential_id)
         except ValueError as e:
             print('The credential id provided in the connection string is'
-                  f' invalid: {str(e)}')
+                  f' invalid: {str(e)}', file=sys.stderr)
             return answer, False
 
         credential_secret = conn_info.get('secret')
@@ -127,7 +135,7 @@ class ConnectionString(Question):
             self._validate_credential_secret(credential_secret)
         except ValueError as e:
             print('The credential secret provided in the connection string is'
-                  f' invalid: {str(e)}')
+                  f' invalid: {str(e)}', file=sys.stderr)
             return answer, False
 
         self._conn_info = conn_info
@@ -224,7 +232,7 @@ class ControlIp(Question):
 
     def _load(self):
         if config_get(Role.config_key) == 'control':
-            return fetch_ip_address() or super()._load()
+            return default_source_address() or super()._load()
         return super()._load()
 
     def ask(self):
@@ -245,7 +253,7 @@ class ComputeIp(Question):
     def _load(self):
         role = config_get(Role.config_key)
         if role == 'compute':
-            return fetch_ip_address() or super().load()
+            return default_source_address() or super().load()
 
         return super()._load()
 

tools/init/init/shell.py

@@ -199,7 +199,13 @@ def download(url: str, output: str) -> None:
     wget.download(url, output)
 
 
-def fetch_ip_address():
+def fallback_source_address():
+    '''Get an ip address through which the default gateway is accessible.
+
+    Note that Linux kernel allows multiple default gateways to be present with
+    the same or different metrics which may lead to unexpected behaviors. This
+    situation is unlikely but needs to be taken into account.
+    '''
     try:
         interface = netifaces.gateways()['default'][netifaces.AF_INET][1]
         return netifaces.ifaddresses(interface)[netifaces.AF_INET][0]['addr']
@@ -208,6 +214,11 @@ def fetch_ip_address():
         return None
 
 
+def default_source_address():
+    '''Get a default source address.'''
+    return config_get('config.network.default-source-ip')
+
+
 def default_network():
     """Get info about the default netowrk.
 

tox.ini

@@ -28,6 +28,7 @@ commands =
     # Specify tests in sequence, as they can't run in parallel if not
     # using multipass.
     bash -c "unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY ; {toxinidir}/tests/test_basic.py"
+    bash -c "unset http_proxy https_proxy HTTP_PROXY HTTPS_PROXY ; {toxinidir}/tests/test_cluster.py"
 
 [testenv:multipass]
 # Default testing environment for a human operated machine. Builds the