Added refresh tests

Refactored test framework so that we have more flexibility in terms of
installing various versions of microstack before and after running
some tests. Moved in class "globals" into per instance variables,
to avoid broken cases with incomplete cleanup.

Added test_refresh.py, plus matching env in tox.

Refresh tests will fail currently, because we have some pending issues
that break refreshes. Fixing those is a subject for a different
commit.

Refactored cluster_test.py and control_test.py to use new framework.
Should (and do) pass.

Framework now cleans up multipass hosts regardless of whether or not
the tests passed. Leaning on the .tar.gz for local troubleshooting
helps us make it better for in gate troubleshooting.

Change-Id: I6a45b39132f5959c2944fe1ebbe10f71408ee777
This commit is contained in:
Pete Vander Giessen 2019-11-14 16:21:11 +00:00
parent 2f5847d6ad
commit 960685b91e
6 changed files with 292 additions and 167 deletions

View File

@ -3,9 +3,13 @@ import json
import unittest
import os
import subprocess
import time
import xvfbwrapper
from typing import List
import petname
from selenium import webdriver
from selenium.webdriver.common.by import By
# Setup logging
@ -48,85 +52,198 @@ def call(*args: List[str]) -> bool:
return not subprocess.call(args)
class Framework(unittest.TestCase):
def gui_wrapper(func):
"""Start up selenium drivers, run a test, then tear them down."""
PREFIX = []
DUMP_DIR = '/tmp'
MACHINE = ''
DISTRO = 'bionic'
SNAP = 'microstack_stein_amd64.snap'
HORIZON_IP = '10.20.20.1'
INIT_FLAG = 'auto'
def wrapper(cls, *args, **kwargs):
def install_snap(self, channel='dangerous', snap=None):
# Setup Selenium Driver
cls.display = xvfbwrapper.Xvfb(width=1280, height=720)
cls.display.start()
cls.driver = webdriver.PhantomJS()
# Run function
try:
return func(cls, *args, **kwargs)
finally:
# Tear down driver
cls.driver.quit()
cls.display.stop()
return wrapper
class Host():
"""A host with MicroStack installed."""
def __init__(self):
self.prefix = []
self.dump_dir = '/tmp'
self.machine = ''
self.distro = 'bionic'
self.snap = 'microstack_stein_amd64.snap'
self.horizon_ip = '10.20.20.1'
self.host_type = 'localhost'
if os.environ.get('MULTIPASS'):
self.host_type = 'multipass'
print("Booting a Multipass VM ...")
self.multipass()
def install(self, snap=None, channel='dangerous'):
if snap is None:
snap = self.SNAP
snap = self.snap
print("Installing {}".format(snap))
check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic',
check(*self.prefix, 'sudo', 'snap', 'install', '--classic',
'--{}'.format(channel), snap)
def init_snap(self, flag='auto'):
check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag))
def init(self, flag='auto'):
print("Initializing the snap with --{}".format(flag))
check(*self.prefix, 'sudo', 'microstack.init', '--{}'.format(flag))
def multipass(self):
self.MACHINE = petname.generate()
self.PREFIX = ['multipass', 'exec', self.MACHINE, '--']
distro = os.environ.get('DISTRO') or self.DISTRO
self.machine = petname.generate()
self.prefix = ['multipass', 'exec', self.machine, '--']
distro = os.environ.get('distro') or self.distro
check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass')
check('multipass', 'launch', '--cpus', '2', '--mem', '8G', distro,
'--name', self.MACHINE)
check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE))
'--name', self.machine)
check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine))
# Figure out machine's ip
info = check_output('multipass', 'info', self.MACHINE, '--format',
info = check_output('multipass', 'info', self.machine, '--format',
'json')
info = json.loads(info)
self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0]
self.horizon_ip = info['info'][self.machine]['ipv4'][0]
def dump_logs(self):
# TODO: make unique log name
if check_output('whoami') == 'zuul':
self.DUMP_DIR = "/home/zuul/zuul-output/logs"
self.dump_dir = "/home/zuul/zuul-output/logs"
check(*self.PREFIX,
check(*self.prefix,
'sudo', 'tar', 'cvzf',
'{}/dump.tar.gz'.format(self.DUMP_DIR),
'{}/dump.tar.gz'.format(self.dump_dir),
'/var/snap/microstack/common/log',
'/var/snap/microstack/common/etc',
'/var/log/syslog')
if 'multipass' in self.PREFIX:
if 'multipass' in self.prefix:
check('multipass', 'copy-files',
'{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.')
'{}:/tmp/dump.tar.gz'.format(self.machine), '.')
print('Saved dump.tar.gz to local working dir.')
def setUp(self):
self.passed = False # HACK: trigger (or skip) cleanup.
if os.environ.get('MULTIPASS'):
print("Booting a Multipass VM ...")
self.multipass()
print("Installing {}".format(self.SNAP))
self.install_snap()
print("Initializing the snap with --{}".format(self.INIT_FLAG))
self.init_snap(self.INIT_FLAG)
def tearDown(self):
"""Either dump logs in the case of failure, or clean up."""
if not self.passed:
# Skip teardown in the case of failures, so that we can
# inspect them.
# TODO: I'd like to use the errors and failures list in
# the test result, but I was having trouble getting to it
# from this routine. Need to do more digging and possibly
# elimiate the self.passed hack.
print("Tests failed. Dumping logs and exiting.")
return self.dump_logs()
print("Tests complete. Tearing down.")
if 'multipass' in self.PREFIX:
check('sudo', 'multipass', 'delete', self.MACHINE)
def teardown(self):
if 'multipass' in self.prefix:
check('sudo', 'multipass', 'delete', self.machine)
check('sudo', 'multipass', 'purge')
else:
check('sudo', 'snap', 'remove', '--purge', 'microstack')
class Framework(unittest.TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.HOSTS = []
def get_host(self):
if self.HOSTS:
return self.HOSTS[0]
host = Host()
self.HOSTS.append(host)
return host
def add_host(self):
host = Host()
self.HOSTS.append(host)
return host
def verify_instance_networking(self, host, instance_name):
"""Verify that we have networking on an instance
We should be able to ping the instance.
And we should be able to reach the Internet.
"""
prefix = host.prefix
# Ping the instance
print("Testing ping ...")
ip = None
servers = check_output(*prefix, '/snap/bin/microstack.openstack',
'server', 'list', '--format', 'json')
servers = json.loads(servers)
for server in servers:
if server['Name'] == instance_name:
ip = server['Networks'].split(",")[1].strip()
break
self.assertTrue(ip)
pings = 1
max_pings = 600 # ~10 minutes!
while not call(*prefix, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(True, msg='Max pings reached!')
print("Testing instances' ability to connect to the Internet")
# Test Internet connectivity
attempts = 1
max_attempts = 300 # ~10 minutes!
username = check_output(*prefix, 'whoami')
while not call(
*prefix,
'ssh',
'-oStrictHostKeyChecking=no',
'-i', '/home/{}/.ssh/id_microstack'.format(username),
'cirros@{}'.format(ip),
'--', 'ping', '-c1', '91.189.94.250'):
attempts += 1
if attempts > max_attempts:
self.assertFalse(
True,
msg='Unable to access the Internet!')
time.sleep(1)
@gui_wrapper
def verify_gui(self, host):
"""Verify that Horizon Dashboard works
We should be able to reach the dashboard.
We should be able to login.
"""
# Test
print('Verifying GUI for (IP: {})'.format(host.horizon_ip))
# Verify that our GUI is working properly
self.driver.get("http://{}/".format(host.horizon_ip))
# Login to horizon!
self.driver.find_element(By.ID, "id_username").click()
self.driver.find_element(By.ID, "id_username").send_keys("admin")
self.driver.find_element(By.ID, "id_password").send_keys("keystone")
self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
# Verify that we can click something on the dashboard -- e.g.,
# we're still not sitting at the login screen.
self.driver.find_element(By.LINK_TEXT, "Images").click()
def setUp(self):
self.passed = False # HACK: trigger (or skip) cleanup.
def tearDown(self):
"""Clean hosts up, possibly leaving debug information behind."""
print("Tests complete. Cleaning up.")
while self.HOSTS:
host = self.HOSTS.pop()
if not self.passed:
print("Dumping logs for {}".format(host.machine))
host.dump_logs()
host.teardown()

View File

@ -14,14 +14,10 @@ Web IDE.
"""
import json
import os
import sys
import time
import unittest
import xvfbwrapper
from selenium import webdriver
from selenium.webdriver.common.by import By
sys.path.append(os.getcwd())
@ -30,20 +26,6 @@ from tests.framework import Framework, check, check_output, call # noqa E402
class TestBasics(Framework):
def setUp(self):
super(TestBasics, self).setUp()
# Setup Selenium Driver
self.display = xvfbwrapper.Xvfb(width=1280, height=720)
self.display.start()
self.driver = webdriver.PhantomJS()
def tearDown(self):
# Tear down selenium driver
self.driver.quit()
self.display.stop()
super(TestBasics, self).tearDown()
def test_basics(self):
"""Basic test
@ -51,16 +33,13 @@ class TestBasics(Framework):
open the Horizon GUI.
"""
launch = '/snap/bin/microstack.launch'
openstack = '/snap/bin/microstack.openstack'
print("Testing microstack.launch ...")
check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast',
'--retry')
host = self.get_host()
host.install()
host.init()
prefix = host.prefix
endpoints = check_output(
*self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list')
*prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list')
# Endpoints should be listening on 10.20.20.1
self.assertTrue("10.20.20.1" in endpoints)
@ -68,66 +47,23 @@ class TestBasics(Framework):
# Endpoints should not contain localhost
self.assertFalse("localhost" in endpoints)
if 'multipass' in self.PREFIX:
# Verify that microstack.launch completed successfully
# Skip these tests in the gate, as they are not reliable there.
# TODO: fix these in the gate!
# Ping the instance
ip = None
servers = check_output(*self.PREFIX, openstack,
'server', 'list', '--format', 'json')
servers = json.loads(servers)
for server in servers:
if server['Name'] == 'breakfast':
ip = server['Networks'].split(",")[1].strip()
break
self.assertTrue(ip)
pings = 1
max_pings = 600 # ~10 minutes!
while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(True, msg='Max pings reached!')
print("Testing instances' ability to connect to the Internet")
# Test Internet connectivity
attempts = 1
max_attempts = 300 # ~10 minutes!
username = check_output(*self.PREFIX, 'whoami')
while not call(
*self.PREFIX,
'ssh',
'-oStrictHostKeyChecking=no',
'-i', '/home/{}/.ssh/id_microstack'.format(username),
'cirros@{}'.format(ip),
'--', 'ping', '-c1', '91.189.94.250'):
attempts += 1
if attempts > max_attempts:
self.assertFalse(
True,
msg='Unable to access the Internet!')
time.sleep(1)
# We should be able to launch an instance
print("Testing microstack.launch ...")
check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'breakfast', '--retry')
# ... and ping it
# Skip these tests in the gate, as they are not reliable there.
# TODO: fix these in the gate!
if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')
else:
# Artificial wait, to allow for stuff to settle for the GUI test.
# TODO: get rid of this, when we drop the ping tests back int.
time.sleep(10)
print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP))
# Verify that our GUI is working properly
self.driver.get("http://{}/".format(self.HORIZON_IP))
# Login to horizon!
self.driver.find_element(By.ID, "id_username").click()
self.driver.find_element(By.ID, "id_username").send_keys("admin")
self.driver.find_element(By.ID, "id_password").send_keys("keystone")
self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
# Verify that we can click something on the dashboard -- e.g.,
# we're still not sitting at the login screen.
self.driver.find_element(By.LINK_TEXT, "Images").click()
# The Horizon Dashboard should function
self.verify_gui(host)
self.passed = True

View File

@ -12,7 +12,6 @@ vms.
import json
import os
import petname
import sys
import unittest
@ -26,27 +25,6 @@ os.environ['MULTIPASS'] = 'true' # TODO better way to do this.
class TestCluster(Framework):
INIT_FLAG = 'control'
def _compute_node(self, channel='dangerous'):
"""Make a compute node.
TODO: refactor framework so that we can fold a lot of this
into the parent framework. There's a lot of dupe code here.
"""
machine = petname.generate()
prefix = ['multipass', 'exec', machine, '--']
check('multipass', 'launch', '--cpus', '2', '--mem', '8G',
self.DISTRO, '--name', machine)
check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine))
check(*prefix, 'sudo', 'snap', 'install', '--classic',
'--{}'.format(channel), self.SNAP)
return machine, prefix
def test_cluster(self):
# After the setUp step, we should have a control node running
@ -54,18 +32,26 @@ class TestCluster(Framework):
# address.
openstack = '/snap/bin/microstack.openstack'
control_host = self.get_host()
control_host.install()
control_host.init(flag='control')
cluster_password = check_output(*self.PREFIX, 'sudo', 'snap',
control_prefix = control_host.prefix
cluster_password = check_output(*control_prefix, 'sudo', 'snap',
'get', 'microstack',
'config.cluster.password')
control_ip = check_output(*self.PREFIX, 'sudo', 'snap',
control_ip = check_output(*control_prefix, 'sudo', 'snap',
'get', 'microstack',
'config.network.control-ip')
self.assertTrue(cluster_password)
self.assertTrue(control_ip)
compute_machine, compute_prefix = self._compute_node()
compute_host = self.add_host()
compute_host.install()
compute_machine = compute_host.machine
compute_prefix = compute_host.prefix
# TODO add the following to args for init
check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack',
@ -110,7 +96,7 @@ class TestCluster(Framework):
max_pings = 60 # ~1 minutes
# Ping the machine from the control node (we don't have
# networking wired up for the other nodes).
while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
while not call(*control_prefix, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(

View File

@ -2,12 +2,7 @@
"""
control_test.py
This is a test to verify that a control node gets setup properly. We verify:
1) We can install the snap.
2) Nova services are not running
3) Other essential services are running
4) TODO: the horizon dashboard works.
This is a test to verify that a control node gets setup properly.
"""
@ -23,17 +18,19 @@ from tests.framework import Framework, check, check_output # noqa E402
class TestControlNode(Framework):
INIT_FLAG = 'control'
def test_control_node(self):
"""A control node has all services running, so this shouldn't be any
different than our standard setup.
"""
host = self.get_host()
host.install()
host.init(flag='control')
print("Checking output of services ...")
services = check_output(
*self.PREFIX, 'systemctl', 'status', 'snap.microstack.*',
*host.prefix, 'systemctl', 'status', 'snap.microstack.*',
'--no-page')
print("services: @@@")
@ -41,6 +38,7 @@ class TestControlNode(Framework):
self.assertTrue('neutron-' in services)
self.assertTrue('keystone-' in services)
self.assertTrue('nova-' in services)
self.passed = True

78
tests/test_refresh.py Executable file
View File

@ -0,0 +1,78 @@
#!/usr/bin/env python
"""
refresh_test.py
Verify that existing installs can refresh to our newly built snap.
"""
import json
import os
import sys
import unittest
sys.path.append(os.getcwd())
from tests.framework import Framework, check, check_output, call # noqa E402
class TestRefresh(Framework):
"""Refresh from beta and from edge."""
def test_refresh_from_beta(self):
self._refresh_from('beta')
self.passed = True
def test_refresh_from_edge(self):
self._refresh_from('edge')
self.passed = True
def _refresh_from(self, refresh_from='beta'):
"""Refresh test
Like the basic test, but we refresh first.
"""
print("Installing and verfying {} ...".format(refresh_from))
host = self.get_host()
host.install(snap="microstack", channel=refresh_from)
host.init()
prefix = host.prefix
check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'breakfast', '--retry')
if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')
print("Upgrading ...")
host.install() # Install compiled snap
# Should not need to re-init
print("Verifying that refresh completed successfully ...")
# Check our existing instance, starting it if necessary.
if json.loads(check_output(*prefix, '/snap/bin/microstack.openstack',
'server', 'show', 'breakfast',
'--format', 'json'))['status'] == 'SHUTOFF':
print("Starting breakfast (TODO: auto start.)")
check(*prefix, '/snap/bin/microstack.openstack', 'server', 'start',
'breakfast')
# Launch another instance
check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'lunch', '--retry')
# Verify networking
if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')
self.verify_instance_networking(host, 'lunch')
# Verify GUI
self.verify_gui(host)
if __name__ == '__main__':
# Run our tests, ignoring deprecation warnings and warnings about
# unclosed sockets. (TODO: setup a selenium server so that we can
# move from PhantomJS, which is deprecated, to to Selenium headless.)
unittest.main(warnings='ignore')

10
tox.ini
View File

@ -49,6 +49,16 @@ commands =
setenv =
MULTIPASS=true
[testenv:refresh]
# Verify that we can refresh MicroStack
setenv =
MULTIPASS=true
commands =
{toxinidir}/tools/basic_setup.sh
flake8 {toxinidir}/tests/
{toxinidir}/tests/test_refresh.py
[testenv:xenial]
# Run basic tests, under xenial.
setenv =