diff --git a/tests/framework.py b/tests/framework.py index b1e138d..4cf84fb 100644 --- a/tests/framework.py +++ b/tests/framework.py @@ -3,9 +3,13 @@ import json import unittest import os import subprocess +import time +import xvfbwrapper from typing import List import petname +from selenium import webdriver +from selenium.webdriver.common.by import By # Setup logging @@ -48,85 +52,198 @@ def call(*args: List[str]) -> bool: return not subprocess.call(args) -class Framework(unittest.TestCase): +def gui_wrapper(func): + """Start up selenium drivers, run a test, then tear them down.""" - PREFIX = [] - DUMP_DIR = '/tmp' - MACHINE = '' - DISTRO = 'bionic' - SNAP = 'microstack_stein_amd64.snap' - HORIZON_IP = '10.20.20.1' - INIT_FLAG = 'auto' + def wrapper(cls, *args, **kwargs): - def install_snap(self, channel='dangerous', snap=None): + # Setup Selenium Driver + cls.display = xvfbwrapper.Xvfb(width=1280, height=720) + cls.display.start() + cls.driver = webdriver.PhantomJS() + + # Run function + try: + return func(cls, *args, **kwargs) + + finally: + # Tear down driver + cls.driver.quit() + cls.display.stop() + + return wrapper + + +class Host(): + """A host with MicroStack installed.""" + + def __init__(self): + self.prefix = [] + self.dump_dir = '/tmp' + self.machine = '' + self.distro = 'bionic' + self.snap = 'microstack_stein_amd64.snap' + self.horizon_ip = '10.20.20.1' + self.host_type = 'localhost' + + if os.environ.get('MULTIPASS'): + self.host_type = 'multipass' + print("Booting a Multipass VM ...") + self.multipass() + + def install(self, snap=None, channel='dangerous'): if snap is None: - snap = self.SNAP + snap = self.snap + print("Installing {}".format(snap)) - check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic', + check(*self.prefix, 'sudo', 'snap', 'install', '--classic', '--{}'.format(channel), snap) - def init_snap(self, flag='auto'): - check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag)) + def init(self, flag='auto'): + print("Initializing the snap with --{}".format(flag)) + check(*self.prefix, 'sudo', 'microstack.init', '--{}'.format(flag)) def multipass(self): - - self.MACHINE = petname.generate() - self.PREFIX = ['multipass', 'exec', self.MACHINE, '--'] - distro = os.environ.get('DISTRO') or self.DISTRO + self.machine = petname.generate() + self.prefix = ['multipass', 'exec', self.machine, '--'] + distro = os.environ.get('distro') or self.distro check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass') check('multipass', 'launch', '--cpus', '2', '--mem', '8G', distro, - '--name', self.MACHINE) - check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE)) + '--name', self.machine) + check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine)) # Figure out machine's ip - info = check_output('multipass', 'info', self.MACHINE, '--format', + info = check_output('multipass', 'info', self.machine, '--format', 'json') info = json.loads(info) - self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0] + self.horizon_ip = info['info'][self.machine]['ipv4'][0] def dump_logs(self): + # TODO: make unique log name if check_output('whoami') == 'zuul': - self.DUMP_DIR = "/home/zuul/zuul-output/logs" + self.dump_dir = "/home/zuul/zuul-output/logs" - check(*self.PREFIX, + check(*self.prefix, 'sudo', 'tar', 'cvzf', - '{}/dump.tar.gz'.format(self.DUMP_DIR), + '{}/dump.tar.gz'.format(self.dump_dir), '/var/snap/microstack/common/log', '/var/snap/microstack/common/etc', '/var/log/syslog') - if 'multipass' in self.PREFIX: + if 'multipass' in self.prefix: check('multipass', 'copy-files', - '{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.') + '{}:/tmp/dump.tar.gz'.format(self.machine), '.') print('Saved dump.tar.gz to local working dir.') - def setUp(self): - self.passed = False # HACK: trigger (or skip) cleanup. - if os.environ.get('MULTIPASS'): - print("Booting a Multipass VM ...") - self.multipass() - print("Installing {}".format(self.SNAP)) - self.install_snap() - print("Initializing the snap with --{}".format(self.INIT_FLAG)) - self.init_snap(self.INIT_FLAG) - - def tearDown(self): - """Either dump logs in the case of failure, or clean up.""" - - if not self.passed: - # Skip teardown in the case of failures, so that we can - # inspect them. - # TODO: I'd like to use the errors and failures list in - # the test result, but I was having trouble getting to it - # from this routine. Need to do more digging and possibly - # elimiate the self.passed hack. - print("Tests failed. Dumping logs and exiting.") - return self.dump_logs() - - print("Tests complete. Tearing down.") - if 'multipass' in self.PREFIX: - check('sudo', 'multipass', 'delete', self.MACHINE) + def teardown(self): + if 'multipass' in self.prefix: + check('sudo', 'multipass', 'delete', self.machine) check('sudo', 'multipass', 'purge') else: check('sudo', 'snap', 'remove', '--purge', 'microstack') + + +class Framework(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.HOSTS = [] + + def get_host(self): + if self.HOSTS: + return self.HOSTS[0] + host = Host() + self.HOSTS.append(host) + return host + + def add_host(self): + host = Host() + self.HOSTS.append(host) + return host + + def verify_instance_networking(self, host, instance_name): + """Verify that we have networking on an instance + + We should be able to ping the instance. + + And we should be able to reach the Internet. + + """ + prefix = host.prefix + + # Ping the instance + print("Testing ping ...") + ip = None + servers = check_output(*prefix, '/snap/bin/microstack.openstack', + 'server', 'list', '--format', 'json') + servers = json.loads(servers) + for server in servers: + if server['Name'] == instance_name: + ip = server['Networks'].split(",")[1].strip() + break + + self.assertTrue(ip) + + pings = 1 + max_pings = 600 # ~10 minutes! + while not call(*prefix, 'ping', '-c1', '-w1', ip): + pings += 1 + if pings > max_pings: + self.assertFalse(True, msg='Max pings reached!') + + print("Testing instances' ability to connect to the Internet") + # Test Internet connectivity + attempts = 1 + max_attempts = 300 # ~10 minutes! + username = check_output(*prefix, 'whoami') + + while not call( + *prefix, + 'ssh', + '-oStrictHostKeyChecking=no', + '-i', '/home/{}/.ssh/id_microstack'.format(username), + 'cirros@{}'.format(ip), + '--', 'ping', '-c1', '91.189.94.250'): + attempts += 1 + if attempts > max_attempts: + self.assertFalse( + True, + msg='Unable to access the Internet!') + time.sleep(1) + + @gui_wrapper + def verify_gui(self, host): + """Verify that Horizon Dashboard works + + We should be able to reach the dashboard. + + We should be able to login. + + """ + # Test + print('Verifying GUI for (IP: {})'.format(host.horizon_ip)) + # Verify that our GUI is working properly + self.driver.get("http://{}/".format(host.horizon_ip)) + # Login to horizon! + self.driver.find_element(By.ID, "id_username").click() + self.driver.find_element(By.ID, "id_username").send_keys("admin") + self.driver.find_element(By.ID, "id_password").send_keys("keystone") + self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click() + # Verify that we can click something on the dashboard -- e.g., + # we're still not sitting at the login screen. + self.driver.find_element(By.LINK_TEXT, "Images").click() + + def setUp(self): + self.passed = False # HACK: trigger (or skip) cleanup. + + def tearDown(self): + """Clean hosts up, possibly leaving debug information behind.""" + + print("Tests complete. Cleaning up.") + while self.HOSTS: + host = self.HOSTS.pop() + if not self.passed: + print("Dumping logs for {}".format(host.machine)) + host.dump_logs() + host.teardown() diff --git a/tests/test_basic.py b/tests/test_basic.py index c3b01e4..7ed93a0 100755 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -14,14 +14,10 @@ Web IDE. """ -import json import os import sys import time import unittest -import xvfbwrapper -from selenium import webdriver -from selenium.webdriver.common.by import By sys.path.append(os.getcwd()) @@ -30,20 +26,6 @@ from tests.framework import Framework, check, check_output, call # noqa E402 class TestBasics(Framework): - def setUp(self): - super(TestBasics, self).setUp() - # Setup Selenium Driver - self.display = xvfbwrapper.Xvfb(width=1280, height=720) - self.display.start() - self.driver = webdriver.PhantomJS() - - def tearDown(self): - # Tear down selenium driver - self.driver.quit() - self.display.stop() - - super(TestBasics, self).tearDown() - def test_basics(self): """Basic test @@ -51,16 +33,13 @@ class TestBasics(Framework): open the Horizon GUI. """ - launch = '/snap/bin/microstack.launch' - openstack = '/snap/bin/microstack.openstack' - - print("Testing microstack.launch ...") - - check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast', - '--retry') + host = self.get_host() + host.install() + host.init() + prefix = host.prefix endpoints = check_output( - *self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list') + *prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list') # Endpoints should be listening on 10.20.20.1 self.assertTrue("10.20.20.1" in endpoints) @@ -68,66 +47,23 @@ class TestBasics(Framework): # Endpoints should not contain localhost self.assertFalse("localhost" in endpoints) - if 'multipass' in self.PREFIX: - # Verify that microstack.launch completed successfully - # Skip these tests in the gate, as they are not reliable there. - # TODO: fix these in the gate! - - # Ping the instance - ip = None - servers = check_output(*self.PREFIX, openstack, - 'server', 'list', '--format', 'json') - servers = json.loads(servers) - for server in servers: - if server['Name'] == 'breakfast': - ip = server['Networks'].split(",")[1].strip() - break - - self.assertTrue(ip) - - pings = 1 - max_pings = 600 # ~10 minutes! - while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip): - pings += 1 - if pings > max_pings: - self.assertFalse(True, msg='Max pings reached!') - - print("Testing instances' ability to connect to the Internet") - # Test Internet connectivity - attempts = 1 - max_attempts = 300 # ~10 minutes! - username = check_output(*self.PREFIX, 'whoami') - - while not call( - *self.PREFIX, - 'ssh', - '-oStrictHostKeyChecking=no', - '-i', '/home/{}/.ssh/id_microstack'.format(username), - 'cirros@{}'.format(ip), - '--', 'ping', '-c1', '91.189.94.250'): - attempts += 1 - if attempts > max_attempts: - self.assertFalse( - True, - msg='Unable to access the Internet!') - time.sleep(1) + # We should be able to launch an instance + print("Testing microstack.launch ...") + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'breakfast', '--retry') + # ... and ping it + # Skip these tests in the gate, as they are not reliable there. + # TODO: fix these in the gate! + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') else: # Artificial wait, to allow for stuff to settle for the GUI test. # TODO: get rid of this, when we drop the ping tests back int. time.sleep(10) - print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP)) - # Verify that our GUI is working properly - self.driver.get("http://{}/".format(self.HORIZON_IP)) - # Login to horizon! - self.driver.find_element(By.ID, "id_username").click() - self.driver.find_element(By.ID, "id_username").send_keys("admin") - self.driver.find_element(By.ID, "id_password").send_keys("keystone") - self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click() - # Verify that we can click something on the dashboard -- e.g., - # we're still not sitting at the login screen. - self.driver.find_element(By.LINK_TEXT, "Images").click() + # The Horizon Dashboard should function + self.verify_gui(host) self.passed = True diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 815977b..58a6c3c 100755 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -12,7 +12,6 @@ vms. import json import os -import petname import sys import unittest @@ -26,27 +25,6 @@ os.environ['MULTIPASS'] = 'true' # TODO better way to do this. class TestCluster(Framework): - INIT_FLAG = 'control' - - def _compute_node(self, channel='dangerous'): - """Make a compute node. - - TODO: refactor framework so that we can fold a lot of this - into the parent framework. There's a lot of dupe code here. - - """ - machine = petname.generate() - prefix = ['multipass', 'exec', machine, '--'] - - check('multipass', 'launch', '--cpus', '2', '--mem', '8G', - self.DISTRO, '--name', machine) - - check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine)) - check(*prefix, 'sudo', 'snap', 'install', '--classic', - '--{}'.format(channel), self.SNAP) - - return machine, prefix - def test_cluster(self): # After the setUp step, we should have a control node running @@ -54,18 +32,26 @@ class TestCluster(Framework): # address. openstack = '/snap/bin/microstack.openstack' + control_host = self.get_host() + control_host.install() + control_host.init(flag='control') - cluster_password = check_output(*self.PREFIX, 'sudo', 'snap', + control_prefix = control_host.prefix + cluster_password = check_output(*control_prefix, 'sudo', 'snap', 'get', 'microstack', 'config.cluster.password') - control_ip = check_output(*self.PREFIX, 'sudo', 'snap', + control_ip = check_output(*control_prefix, 'sudo', 'snap', 'get', 'microstack', 'config.network.control-ip') self.assertTrue(cluster_password) self.assertTrue(control_ip) - compute_machine, compute_prefix = self._compute_node() + compute_host = self.add_host() + compute_host.install() + + compute_machine = compute_host.machine + compute_prefix = compute_host.prefix # TODO add the following to args for init check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack', @@ -110,7 +96,7 @@ class TestCluster(Framework): max_pings = 60 # ~1 minutes # Ping the machine from the control node (we don't have # networking wired up for the other nodes). - while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip): + while not call(*control_prefix, 'ping', '-c1', '-w1', ip): pings += 1 if pings > max_pings: self.assertFalse( diff --git a/tests/test_control.py b/tests/test_control.py index 7023a5a..53956e8 100755 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -2,12 +2,7 @@ """ control_test.py -This is a test to verify that a control node gets setup properly. We verify: - -1) We can install the snap. -2) Nova services are not running -3) Other essential services are running -4) TODO: the horizon dashboard works. +This is a test to verify that a control node gets setup properly. """ @@ -23,17 +18,19 @@ from tests.framework import Framework, check, check_output # noqa E402 class TestControlNode(Framework): - INIT_FLAG = 'control' - def test_control_node(self): """A control node has all services running, so this shouldn't be any different than our standard setup. """ + host = self.get_host() + host.install() + host.init(flag='control') + print("Checking output of services ...") services = check_output( - *self.PREFIX, 'systemctl', 'status', 'snap.microstack.*', + *host.prefix, 'systemctl', 'status', 'snap.microstack.*', '--no-page') print("services: @@@") @@ -41,6 +38,7 @@ class TestControlNode(Framework): self.assertTrue('neutron-' in services) self.assertTrue('keystone-' in services) + self.assertTrue('nova-' in services) self.passed = True diff --git a/tests/test_refresh.py b/tests/test_refresh.py new file mode 100755 index 0000000..219b512 --- /dev/null +++ b/tests/test_refresh.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +""" +refresh_test.py + +Verify that existing installs can refresh to our newly built snap. + +""" +import json +import os +import sys +import unittest + +sys.path.append(os.getcwd()) + +from tests.framework import Framework, check, check_output, call # noqa E402 + + +class TestRefresh(Framework): + """Refresh from beta and from edge.""" + + def test_refresh_from_beta(self): + self._refresh_from('beta') + self.passed = True + + def test_refresh_from_edge(self): + self._refresh_from('edge') + self.passed = True + + def _refresh_from(self, refresh_from='beta'): + """Refresh test + + Like the basic test, but we refresh first. + + """ + print("Installing and verfying {} ...".format(refresh_from)) + host = self.get_host() + host.install(snap="microstack", channel=refresh_from) + host.init() + prefix = host.prefix + + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'breakfast', '--retry') + + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') + + print("Upgrading ...") + host.install() # Install compiled snap + # Should not need to re-init + + print("Verifying that refresh completed successfully ...") + + # Check our existing instance, starting it if necessary. + if json.loads(check_output(*prefix, '/snap/bin/microstack.openstack', + 'server', 'show', 'breakfast', + '--format', 'json'))['status'] == 'SHUTOFF': + print("Starting breakfast (TODO: auto start.)") + check(*prefix, '/snap/bin/microstack.openstack', 'server', 'start', + 'breakfast') + + # Launch another instance + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'lunch', '--retry') + + # Verify networking + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') + self.verify_instance_networking(host, 'lunch') + + # Verify GUI + self.verify_gui(host) + + +if __name__ == '__main__': + # Run our tests, ignoring deprecation warnings and warnings about + # unclosed sockets. (TODO: setup a selenium server so that we can + # move from PhantomJS, which is deprecated, to to Selenium headless.) + unittest.main(warnings='ignore') diff --git a/tox.ini b/tox.ini index 7a0aa86..a0531f8 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,16 @@ commands = setenv = MULTIPASS=true +[testenv:refresh] +# Verify that we can refresh MicroStack +setenv = + MULTIPASS=true + +commands = + {toxinidir}/tools/basic_setup.sh + flake8 {toxinidir}/tests/ + {toxinidir}/tests/test_refresh.py + [testenv:xenial] # Run basic tests, under xenial. setenv =