From 960685b91e4bcf5c64ca13fb02a7f523af15048f Mon Sep 17 00:00:00 2001 From: Pete Vander Giessen Date: Thu, 14 Nov 2019 16:21:11 +0000 Subject: [PATCH] Added refresh tests Refactored test framework so that we have more flexibility in terms of installing various versions of microstack before and after running some tests. Moved in class "globals" into per instance variables, to avoid broken cases with incomplete cleanup. Added test_refresh.py, plus matching env in tox. Refresh tests will fail currently, because we have some pending issues that break refreshes. Fixing those is a subject for a different commit. Refactored cluster_test.py and control_test.py to use new framework. Should (and do) pass. Framework now cleans up multipass hosts regardless of whether or not the tests passed. Leaning on the .tar.gz for local troubleshooting helps us make it better for in gate troubleshooting. Change-Id: I6a45b39132f5959c2944fe1ebbe10f71408ee777 --- tests/framework.py | 221 ++++++++++++++++++++++++++++++++---------- tests/test_basic.py | 96 +++--------------- tests/test_cluster.py | 38 +++----- tests/test_control.py | 16 ++- tests/test_refresh.py | 78 +++++++++++++++ tox.ini | 10 ++ 6 files changed, 292 insertions(+), 167 deletions(-) create mode 100755 tests/test_refresh.py diff --git a/tests/framework.py b/tests/framework.py index b1e138d..4cf84fb 100644 --- a/tests/framework.py +++ b/tests/framework.py @@ -3,9 +3,13 @@ import json import unittest import os import subprocess +import time +import xvfbwrapper from typing import List import petname +from selenium import webdriver +from selenium.webdriver.common.by import By # Setup logging @@ -48,85 +52,198 @@ def call(*args: List[str]) -> bool: return not subprocess.call(args) -class Framework(unittest.TestCase): +def gui_wrapper(func): + """Start up selenium drivers, run a test, then tear them down.""" - PREFIX = [] - DUMP_DIR = '/tmp' - MACHINE = '' - DISTRO = 'bionic' - SNAP = 'microstack_stein_amd64.snap' - HORIZON_IP = '10.20.20.1' - INIT_FLAG = 'auto' + def wrapper(cls, *args, **kwargs): - def install_snap(self, channel='dangerous', snap=None): + # Setup Selenium Driver + cls.display = xvfbwrapper.Xvfb(width=1280, height=720) + cls.display.start() + cls.driver = webdriver.PhantomJS() + + # Run function + try: + return func(cls, *args, **kwargs) + + finally: + # Tear down driver + cls.driver.quit() + cls.display.stop() + + return wrapper + + +class Host(): + """A host with MicroStack installed.""" + + def __init__(self): + self.prefix = [] + self.dump_dir = '/tmp' + self.machine = '' + self.distro = 'bionic' + self.snap = 'microstack_stein_amd64.snap' + self.horizon_ip = '10.20.20.1' + self.host_type = 'localhost' + + if os.environ.get('MULTIPASS'): + self.host_type = 'multipass' + print("Booting a Multipass VM ...") + self.multipass() + + def install(self, snap=None, channel='dangerous'): if snap is None: - snap = self.SNAP + snap = self.snap + print("Installing {}".format(snap)) - check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic', + check(*self.prefix, 'sudo', 'snap', 'install', '--classic', '--{}'.format(channel), snap) - def init_snap(self, flag='auto'): - check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag)) + def init(self, flag='auto'): + print("Initializing the snap with --{}".format(flag)) + check(*self.prefix, 'sudo', 'microstack.init', '--{}'.format(flag)) def multipass(self): - - self.MACHINE = petname.generate() - self.PREFIX = ['multipass', 'exec', self.MACHINE, '--'] - distro = os.environ.get('DISTRO') or self.DISTRO + self.machine = petname.generate() + self.prefix = ['multipass', 'exec', self.machine, '--'] + distro = os.environ.get('distro') or self.distro check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass') check('multipass', 'launch', '--cpus', '2', '--mem', '8G', distro, - '--name', self.MACHINE) - check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE)) + '--name', self.machine) + check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine)) # Figure out machine's ip - info = check_output('multipass', 'info', self.MACHINE, '--format', + info = check_output('multipass', 'info', self.machine, '--format', 'json') info = json.loads(info) - self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0] + self.horizon_ip = info['info'][self.machine]['ipv4'][0] def dump_logs(self): + # TODO: make unique log name if check_output('whoami') == 'zuul': - self.DUMP_DIR = "/home/zuul/zuul-output/logs" + self.dump_dir = "/home/zuul/zuul-output/logs" - check(*self.PREFIX, + check(*self.prefix, 'sudo', 'tar', 'cvzf', - '{}/dump.tar.gz'.format(self.DUMP_DIR), + '{}/dump.tar.gz'.format(self.dump_dir), '/var/snap/microstack/common/log', '/var/snap/microstack/common/etc', '/var/log/syslog') - if 'multipass' in self.PREFIX: + if 'multipass' in self.prefix: check('multipass', 'copy-files', - '{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.') + '{}:/tmp/dump.tar.gz'.format(self.machine), '.') print('Saved dump.tar.gz to local working dir.') - def setUp(self): - self.passed = False # HACK: trigger (or skip) cleanup. - if os.environ.get('MULTIPASS'): - print("Booting a Multipass VM ...") - self.multipass() - print("Installing {}".format(self.SNAP)) - self.install_snap() - print("Initializing the snap with --{}".format(self.INIT_FLAG)) - self.init_snap(self.INIT_FLAG) - - def tearDown(self): - """Either dump logs in the case of failure, or clean up.""" - - if not self.passed: - # Skip teardown in the case of failures, so that we can - # inspect them. - # TODO: I'd like to use the errors and failures list in - # the test result, but I was having trouble getting to it - # from this routine. Need to do more digging and possibly - # elimiate the self.passed hack. - print("Tests failed. Dumping logs and exiting.") - return self.dump_logs() - - print("Tests complete. Tearing down.") - if 'multipass' in self.PREFIX: - check('sudo', 'multipass', 'delete', self.MACHINE) + def teardown(self): + if 'multipass' in self.prefix: + check('sudo', 'multipass', 'delete', self.machine) check('sudo', 'multipass', 'purge') else: check('sudo', 'snap', 'remove', '--purge', 'microstack') + + +class Framework(unittest.TestCase): + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.HOSTS = [] + + def get_host(self): + if self.HOSTS: + return self.HOSTS[0] + host = Host() + self.HOSTS.append(host) + return host + + def add_host(self): + host = Host() + self.HOSTS.append(host) + return host + + def verify_instance_networking(self, host, instance_name): + """Verify that we have networking on an instance + + We should be able to ping the instance. + + And we should be able to reach the Internet. + + """ + prefix = host.prefix + + # Ping the instance + print("Testing ping ...") + ip = None + servers = check_output(*prefix, '/snap/bin/microstack.openstack', + 'server', 'list', '--format', 'json') + servers = json.loads(servers) + for server in servers: + if server['Name'] == instance_name: + ip = server['Networks'].split(",")[1].strip() + break + + self.assertTrue(ip) + + pings = 1 + max_pings = 600 # ~10 minutes! + while not call(*prefix, 'ping', '-c1', '-w1', ip): + pings += 1 + if pings > max_pings: + self.assertFalse(True, msg='Max pings reached!') + + print("Testing instances' ability to connect to the Internet") + # Test Internet connectivity + attempts = 1 + max_attempts = 300 # ~10 minutes! + username = check_output(*prefix, 'whoami') + + while not call( + *prefix, + 'ssh', + '-oStrictHostKeyChecking=no', + '-i', '/home/{}/.ssh/id_microstack'.format(username), + 'cirros@{}'.format(ip), + '--', 'ping', '-c1', '91.189.94.250'): + attempts += 1 + if attempts > max_attempts: + self.assertFalse( + True, + msg='Unable to access the Internet!') + time.sleep(1) + + @gui_wrapper + def verify_gui(self, host): + """Verify that Horizon Dashboard works + + We should be able to reach the dashboard. + + We should be able to login. + + """ + # Test + print('Verifying GUI for (IP: {})'.format(host.horizon_ip)) + # Verify that our GUI is working properly + self.driver.get("http://{}/".format(host.horizon_ip)) + # Login to horizon! + self.driver.find_element(By.ID, "id_username").click() + self.driver.find_element(By.ID, "id_username").send_keys("admin") + self.driver.find_element(By.ID, "id_password").send_keys("keystone") + self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click() + # Verify that we can click something on the dashboard -- e.g., + # we're still not sitting at the login screen. + self.driver.find_element(By.LINK_TEXT, "Images").click() + + def setUp(self): + self.passed = False # HACK: trigger (or skip) cleanup. + + def tearDown(self): + """Clean hosts up, possibly leaving debug information behind.""" + + print("Tests complete. Cleaning up.") + while self.HOSTS: + host = self.HOSTS.pop() + if not self.passed: + print("Dumping logs for {}".format(host.machine)) + host.dump_logs() + host.teardown() diff --git a/tests/test_basic.py b/tests/test_basic.py index c3b01e4..7ed93a0 100755 --- a/tests/test_basic.py +++ b/tests/test_basic.py @@ -14,14 +14,10 @@ Web IDE. """ -import json import os import sys import time import unittest -import xvfbwrapper -from selenium import webdriver -from selenium.webdriver.common.by import By sys.path.append(os.getcwd()) @@ -30,20 +26,6 @@ from tests.framework import Framework, check, check_output, call # noqa E402 class TestBasics(Framework): - def setUp(self): - super(TestBasics, self).setUp() - # Setup Selenium Driver - self.display = xvfbwrapper.Xvfb(width=1280, height=720) - self.display.start() - self.driver = webdriver.PhantomJS() - - def tearDown(self): - # Tear down selenium driver - self.driver.quit() - self.display.stop() - - super(TestBasics, self).tearDown() - def test_basics(self): """Basic test @@ -51,16 +33,13 @@ class TestBasics(Framework): open the Horizon GUI. """ - launch = '/snap/bin/microstack.launch' - openstack = '/snap/bin/microstack.openstack' - - print("Testing microstack.launch ...") - - check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast', - '--retry') + host = self.get_host() + host.install() + host.init() + prefix = host.prefix endpoints = check_output( - *self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list') + *prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list') # Endpoints should be listening on 10.20.20.1 self.assertTrue("10.20.20.1" in endpoints) @@ -68,66 +47,23 @@ class TestBasics(Framework): # Endpoints should not contain localhost self.assertFalse("localhost" in endpoints) - if 'multipass' in self.PREFIX: - # Verify that microstack.launch completed successfully - # Skip these tests in the gate, as they are not reliable there. - # TODO: fix these in the gate! - - # Ping the instance - ip = None - servers = check_output(*self.PREFIX, openstack, - 'server', 'list', '--format', 'json') - servers = json.loads(servers) - for server in servers: - if server['Name'] == 'breakfast': - ip = server['Networks'].split(",")[1].strip() - break - - self.assertTrue(ip) - - pings = 1 - max_pings = 600 # ~10 minutes! - while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip): - pings += 1 - if pings > max_pings: - self.assertFalse(True, msg='Max pings reached!') - - print("Testing instances' ability to connect to the Internet") - # Test Internet connectivity - attempts = 1 - max_attempts = 300 # ~10 minutes! - username = check_output(*self.PREFIX, 'whoami') - - while not call( - *self.PREFIX, - 'ssh', - '-oStrictHostKeyChecking=no', - '-i', '/home/{}/.ssh/id_microstack'.format(username), - 'cirros@{}'.format(ip), - '--', 'ping', '-c1', '91.189.94.250'): - attempts += 1 - if attempts > max_attempts: - self.assertFalse( - True, - msg='Unable to access the Internet!') - time.sleep(1) + # We should be able to launch an instance + print("Testing microstack.launch ...") + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'breakfast', '--retry') + # ... and ping it + # Skip these tests in the gate, as they are not reliable there. + # TODO: fix these in the gate! + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') else: # Artificial wait, to allow for stuff to settle for the GUI test. # TODO: get rid of this, when we drop the ping tests back int. time.sleep(10) - print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP)) - # Verify that our GUI is working properly - self.driver.get("http://{}/".format(self.HORIZON_IP)) - # Login to horizon! - self.driver.find_element(By.ID, "id_username").click() - self.driver.find_element(By.ID, "id_username").send_keys("admin") - self.driver.find_element(By.ID, "id_password").send_keys("keystone") - self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click() - # Verify that we can click something on the dashboard -- e.g., - # we're still not sitting at the login screen. - self.driver.find_element(By.LINK_TEXT, "Images").click() + # The Horizon Dashboard should function + self.verify_gui(host) self.passed = True diff --git a/tests/test_cluster.py b/tests/test_cluster.py index 815977b..58a6c3c 100755 --- a/tests/test_cluster.py +++ b/tests/test_cluster.py @@ -12,7 +12,6 @@ vms. import json import os -import petname import sys import unittest @@ -26,27 +25,6 @@ os.environ['MULTIPASS'] = 'true' # TODO better way to do this. class TestCluster(Framework): - INIT_FLAG = 'control' - - def _compute_node(self, channel='dangerous'): - """Make a compute node. - - TODO: refactor framework so that we can fold a lot of this - into the parent framework. There's a lot of dupe code here. - - """ - machine = petname.generate() - prefix = ['multipass', 'exec', machine, '--'] - - check('multipass', 'launch', '--cpus', '2', '--mem', '8G', - self.DISTRO, '--name', machine) - - check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine)) - check(*prefix, 'sudo', 'snap', 'install', '--classic', - '--{}'.format(channel), self.SNAP) - - return machine, prefix - def test_cluster(self): # After the setUp step, we should have a control node running @@ -54,18 +32,26 @@ class TestCluster(Framework): # address. openstack = '/snap/bin/microstack.openstack' + control_host = self.get_host() + control_host.install() + control_host.init(flag='control') - cluster_password = check_output(*self.PREFIX, 'sudo', 'snap', + control_prefix = control_host.prefix + cluster_password = check_output(*control_prefix, 'sudo', 'snap', 'get', 'microstack', 'config.cluster.password') - control_ip = check_output(*self.PREFIX, 'sudo', 'snap', + control_ip = check_output(*control_prefix, 'sudo', 'snap', 'get', 'microstack', 'config.network.control-ip') self.assertTrue(cluster_password) self.assertTrue(control_ip) - compute_machine, compute_prefix = self._compute_node() + compute_host = self.add_host() + compute_host.install() + + compute_machine = compute_host.machine + compute_prefix = compute_host.prefix # TODO add the following to args for init check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack', @@ -110,7 +96,7 @@ class TestCluster(Framework): max_pings = 60 # ~1 minutes # Ping the machine from the control node (we don't have # networking wired up for the other nodes). - while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip): + while not call(*control_prefix, 'ping', '-c1', '-w1', ip): pings += 1 if pings > max_pings: self.assertFalse( diff --git a/tests/test_control.py b/tests/test_control.py index 7023a5a..53956e8 100755 --- a/tests/test_control.py +++ b/tests/test_control.py @@ -2,12 +2,7 @@ """ control_test.py -This is a test to verify that a control node gets setup properly. We verify: - -1) We can install the snap. -2) Nova services are not running -3) Other essential services are running -4) TODO: the horizon dashboard works. +This is a test to verify that a control node gets setup properly. """ @@ -23,17 +18,19 @@ from tests.framework import Framework, check, check_output # noqa E402 class TestControlNode(Framework): - INIT_FLAG = 'control' - def test_control_node(self): """A control node has all services running, so this shouldn't be any different than our standard setup. """ + host = self.get_host() + host.install() + host.init(flag='control') + print("Checking output of services ...") services = check_output( - *self.PREFIX, 'systemctl', 'status', 'snap.microstack.*', + *host.prefix, 'systemctl', 'status', 'snap.microstack.*', '--no-page') print("services: @@@") @@ -41,6 +38,7 @@ class TestControlNode(Framework): self.assertTrue('neutron-' in services) self.assertTrue('keystone-' in services) + self.assertTrue('nova-' in services) self.passed = True diff --git a/tests/test_refresh.py b/tests/test_refresh.py new file mode 100755 index 0000000..219b512 --- /dev/null +++ b/tests/test_refresh.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +""" +refresh_test.py + +Verify that existing installs can refresh to our newly built snap. + +""" +import json +import os +import sys +import unittest + +sys.path.append(os.getcwd()) + +from tests.framework import Framework, check, check_output, call # noqa E402 + + +class TestRefresh(Framework): + """Refresh from beta and from edge.""" + + def test_refresh_from_beta(self): + self._refresh_from('beta') + self.passed = True + + def test_refresh_from_edge(self): + self._refresh_from('edge') + self.passed = True + + def _refresh_from(self, refresh_from='beta'): + """Refresh test + + Like the basic test, but we refresh first. + + """ + print("Installing and verfying {} ...".format(refresh_from)) + host = self.get_host() + host.install(snap="microstack", channel=refresh_from) + host.init() + prefix = host.prefix + + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'breakfast', '--retry') + + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') + + print("Upgrading ...") + host.install() # Install compiled snap + # Should not need to re-init + + print("Verifying that refresh completed successfully ...") + + # Check our existing instance, starting it if necessary. + if json.loads(check_output(*prefix, '/snap/bin/microstack.openstack', + 'server', 'show', 'breakfast', + '--format', 'json'))['status'] == 'SHUTOFF': + print("Starting breakfast (TODO: auto start.)") + check(*prefix, '/snap/bin/microstack.openstack', 'server', 'start', + 'breakfast') + + # Launch another instance + check(*prefix, '/snap/bin/microstack.launch', 'cirros', + '--name', 'lunch', '--retry') + + # Verify networking + if 'multipass' in prefix: + self.verify_instance_networking(host, 'breakfast') + self.verify_instance_networking(host, 'lunch') + + # Verify GUI + self.verify_gui(host) + + +if __name__ == '__main__': + # Run our tests, ignoring deprecation warnings and warnings about + # unclosed sockets. (TODO: setup a selenium server so that we can + # move from PhantomJS, which is deprecated, to to Selenium headless.) + unittest.main(warnings='ignore') diff --git a/tox.ini b/tox.ini index 7a0aa86..a0531f8 100644 --- a/tox.ini +++ b/tox.ini @@ -49,6 +49,16 @@ commands = setenv = MULTIPASS=true +[testenv:refresh] +# Verify that we can refresh MicroStack +setenv = + MULTIPASS=true + +commands = + {toxinidir}/tools/basic_setup.sh + flake8 {toxinidir}/tests/ + {toxinidir}/tests/test_refresh.py + [testenv:xenial] # Run basic tests, under xenial. setenv =