Added refresh tests

Refactored test framework so that we have more flexibility in terms of
installing various versions of microstack before and after running
some tests. Moved in class "globals" into per instance variables,
to avoid broken cases with incomplete cleanup.

Added test_refresh.py, plus matching env in tox.

Refresh tests will fail currently, because we have some pending issues
that break refreshes. Fixing those is a subject for a different
commit.

Refactored cluster_test.py and control_test.py to use new framework.
Should (and do) pass.

Framework now cleans up multipass hosts regardless of whether or not
the tests passed. Leaning on the .tar.gz for local troubleshooting
helps us make it better for in gate troubleshooting.

Change-Id: I6a45b39132f5959c2944fe1ebbe10f71408ee777

tests/framework.py

@@ -3,9 +3,13 @@ import json
 import unittest
 import os
 import subprocess
+import time
+import xvfbwrapper
 from typing import List
 
 import petname
+from selenium import webdriver
+from selenium.webdriver.common.by import By
 
 
 # Setup logging
@@ -48,85 +52,198 @@ def call(*args: List[str]) -> bool:
     return not subprocess.call(args)
 
 
-class Framework(unittest.TestCase):
+def gui_wrapper(func):
+    """Start up selenium drivers, run a test, then tear them down."""
 
-    PREFIX = []
+    def wrapper(cls, *args, **kwargs):
-    DUMP_DIR = '/tmp'
-    MACHINE = ''
-    DISTRO = 'bionic'
-    SNAP = 'microstack_stein_amd64.snap'
-    HORIZON_IP = '10.20.20.1'
-    INIT_FLAG = 'auto'
 
-    def install_snap(self, channel='dangerous', snap=None):
+        # Setup Selenium Driver
+        cls.display = xvfbwrapper.Xvfb(width=1280, height=720)
+        cls.display.start()
+        cls.driver = webdriver.PhantomJS()
+
+        # Run function
+        try:
+            return func(cls, *args, **kwargs)
+
+        finally:
+            # Tear down driver
+            cls.driver.quit()
+            cls.display.stop()
+
+    return wrapper
+
+
+class Host():
+    """A host with MicroStack installed."""
+
+    def __init__(self):
+        self.prefix = []
+        self.dump_dir = '/tmp'
+        self.machine = ''
+        self.distro = 'bionic'
+        self.snap = 'microstack_stein_amd64.snap'
+        self.horizon_ip = '10.20.20.1'
+        self.host_type = 'localhost'
+
+        if os.environ.get('MULTIPASS'):
+            self.host_type = 'multipass'
+            print("Booting a Multipass VM ...")
+            self.multipass()
+
+    def install(self, snap=None, channel='dangerous'):
         if snap is None:
-            snap = self.SNAP
+            snap = self.snap
+        print("Installing {}".format(snap))
 
-        check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic',
+        check(*self.prefix, 'sudo', 'snap', 'install', '--classic',
               '--{}'.format(channel), snap)
 
-    def init_snap(self, flag='auto'):
+    def init(self, flag='auto'):
-        check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag))
+        print("Initializing the snap with --{}".format(flag))
+        check(*self.prefix, 'sudo', 'microstack.init', '--{}'.format(flag))
 
     def multipass(self):
-
+        self.machine = petname.generate()
-        self.MACHINE = petname.generate()
+        self.prefix = ['multipass', 'exec', self.machine, '--']
-        self.PREFIX = ['multipass', 'exec', self.MACHINE, '--']
+        distro = os.environ.get('distro') or self.distro
-        distro = os.environ.get('DISTRO') or self.DISTRO
 
         check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass')
 
         check('multipass', 'launch', '--cpus', '2', '--mem', '8G', distro,
-              '--name', self.MACHINE)
+              '--name', self.machine)
-        check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE))
+        check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine))
 
         # Figure out machine's ip
-        info = check_output('multipass', 'info', self.MACHINE, '--format',
+        info = check_output('multipass', 'info', self.machine, '--format',
                             'json')
         info = json.loads(info)
-        self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0]
+        self.horizon_ip = info['info'][self.machine]['ipv4'][0]
 
     def dump_logs(self):
+        # TODO: make unique log name
         if check_output('whoami') == 'zuul':
-            self.DUMP_DIR = "/home/zuul/zuul-output/logs"
+            self.dump_dir = "/home/zuul/zuul-output/logs"
 
-        check(*self.PREFIX,
+        check(*self.prefix,
               'sudo', 'tar', 'cvzf',
-              '{}/dump.tar.gz'.format(self.DUMP_DIR),
+              '{}/dump.tar.gz'.format(self.dump_dir),
               '/var/snap/microstack/common/log',
               '/var/snap/microstack/common/etc',
               '/var/log/syslog')
-        if 'multipass' in self.PREFIX:
+        if 'multipass' in self.prefix:
             check('multipass', 'copy-files',
-                  '{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.')
+                  '{}:/tmp/dump.tar.gz'.format(self.machine), '.')
         print('Saved dump.tar.gz to local working dir.')
 
-    def setUp(self):
+    def teardown(self):
-        self.passed = False  # HACK: trigger (or skip) cleanup.
+        if 'multipass' in self.prefix:
-        if os.environ.get('MULTIPASS'):
+            check('sudo', 'multipass', 'delete', self.machine)
-            print("Booting a Multipass VM ...")
-            self.multipass()
-        print("Installing {}".format(self.SNAP))
-        self.install_snap()
-        print("Initializing the snap with --{}".format(self.INIT_FLAG))
-        self.init_snap(self.INIT_FLAG)
-
-    def tearDown(self):
-        """Either dump logs in the case of failure, or clean up."""
-
-        if not self.passed:
-            # Skip teardown in the case of failures, so that we can
-            # inspect them.
-            # TODO: I'd like to use the errors and failures list in
-            # the test result, but I was having trouble getting to it
-            # from this routine. Need to do more digging and possibly
-            # elimiate the self.passed hack.
-            print("Tests failed. Dumping logs and exiting.")
-            return self.dump_logs()
-
-        print("Tests complete. Tearing down.")
-        if 'multipass' in self.PREFIX:
-            check('sudo', 'multipass', 'delete', self.MACHINE)
             check('sudo', 'multipass', 'purge')
         else:
             check('sudo', 'snap', 'remove', '--purge', 'microstack')
+
+
+class Framework(unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.HOSTS = []
+
+    def get_host(self):
+        if self.HOSTS:
+            return self.HOSTS[0]
+        host = Host()
+        self.HOSTS.append(host)
+        return host
+
+    def add_host(self):
+        host = Host()
+        self.HOSTS.append(host)
+        return host
+
+    def verify_instance_networking(self, host, instance_name):
+        """Verify that we have networking on an instance
+
+        We should be able to ping the instance.
+
+        And we should be able to reach the Internet.
+
+        """
+        prefix = host.prefix
+
+        # Ping the instance
+        print("Testing ping ...")
+        ip = None
+        servers = check_output(*prefix, '/snap/bin/microstack.openstack',
+                               'server', 'list', '--format', 'json')
+        servers = json.loads(servers)
+        for server in servers:
+            if server['Name'] == instance_name:
+                ip = server['Networks'].split(",")[1].strip()
+                break
+
+        self.assertTrue(ip)
+
+        pings = 1
+        max_pings = 600  # ~10 minutes!
+        while not call(*prefix, 'ping', '-c1', '-w1', ip):
+            pings += 1
+            if pings > max_pings:
+                self.assertFalse(True, msg='Max pings reached!')
+
+        print("Testing instances' ability to connect to the Internet")
+        # Test Internet connectivity
+        attempts = 1
+        max_attempts = 300  # ~10 minutes!
+        username = check_output(*prefix, 'whoami')
+
+        while not call(
+                *prefix,
+                'ssh',
+                '-oStrictHostKeyChecking=no',
+                '-i', '/home/{}/.ssh/id_microstack'.format(username),
+                'cirros@{}'.format(ip),
+                '--', 'ping', '-c1', '91.189.94.250'):
+            attempts += 1
+            if attempts > max_attempts:
+                self.assertFalse(
+                    True,
+                    msg='Unable to access the Internet!')
+            time.sleep(1)
+
+    @gui_wrapper
+    def verify_gui(self, host):
+        """Verify that Horizon Dashboard works
+
+        We should be able to reach the dashboard.
+
+        We should be able to login.
+
+        """
+        # Test
+        print('Verifying GUI for (IP: {})'.format(host.horizon_ip))
+        # Verify that our GUI is working properly
+        self.driver.get("http://{}/".format(host.horizon_ip))
+        # Login to horizon!
+        self.driver.find_element(By.ID, "id_username").click()
+        self.driver.find_element(By.ID, "id_username").send_keys("admin")
+        self.driver.find_element(By.ID, "id_password").send_keys("keystone")
+        self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
+        # Verify that we can click something on the dashboard -- e.g.,
+        # we're still not sitting at the login screen.
+        self.driver.find_element(By.LINK_TEXT, "Images").click()
+
+    def setUp(self):
+        self.passed = False  # HACK: trigger (or skip) cleanup.
+
+    def tearDown(self):
+        """Clean hosts up, possibly leaving debug information behind."""
+
+        print("Tests complete. Cleaning up.")
+        while self.HOSTS:
+            host = self.HOSTS.pop()
+            if not self.passed:
+                print("Dumping logs for {}".format(host.machine))
+                host.dump_logs()
+            host.teardown()

tests/test_basic.py

@@ -14,14 +14,10 @@ Web IDE.
 
 """
 
-import json
 import os
 import sys
 import time
 import unittest
-import xvfbwrapper
-from selenium import webdriver
-from selenium.webdriver.common.by import By
 
 sys.path.append(os.getcwd())
 
@@ -30,20 +26,6 @@ from tests.framework import Framework, check, check_output, call  # noqa E402
 
 class TestBasics(Framework):
 
-    def setUp(self):
-        super(TestBasics, self).setUp()
-        # Setup Selenium Driver
-        self.display = xvfbwrapper.Xvfb(width=1280, height=720)
-        self.display.start()
-        self.driver = webdriver.PhantomJS()
-
-    def tearDown(self):
-        # Tear down selenium driver
-        self.driver.quit()
-        self.display.stop()
-
-        super(TestBasics, self).tearDown()
-
     def test_basics(self):
         """Basic test
 
@@ -51,16 +33,13 @@ class TestBasics(Framework):
         open the Horizon GUI.
 
         """
-        launch = '/snap/bin/microstack.launch'
+        host = self.get_host()
-        openstack = '/snap/bin/microstack.openstack'
+        host.install()
-
+        host.init()
-        print("Testing microstack.launch ...")
+        prefix = host.prefix
-
-        check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast',
-              '--retry')
 
         endpoints = check_output(
-            *self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list')
+            *prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list')
 
         # Endpoints should be listening on 10.20.20.1
         self.assertTrue("10.20.20.1" in endpoints)
@@ -68,66 +47,23 @@ class TestBasics(Framework):
         # Endpoints should not contain localhost
         self.assertFalse("localhost" in endpoints)
 
-        if 'multipass' in self.PREFIX:
+        # We should be able to launch an instance
-            # Verify that microstack.launch completed successfully
+        print("Testing microstack.launch ...")
-            # Skip these tests in the gate, as they are not reliable there.
+        check(*prefix, '/snap/bin/microstack.launch', 'cirros',
-            # TODO: fix these in the gate!
+              '--name', 'breakfast', '--retry')
-
-            # Ping the instance
-            ip = None
-            servers = check_output(*self.PREFIX, openstack,
-                                   'server', 'list', '--format', 'json')
-            servers = json.loads(servers)
-            for server in servers:
-                if server['Name'] == 'breakfast':
-                    ip = server['Networks'].split(",")[1].strip()
-                    break
-
-            self.assertTrue(ip)
-
-            pings = 1
-            max_pings = 600  # ~10 minutes!
-            while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
-                pings += 1
-                if pings > max_pings:
-                    self.assertFalse(True, msg='Max pings reached!')
-
-            print("Testing instances' ability to connect to the Internet")
-            # Test Internet connectivity
-            attempts = 1
-            max_attempts = 300  # ~10 minutes!
-            username = check_output(*self.PREFIX, 'whoami')
-
-            while not call(
-                    *self.PREFIX,
-                    'ssh',
-                    '-oStrictHostKeyChecking=no',
-                    '-i', '/home/{}/.ssh/id_microstack'.format(username),
-                    'cirros@{}'.format(ip),
-                    '--', 'ping', '-c1', '91.189.94.250'):
-                attempts += 1
-                if attempts > max_attempts:
-                    self.assertFalse(
-                        True,
-                        msg='Unable to access the Internet!')
-                time.sleep(1)
 
+        # ... and ping it
+        # Skip these tests in the gate, as they are not reliable there.
+        # TODO: fix these in the gate!
+        if 'multipass' in prefix:
+            self.verify_instance_networking(host, 'breakfast')
         else:
             # Artificial wait, to allow for stuff to settle for the GUI test.
             # TODO: get rid of this, when we drop the ping tests back int.
             time.sleep(10)
 
-        print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP))
+        # The Horizon Dashboard should function
-        # Verify that our GUI is working properly
+        self.verify_gui(host)
-        self.driver.get("http://{}/".format(self.HORIZON_IP))
-        # Login to horizon!
-        self.driver.find_element(By.ID, "id_username").click()
-        self.driver.find_element(By.ID, "id_username").send_keys("admin")
-        self.driver.find_element(By.ID, "id_password").send_keys("keystone")
-        self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
-        # Verify that we can click something on the dashboard -- e.g.,
-        # we're still not sitting at the login screen.
-        self.driver.find_element(By.LINK_TEXT, "Images").click()
 
         self.passed = True
 

tests/test_cluster.py

@@ -12,7 +12,6 @@ vms.
 
 import json
 import os
-import petname
 import sys
 import unittest
 
@@ -26,27 +25,6 @@ os.environ['MULTIPASS'] = 'true'  # TODO better way to do this.
 
 class TestCluster(Framework):
 
-    INIT_FLAG = 'control'
-
-    def _compute_node(self, channel='dangerous'):
-        """Make a compute node.
-
-        TODO: refactor framework so that we can fold a lot of this
-        into the parent framework. There's a lot of dupe code here.
-
-        """
-        machine = petname.generate()
-        prefix = ['multipass', 'exec', machine, '--']
-
-        check('multipass', 'launch', '--cpus', '2', '--mem', '8G',
-              self.DISTRO, '--name', machine)
-
-        check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine))
-        check(*prefix, 'sudo', 'snap', 'install', '--classic',
-              '--{}'.format(channel), self.SNAP)
-
-        return machine, prefix
-
     def test_cluster(self):
 
         # After the setUp step, we should have a control node running
@@ -54,18 +32,26 @@ class TestCluster(Framework):
         # address.
 
         openstack = '/snap/bin/microstack.openstack'
+        control_host = self.get_host()
+        control_host.install()
+        control_host.init(flag='control')
 
-        cluster_password = check_output(*self.PREFIX, 'sudo', 'snap',
+        control_prefix = control_host.prefix
+        cluster_password = check_output(*control_prefix, 'sudo', 'snap',
                                         'get', 'microstack',
                                         'config.cluster.password')
-        control_ip = check_output(*self.PREFIX, 'sudo', 'snap',
+        control_ip = check_output(*control_prefix, 'sudo', 'snap',
                                   'get', 'microstack',
                                   'config.network.control-ip')
 
         self.assertTrue(cluster_password)
         self.assertTrue(control_ip)
 
-        compute_machine, compute_prefix = self._compute_node()
+        compute_host = self.add_host()
+        compute_host.install()
+
+        compute_machine = compute_host.machine
+        compute_prefix = compute_host.prefix
 
         # TODO add the following to args for init
         check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack',
@@ -110,7 +96,7 @@ class TestCluster(Framework):
         max_pings = 60  # ~1 minutes
         # Ping the machine from the control node (we don't have
         # networking wired up for the other nodes).
-        while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
+        while not call(*control_prefix, 'ping', '-c1', '-w1', ip):
             pings += 1
             if pings > max_pings:
                 self.assertFalse(

tests/test_control.py

@@ -2,12 +2,7 @@
 """
 control_test.py
 
-This is a test to verify that a control node gets setup properly. We verify:
+This is a test to verify that a control node gets setup properly.
-
-1) We can install the snap.
-2) Nova services are not running
-3) Other essential services are running
-4) TODO: the horizon dashboard works.
 
 """
 
@@ -23,17 +18,19 @@ from tests.framework import Framework, check, check_output  # noqa E402
 
 class TestControlNode(Framework):
 
-    INIT_FLAG = 'control'
-
     def test_control_node(self):
         """A control node has all services running, so this shouldn't be any
         different than our standard setup.
 
         """
 
+        host = self.get_host()
+        host.install()
+        host.init(flag='control')
+
         print("Checking output of services ...")
         services = check_output(
-            *self.PREFIX, 'systemctl', 'status', 'snap.microstack.*',
+            *host.prefix, 'systemctl', 'status', 'snap.microstack.*',
             '--no-page')
 
         print("services: @@@")
@@ -41,6 +38,7 @@ class TestControlNode(Framework):
 
         self.assertTrue('neutron-' in services)
         self.assertTrue('keystone-' in services)
+        self.assertTrue('nova-' in services)
 
         self.passed = True
 

tests/test_refresh.py

@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+"""
+refresh_test.py
+
+Verify that existing installs can refresh to our newly built snap.
+
+"""
+import json
+import os
+import sys
+import unittest
+
+sys.path.append(os.getcwd())
+
+from tests.framework import Framework, check, check_output, call  # noqa E402
+
+
+class TestRefresh(Framework):
+    """Refresh from beta and from edge."""
+
+    def test_refresh_from_beta(self):
+        self._refresh_from('beta')
+        self.passed = True
+
+    def test_refresh_from_edge(self):
+        self._refresh_from('edge')
+        self.passed = True
+
+    def _refresh_from(self, refresh_from='beta'):
+        """Refresh test
+
+        Like the basic test, but we refresh first.
+
+        """
+        print("Installing and verfying {} ...".format(refresh_from))
+        host = self.get_host()
+        host.install(snap="microstack", channel=refresh_from)
+        host.init()
+        prefix = host.prefix
+
+        check(*prefix, '/snap/bin/microstack.launch', 'cirros',
+              '--name', 'breakfast', '--retry')
+
+        if 'multipass' in prefix:
+            self.verify_instance_networking(host, 'breakfast')
+
+        print("Upgrading ...")
+        host.install()  # Install compiled snap
+        # Should not need to re-init
+
+        print("Verifying that refresh completed successfully ...")
+
+        # Check our existing instance, starting it if necessary.
+        if json.loads(check_output(*prefix, '/snap/bin/microstack.openstack',
+                                   'server', 'show', 'breakfast',
+                                   '--format', 'json'))['status'] == 'SHUTOFF':
+            print("Starting breakfast (TODO: auto start.)")
+            check(*prefix, '/snap/bin/microstack.openstack', 'server', 'start',
+                  'breakfast')
+
+        # Launch another instance
+        check(*prefix, '/snap/bin/microstack.launch', 'cirros',
+              '--name', 'lunch', '--retry')
+
+        # Verify networking
+        if 'multipass' in prefix:
+            self.verify_instance_networking(host, 'breakfast')
+            self.verify_instance_networking(host, 'lunch')
+
+        # Verify GUI
+        self.verify_gui(host)
+
+
+if __name__ == '__main__':
+    # Run our tests, ignoring deprecation warnings and warnings about
+    # unclosed sockets. (TODO: setup a selenium server so that we can
+    # move from PhantomJS, which is deprecated, to to Selenium headless.)
+    unittest.main(warnings='ignore')

tox.ini

@@ -49,6 +49,16 @@ commands =
 setenv =
     MULTIPASS=true
 
+[testenv:refresh]
+# Verify that we can refresh MicroStack
+setenv =
+    MULTIPASS=true
+
+commands =
+    {toxinidir}/tools/basic_setup.sh
+    flake8 {toxinidir}/tests/
+    {toxinidir}/tests/test_refresh.py
+
 [testenv:xenial]
 # Run basic tests, under xenial.
 setenv =