Added testing for control nodes

Ported basic-test.sh to test_basic.py, and folded in
test_horizonlogin.py.

Made a testing framework for shared components.

Added test_control.py

Got rid of default .stestr.conf, as we're going to have multiple tests
running, and one conf is confusing.

Manually ordering functional tests for now, as stestr noms too much
output, and runs things in parallel, which doesn't work for our
functional tests.

Skipping compute node test for now, as it won't work until we can
connect to a control node with databases and such.

Moved very-basic-test.sh to tools/make-a-microstack.sh. It's really
more of a tool for manual testing than an automated test.

Added test-requirements and updated gitignore.

Moved auto-detection of kvm extensions to init, rather than test, as
it makes more sense there.

Change-Id: Iba7f7fe07cbb066790f802cf2a7c87c68994062c

.gitignore

@@ -9,6 +9,7 @@ microstack_source.tar.bz2
 prime/
 snap/.snapcraft
 stage/
+dump.tar.gz
 
 # Emacs
 *~

.stestr.conf

@@ -1,3 +0,0 @@
-[DEFAULT]
-test_path=./tools/init/tests/
-top_dir=./tools/init/

snap-overlay/bin/launch.sh

@@ -67,7 +67,7 @@ while :; do
     fi
     if [[ $(openstack server list | grep $SERVER | grep ERROR) ]]; then
         openstack server list
-        echo "Uh-oh. There was an error. See /var/snap/microstack/common/log for details."
+        echo "Uh-oh. There was an error. Run `journalctl -xe` for details."
         exit 1
     fi
 done

test-requirements.txt

@@ -1,3 +1,5 @@
+flake8
 petname
 selenium
+stestr
 xvfbwrapper

tests/framework.py

@@ -0,0 +1,131 @@
+import logging
+import json
+import unittest
+import os
+import subprocess
+from typing import List
+
+import petname
+
+
+# Setup logging
+log = logging.getLogger("microstack_test")
+log.setLevel(logging.DEBUG)
+stream = logging.StreamHandler()
+formatter = logging.Formatter(
+    '%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+stream.setFormatter(formatter)
+log.addHandler(stream)
+
+
+def check(*args: List[str]) -> int:
+    """Execute a shell command, raising an error on failed excution.
+
+    :param args: strings to be composed into the bash call.
+
+    """
+    return subprocess.check_call(args)
+
+
+def check_output(*args: List[str]) -> str:
+    """Execute a shell command, returning the output of the command.
+
+    :param args: strings to be composed into the bash call.
+
+    Include our env; pass in any extra keyword args.
+    """
+    return subprocess.check_output(args, universal_newlines=True).strip()
+
+
+def call(*args: List[str]) -> bool:
+    """Execute a shell command.
+
+    Return True if the call executed successfully (returned 0), or
+    False if it returned with an error code (return > 0)
+
+    :param args: strings to be composed into the bash call.
+    """
+    return not subprocess.call(args)
+
+
+class Framework(unittest.TestCase):
+
+    PREFIX = []
+    DUMP_DIR = '/tmp'
+    MACHINE = ''
+    DISTRO = 'bionic'
+    SNAP = 'microstack_stein_amd64.snap'
+    HORIZON_IP = '10.20.20.1'
+    INIT_FLAG = 'auto'
+
+    def install_snap(self, channel='dangerous', snap=None):
+        if snap is None:
+            snap = self.SNAP
+
+        check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic',
+              '--{}'.format(channel), snap)
+
+    def init_snap(self, flag='auto'):
+        check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag))
+
+    def multipass(self):
+
+        self.MACHINE = petname.generate()
+        self.PREFIX = ['multipass', 'exec', self.MACHINE, '--']
+
+        check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass')
+
+        check('multipass', 'launch', '--cpus', '2', '--mem', '8G', self.DISTRO,
+              '--name', self.MACHINE)
+        check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE))
+
+        # Figure out machine's ip
+        info = check_output('multipass', 'info', self.MACHINE, '--format',
+                            'json')
+        info = json.loads(info)
+        self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0]
+
+    def dump_logs(self):
+        if check_output('whoami') == 'zuul':
+            self.DUMP_DIR = "/home/zuul/zuul-output/logs"
+
+        check(*self.PREFIX,
+              'sudo', 'tar', 'cvzf',
+              '{}/dump.tar.gz'.format(self.DUMP_DIR),
+              '/var/snap/microstack/common/log',
+              '/var/snap/microstack/common/etc',
+              '/var/log/syslog')
+        if 'multipass' in self.PREFIX:
+            check('multipass', 'copy-files',
+                  '{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.')
+        print('Saved dump.tar.gz to local working dir.')
+
+    def setUp(self):
+        self.passed = False  # HACK: trigger (or skip) cleanup.
+        if os.environ.get('MULTIPASS'):
+            print("Booting a Multipass VM ...")
+            self.multipass()
+        print("Installing {}".format(self.SNAP))
+        self.install_snap()
+        print("Initializing the snap with --{}".format(self.INIT_FLAG))
+        self.init_snap(self.INIT_FLAG)
+
+    def tearDown(self):
+        """Either dump logs in the case of failure, or clean up."""
+
+        if not self.passed:
+            # Skip teardown in the case of failures, so that we can
+            # inspect them.
+            # TODO: I'd like to use the errors and failures list in
+            # the test result, but I was having trouble getting to it
+            # from this routine. Need to do more digging and possibly
+            # elimiate the self.passed hack.
+            print("Tests failed. Dumping logs and exiting.")
+            return self.dump_logs()
+
+        print("Tests complete. Tearing down.")
+        if 'multipass' in self.PREFIX:
+            check('sudo', 'multipass', 'delete', self.MACHINE)
+            check('sudo', 'multipass', 'purge')
+        else:
+            check('sudo', 'snap', 'remove', '--purge', 'microstack')

tests/test_basic.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python
+"""
+basic_test.py
+
+This is a basic test of microstack functionality. We verify that:
+
+1) We can install the snap.
+2) We can launch a cirros image.
+3) Horizon is running, and we can hit the landing page.
+4) We can login to Horizon successfully.
+
+The Horizon testing bits were are based on code generated by the Selinum
+Web IDE.
+
+"""
+
+import json
+import os
+import sys
+import time
+import unittest
+import xvfbwrapper
+from selenium import webdriver
+from selenium.webdriver.common.by import By
+
+sys.path.append(os.getcwd())
+
+from tests.framework import Framework, check, check_output, call  # noqa E402
+
+
+class TestBasics(Framework):
+
+    def setUp(self):
+        super(TestBasics, self).setUp()
+        # Setup Selenium Driver
+        self.display = xvfbwrapper.Xvfb(width=1280, height=720)
+        self.display.start()
+        self.driver = webdriver.PhantomJS()
+
+    def tearDown(self):
+        # Tear down selenium driver
+        self.driver.quit()
+        self.display.stop()
+
+        super(TestBasics, self).tearDown()
+
+    def test_basics(self):
+        """Basic test
+
+        Install microstack, and verify that we can launch a machine and
+        open the Horizon GUI.
+
+        """
+        launch = '/snap/bin/microstack.launch'
+        openstack = '/snap/bin/microstack.openstack'
+
+        print("Testing microstack.launch ...")
+
+        check(*self.PREFIX, launch, 'breakfast')
+
+        endpoints = check_output(
+            *self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list')
+
+        # Endpoints should be listening on 10.20.20.1
+        self.assertTrue("10.20.20.1" in endpoints)
+
+        # Endpoints should not contain localhost
+        self.assertFalse("localhost" in endpoints)
+
+        # Verify that microstack.launch completed successfully
+
+        # Ping the instance
+        ip = None
+        servers = check_output(*self.PREFIX, openstack,
+                               'server', 'list', '--format', 'json')
+        servers = json.loads(servers)
+        for server in servers:
+            if server['Name'] == 'breakfast':
+                ip = server['Networks'].split(",")[1].strip()
+                break
+
+        self.assertTrue(ip)
+
+        pings = 1
+        max_pings = 40
+        while not call(*self.PREFIX, 'ping', '-c', '1', ip):
+            pings += 1
+            if pings > max_pings:
+                self.assertFalse(True, msg='Max pings reached!')
+
+        print("Testing instances' ability to connect to the Internet")
+        # Test Internet connectivity
+        attempts = 1
+        max_attempts = 40
+        username = check_output(*self.PREFIX, 'whoami')
+
+        while not call(
+                *self.PREFIX,
+                'ssh',
+                '-oStrictHostKeyChecking=no',
+                '-i', '/home/{}/.ssh/id_microstack'.format(username),
+                'cirros@{}'.format(ip),
+                '--', 'ping', '-c', '1', '91.189.94.250'):
+            attempts += 1
+            if attempts > max_attempts:
+                self.assertFalse(True, msg='Unable to access the Internet!')
+            time.sleep(5)
+
+        if 'multipass' in self.PREFIX:
+            print("Opening {}:80 up to the outside world".format(
+                self.HORIZON_IP))
+
+            with open('/tmp/_10_hosts.py', 'w') as hosts:
+                hosts.write("""\
+# Allow all hosts to connect to this machine
+ALLOWED_HOSTS = ['*',]
+""")
+            check('multipass', 'copy-files', '/tmp/_10_hosts.py',
+                  '{}:/tmp/_10_hosts.py'.format(self.MACHINE))
+            check(
+                *self.PREFIX, 'sudo', 'cp', '/tmp/_10_hosts.py',
+                '/var/snap/microstack/common/etc/horizon/local_settings.d/'
+            )
+            check(*self.PREFIX, 'sudo', 'snap', 'restart', 'microstack')
+
+        print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP))
+        # Verify that our GUI is working properly
+        self.driver.get("http://{}/".format(self.HORIZON_IP))
+        # Login to horizon!
+        self.driver.find_element(By.ID, "id_username").click()
+        self.driver.find_element(By.ID, "id_username").send_keys("admin")
+        self.driver.find_element(By.ID, "id_password").send_keys("keystone")
+        self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
+        # Verify that we can click something on the dashboard -- e.g.,
+        # we're still not sitting at the login screen.
+        self.driver.find_element(By.LINK_TEXT, "Images").click()
+
+        self.passed = True
+
+
+if __name__ == '__main__':
+    # Run our tests, ignoring deprecation warnings and warnings about
+    # unclosed sockets. (TODO: setup a selenium server so that we can
+    # move from PhantomJS, which is deprecated, to to Selenium headless.)
+    unittest.main(warnings='ignore')

tests/test_control.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+"""
+control_test.py
+
+This is a test to verify that a control node gets setup properly. We verify:
+
+1) We can install the snap.
+2) Nova services are not running
+3) Other essential services are running
+4) TODO: the horizon dashboard works.
+
+"""
+
+import sys
+import os
+
+import unittest
+
+sys.path.append(os.getcwd())
+
+from tests.framework import Framework, check, check_output  # noqa E402
+
+
+class TestControlNode(Framework):
+
+    INIT_FLAG = 'control'
+
+    def test_control_node(self):
+
+        print("Checking output of services ...")
+        services = check_output(
+            *self.PREFIX, 'systemctl', 'status', 'snap.microstack.*',
+            '--no-page')
+
+        print("services: @@@")
+        print(services)
+
+        self.assertFalse('nova-' in services)
+        self.assertTrue('neutron-' in services)
+        self.assertTrue('keystone-' in services)
+
+        self.passed = True
+
+
+if __name__ == '__main__':
+    # Run our tests, ignoring deprecation warnings and warnings about
+    # unclosed sockets. (TODO: setup a selenium server so that we can
+    # move from PhantomJS, which is deprecated, to to Selenium headless.)
+    unittest.main(warnings='ignore')

tests/test_horizonlogin.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python
-"""
-test_horizonlogin.py
-
-This is a basic test of Horizon functionality. We verify that:
-
-1) Horizon is running, and we can hit the landing page.
-2) We can login successfully.
-
-This is based on code generated by the Selinum Web IDE.
-
-"""
-
-import os
-import socket
-import unittest
-import xvfbwrapper
-from selenium import webdriver
-from selenium.webdriver.common.by import By
-
-
-HORIZON_IP = os.environ.get("HORIZON_IP", "10.20.20.1")
-
-
-class TestHorizonlogin(unittest.TestCase):
-  def setUp(self):
-    self.display = xvfbwrapper.Xvfb(width=1280, height=720)
-    self.display.start()
-    self.driver = webdriver.PhantomJS()
-
-  def tearDown(self):
-    self.driver.quit()
-    self.display.stop()
-
-  def test_horizonlogin(self):
-    self.driver.get("http://{horizon_ip}/".format(horizon_ip=HORIZON_IP))
-    # Login to horizon!
-    self.driver.find_element(By.ID, "id_username").click()
-    self.driver.find_element(By.ID, "id_username").send_keys("admin")
-    self.driver.find_element(By.ID, "id_password").send_keys("keystone")
-    self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
-    # Verify that we can click something on the dashboard -- e.g.,
-    # we're still not sitting at the login screen.
-    self.driver.find_element(By.LINK_TEXT, "Images").click()
-
-if __name__ == '__main__':
-    # Run our tests, ignorning deprecation warnings and warnings about
-    # unclosed sockets. (TODO: setup a selenium server so that we can
-    # move from PhantomJS, which is deprecated, to to Selenium headless.)
-    unittest.main(warnings='ignore')

tools/basic_setup.sh

@@ -2,11 +2,11 @@
 
 set -ex
 
-sudo apt update
+#sudo apt update
 
 # Install the X virtual framebuffer, which is required for selenium
 # tests of the horizon dashboard.
 sudo apt install -y xvfb npm libfontconfig1
-sudo npm install -g phantomjs-prebuilt
+phantomjs -v || sudo npm install -g phantomjs-prebuilt
 # Verify that PhantomJS, our selenium web driver, works.
 phantomjs -v

tools/init/init/main.py

@@ -50,6 +50,7 @@ def main() -> None:
         questions.ExtCidr(),
         questions.OsPassword(),  # TODO: turn this off if COMPUTE.
         questions.IpForwarding(),
+        questions.ForceQemu(),
         # The following are not yet implemented:
         # questions.VmSwappiness(),
         # questions.FileHandleLimits(),

tools/init/init/questions.py

@@ -149,6 +149,37 @@ class IpForwarding(Question):
         check('sysctl', 'net.ipv4.ip_forward=1')
 
 
+class ForceQemu(Question):
+    _type = 'auto'
+
+    def yes(self, answer: str) -> None:
+        """Possibly force us to use qemu emulation rather than kvm."""
+
+        cpuinfo = check_output('cat', '/proc/cpuinfo')
+        if 'vmx' in cpuinfo or 'svm' in cpuinfo:
+            # We have processor extensions installed. No need to Force
+            # Qemu emulation.
+            return
+
+        _path = '{SNAP_COMMON}/etc/nova/nova.conf.d/hypervisor.conf'.format(
+            **_env)
+
+        with open(_path, 'w') as _file:
+            _file.write("""\
+[DEFAULT]
+compute_driver = libvirt.LibvirtDriver
+
+[workarounds]
+disable_rootwrap = True
+
+[libvirt]
+virt_type = qemu
+cpu_mode = host-model
+""")
+
+        # TODO: restart nova services when re-running this after init.
+
+
 class VmSwappiness(Question):
 
     _type = 'boolean'

tools/make-a-microstack.sh

@@ -1,18 +1,18 @@
 #!/bin/bash
 ##############################################################################
 #
-# This is a "very basic" test script for Microstack. It will install
-# the microstack snap on a vm, and dump you into a shell on the vm for
-# troubleshooting.
+# Make a microstack!
 #
-# The multipass snap and the petname debian package must be installed
-# on the host system in order to run this test.
+# This is a tool to very quickly spin up a multipass vm, install
+# microstack (from the compiled local .snap), and get a shell in
+# microstack's environment.
+#
+# It requires that you have installed petname.
 #
 ##############################################################################
 
 set -ex
 
-UPGRADE_FROM="none"
 DISTRO=18.04
 MACHINE=$(petname)
 
@@ -26,5 +26,5 @@ multipass exec $MACHINE -- \
 
 # Drop the user into a snap shell, as root.
 multipass exec $MACHINE -- \
-          sudo snap run --shell microstack.launch
+          sudo snap run --shell microstack.init
 

tox.ini

@@ -20,7 +20,11 @@ basepython=python3
 deps = -r{toxinidir}/test-requirements.txt
 commands =
     {toxinidir}/tools/lxd_build.sh
-    {toxinidir}/tests/basic-test.sh
+    flake8 {toxinidir}/tests/
+    # Specify tests in sequence, as they can't run in parallel if not
+    # using multipass.
+    {toxinidir}/tests/test_basic.py
+    {toxinidir}/tests/test_control.py    
 
 [testenv:multipass]
 # Default testing environment for a human operated machine. Builds the
@@ -31,16 +35,26 @@ commands =
 # a lot of things installed, including potentially the locally built
 # version of MicroStack!
 deps = -r{toxinidir}/test-requirements.txt
+setenv =
+    PATH = /snap/bin:{env:PATH}
+    MULTIPASS=true
 commands =
     {toxinidir}/tools/multipass_build.sh
-    {toxinidir}/tests/basic-test.sh -m
+    flake8 {toxinidir}/tests/
+    {toxinidir}/tests/test_basic.py
+    {toxinidir}/tests/test_control.py    
 
 [testenv:basic]
 # Just run basic_test.sh, with multipass support.
 deps = -r{toxinidir}/test-requirements.txt
+setenv =
+    MULTIPASS=true
+
 commands =
     {toxinidir}/tools/basic_setup.sh
-    {toxinidir}/tests/basic-test.sh -m
+    flake8 {toxinidir}/tests/
+    {toxinidir}/tests/test_basic.py
+    {toxinidir}/tests/test_control.py    
 
 [testenv:init_lint]
 deps = -r{toxinidir}/tools/init/test-requirements.txt
@@ -50,14 +64,5 @@ commands = flake8 {toxinidir}/tools/init/init/
 [testenv:init_unit]
 deps = -r{toxinidir}/tools/init/test-requirements.txt
        -r{toxinidir}/tools/init/requirements.txt
-commands = stestr run {posargs}
-
-[testenv:browser]
-# Run browser tests. Assumes that you have the snap installed and
-# initialized locally, and a valid DISPLAY (install xvfb for a virtual
-# one).
-# TODO: figure out how to integrate this w/ multipass. (e.g. setup
-# port forwarding and call into the mulitpass machine.)
-deps = -r{toxinidir}/test-requirements.txt
 commands =
-    {toxinidir}/tests/test_horizonlogin.py
+    stestr run --top-dir=./tools/init/ --test-path=./tools/init/tests/ {posargs}