Browse Source

Added refresh tests

Refactored test framework so that we have more flexibility in terms of
installing various versions of microstack before and after running
some tests. Moved in class "globals" into per instance variables,
to avoid broken cases with incomplete cleanup.

Added test_refresh.py, plus matching env in tox.

Refresh tests will fail currently, because we have some pending issues
that break refreshes. Fixing those is a subject for a different
commit.

Refactored cluster_test.py and control_test.py to use new framework.
Should (and do) pass.

Framework now cleans up multipass hosts regardless of whether or not
the tests passed. Leaning on the .tar.gz for local troubleshooting
helps us make it better for in gate troubleshooting.

Change-Id: I6a45b39132f5959c2944fe1ebbe10f71408ee777
changes/02/694402/7
Pete Vander Giessen 3 weeks ago
parent
commit
960685b91e
6 changed files with 292 additions and 167 deletions
  1. +168
    -51
      tests/framework.py
  2. +17
    -81
      tests/test_basic.py
  3. +12
    -26
      tests/test_cluster.py
  4. +7
    -9
      tests/test_control.py
  5. +78
    -0
      tests/test_refresh.py
  6. +10
    -0
      tox.ini

+ 168
- 51
tests/framework.py View File

@@ -3,9 +3,13 @@ import json
import unittest
import os
import subprocess
import time
import xvfbwrapper
from typing import List

import petname
from selenium import webdriver
from selenium.webdriver.common.by import By


# Setup logging
@@ -48,85 +52,198 @@ def call(*args: List[str]) -> bool:
return not subprocess.call(args)


class Framework(unittest.TestCase):
def gui_wrapper(func):
"""Start up selenium drivers, run a test, then tear them down."""

def wrapper(cls, *args, **kwargs):

# Setup Selenium Driver
cls.display = xvfbwrapper.Xvfb(width=1280, height=720)
cls.display.start()
cls.driver = webdriver.PhantomJS()

# Run function
try:
return func(cls, *args, **kwargs)

finally:
# Tear down driver
cls.driver.quit()
cls.display.stop()

return wrapper


PREFIX = []
DUMP_DIR = '/tmp'
MACHINE = ''
DISTRO = 'bionic'
SNAP = 'microstack_stein_amd64.snap'
HORIZON_IP = '10.20.20.1'
INIT_FLAG = 'auto'
class Host():
"""A host with MicroStack installed."""

def install_snap(self, channel='dangerous', snap=None):
def __init__(self):
self.prefix = []
self.dump_dir = '/tmp'
self.machine = ''
self.distro = 'bionic'
self.snap = 'microstack_stein_amd64.snap'
self.horizon_ip = '10.20.20.1'
self.host_type = 'localhost'

if os.environ.get('MULTIPASS'):
self.host_type = 'multipass'
print("Booting a Multipass VM ...")
self.multipass()

def install(self, snap=None, channel='dangerous'):
if snap is None:
snap = self.SNAP
snap = self.snap
print("Installing {}".format(snap))

check(*self.PREFIX, 'sudo', 'snap', 'install', '--classic',
check(*self.prefix, 'sudo', 'snap', 'install', '--classic',
'--{}'.format(channel), snap)

def init_snap(self, flag='auto'):
check(*self.PREFIX, 'sudo', 'microstack.init', '--{}'.format(flag))
def init(self, flag='auto'):
print("Initializing the snap with --{}".format(flag))
check(*self.prefix, 'sudo', 'microstack.init', '--{}'.format(flag))

def multipass(self):

self.MACHINE = petname.generate()
self.PREFIX = ['multipass', 'exec', self.MACHINE, '--']
distro = os.environ.get('DISTRO') or self.DISTRO
self.machine = petname.generate()
self.prefix = ['multipass', 'exec', self.machine, '--']
distro = os.environ.get('distro') or self.distro

check('sudo', 'snap', 'install', '--classic', '--edge', 'multipass')

check('multipass', 'launch', '--cpus', '2', '--mem', '8G', distro,
'--name', self.MACHINE)
check('multipass', 'copy-files', self.SNAP, '{}:'.format(self.MACHINE))
'--name', self.machine)
check('multipass', 'copy-files', self.snap, '{}:'.format(self.machine))

# Figure out machine's ip
info = check_output('multipass', 'info', self.MACHINE, '--format',
info = check_output('multipass', 'info', self.machine, '--format',
'json')
info = json.loads(info)
self.HORIZON_IP = info['info'][self.MACHINE]['ipv4'][0]
self.horizon_ip = info['info'][self.machine]['ipv4'][0]

def dump_logs(self):
# TODO: make unique log name
if check_output('whoami') == 'zuul':
self.DUMP_DIR = "/home/zuul/zuul-output/logs"
self.dump_dir = "/home/zuul/zuul-output/logs"

check(*self.PREFIX,
check(*self.prefix,
'sudo', 'tar', 'cvzf',
'{}/dump.tar.gz'.format(self.DUMP_DIR),
'{}/dump.tar.gz'.format(self.dump_dir),
'/var/snap/microstack/common/log',
'/var/snap/microstack/common/etc',
'/var/log/syslog')
if 'multipass' in self.PREFIX:
if 'multipass' in self.prefix:
check('multipass', 'copy-files',
'{}:/tmp/dump.tar.gz'.format(self.MACHINE), '.')
'{}:/tmp/dump.tar.gz'.format(self.machine), '.')
print('Saved dump.tar.gz to local working dir.')

def teardown(self):
if 'multipass' in self.prefix:
check('sudo', 'multipass', 'delete', self.machine)
check('sudo', 'multipass', 'purge')
else:
check('sudo', 'snap', 'remove', '--purge', 'microstack')


class Framework(unittest.TestCase):

def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.HOSTS = []

def get_host(self):
if self.HOSTS:
return self.HOSTS[0]
host = Host()
self.HOSTS.append(host)
return host

def add_host(self):
host = Host()
self.HOSTS.append(host)
return host

def verify_instance_networking(self, host, instance_name):
"""Verify that we have networking on an instance

We should be able to ping the instance.

And we should be able to reach the Internet.

"""
prefix = host.prefix

# Ping the instance
print("Testing ping ...")
ip = None
servers = check_output(*prefix, '/snap/bin/microstack.openstack',
'server', 'list', '--format', 'json')
servers = json.loads(servers)
for server in servers:
if server['Name'] == instance_name:
ip = server['Networks'].split(",")[1].strip()
break

self.assertTrue(ip)

pings = 1
max_pings = 600 # ~10 minutes!
while not call(*prefix, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(True, msg='Max pings reached!')

print("Testing instances' ability to connect to the Internet")
# Test Internet connectivity
attempts = 1
max_attempts = 300 # ~10 minutes!
username = check_output(*prefix, 'whoami')

while not call(
*prefix,
'ssh',
'-oStrictHostKeyChecking=no',
'-i', '/home/{}/.ssh/id_microstack'.format(username),
'cirros@{}'.format(ip),
'--', 'ping', '-c1', '91.189.94.250'):
attempts += 1
if attempts > max_attempts:
self.assertFalse(
True,
msg='Unable to access the Internet!')
time.sleep(1)

@gui_wrapper
def verify_gui(self, host):
"""Verify that Horizon Dashboard works

We should be able to reach the dashboard.

We should be able to login.

"""
# Test
print('Verifying GUI for (IP: {})'.format(host.horizon_ip))
# Verify that our GUI is working properly
self.driver.get("http://{}/".format(host.horizon_ip))
# Login to horizon!
self.driver.find_element(By.ID, "id_username").click()
self.driver.find_element(By.ID, "id_username").send_keys("admin")
self.driver.find_element(By.ID, "id_password").send_keys("keystone")
self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
# Verify that we can click something on the dashboard -- e.g.,
# we're still not sitting at the login screen.
self.driver.find_element(By.LINK_TEXT, "Images").click()

def setUp(self):
self.passed = False # HACK: trigger (or skip) cleanup.
if os.environ.get('MULTIPASS'):
print("Booting a Multipass VM ...")
self.multipass()
print("Installing {}".format(self.SNAP))
self.install_snap()
print("Initializing the snap with --{}".format(self.INIT_FLAG))
self.init_snap(self.INIT_FLAG)

def tearDown(self):
"""Either dump logs in the case of failure, or clean up."""

if not self.passed:
# Skip teardown in the case of failures, so that we can
# inspect them.
# TODO: I'd like to use the errors and failures list in
# the test result, but I was having trouble getting to it
# from this routine. Need to do more digging and possibly
# elimiate the self.passed hack.
print("Tests failed. Dumping logs and exiting.")
return self.dump_logs()

print("Tests complete. Tearing down.")
if 'multipass' in self.PREFIX:
check('sudo', 'multipass', 'delete', self.MACHINE)
check('sudo', 'multipass', 'purge')
else:
check('sudo', 'snap', 'remove', '--purge', 'microstack')
"""Clean hosts up, possibly leaving debug information behind."""

print("Tests complete. Cleaning up.")
while self.HOSTS:
host = self.HOSTS.pop()
if not self.passed:
print("Dumping logs for {}".format(host.machine))
host.dump_logs()
host.teardown()

+ 17
- 81
tests/test_basic.py View File

@@ -14,14 +14,10 @@ Web IDE.

"""

import json
import os
import sys
import time
import unittest
import xvfbwrapper
from selenium import webdriver
from selenium.webdriver.common.by import By

sys.path.append(os.getcwd())

@@ -30,20 +26,6 @@ from tests.framework import Framework, check, check_output, call # noqa E402

class TestBasics(Framework):

def setUp(self):
super(TestBasics, self).setUp()
# Setup Selenium Driver
self.display = xvfbwrapper.Xvfb(width=1280, height=720)
self.display.start()
self.driver = webdriver.PhantomJS()

def tearDown(self):
# Tear down selenium driver
self.driver.quit()
self.display.stop()

super(TestBasics, self).tearDown()

def test_basics(self):
"""Basic test

@@ -51,16 +33,13 @@ class TestBasics(Framework):
open the Horizon GUI.

"""
launch = '/snap/bin/microstack.launch'
openstack = '/snap/bin/microstack.openstack'

print("Testing microstack.launch ...")

check(*self.PREFIX, launch, 'cirros', '--name', 'breakfast',
'--retry')
host = self.get_host()
host.install()
host.init()
prefix = host.prefix

endpoints = check_output(
*self.PREFIX, '/snap/bin/microstack.openstack', 'endpoint', 'list')
*prefix, '/snap/bin/microstack.openstack', 'endpoint', 'list')

# Endpoints should be listening on 10.20.20.1
self.assertTrue("10.20.20.1" in endpoints)
@@ -68,66 +47,23 @@ class TestBasics(Framework):
# Endpoints should not contain localhost
self.assertFalse("localhost" in endpoints)

if 'multipass' in self.PREFIX:
# Verify that microstack.launch completed successfully
# Skip these tests in the gate, as they are not reliable there.
# TODO: fix these in the gate!

# Ping the instance
ip = None
servers = check_output(*self.PREFIX, openstack,
'server', 'list', '--format', 'json')
servers = json.loads(servers)
for server in servers:
if server['Name'] == 'breakfast':
ip = server['Networks'].split(",")[1].strip()
break

self.assertTrue(ip)

pings = 1
max_pings = 600 # ~10 minutes!
while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(True, msg='Max pings reached!')

print("Testing instances' ability to connect to the Internet")
# Test Internet connectivity
attempts = 1
max_attempts = 300 # ~10 minutes!
username = check_output(*self.PREFIX, 'whoami')

while not call(
*self.PREFIX,
'ssh',
'-oStrictHostKeyChecking=no',
'-i', '/home/{}/.ssh/id_microstack'.format(username),
'cirros@{}'.format(ip),
'--', 'ping', '-c1', '91.189.94.250'):
attempts += 1
if attempts > max_attempts:
self.assertFalse(
True,
msg='Unable to access the Internet!')
time.sleep(1)

# We should be able to launch an instance
print("Testing microstack.launch ...")
check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'breakfast', '--retry')

# ... and ping it
# Skip these tests in the gate, as they are not reliable there.
# TODO: fix these in the gate!
if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')
else:
# Artificial wait, to allow for stuff to settle for the GUI test.
# TODO: get rid of this, when we drop the ping tests back int.
time.sleep(10)

print('Verifying GUI for (IP: {})'.format(self.HORIZON_IP))
# Verify that our GUI is working properly
self.driver.get("http://{}/".format(self.HORIZON_IP))
# Login to horizon!
self.driver.find_element(By.ID, "id_username").click()
self.driver.find_element(By.ID, "id_username").send_keys("admin")
self.driver.find_element(By.ID, "id_password").send_keys("keystone")
self.driver.find_element(By.CSS_SELECTOR, "#loginBtn > span").click()
# Verify that we can click something on the dashboard -- e.g.,
# we're still not sitting at the login screen.
self.driver.find_element(By.LINK_TEXT, "Images").click()
# The Horizon Dashboard should function
self.verify_gui(host)

self.passed = True


+ 12
- 26
tests/test_cluster.py View File

@@ -12,7 +12,6 @@ vms.

import json
import os
import petname
import sys
import unittest

@@ -26,27 +25,6 @@ os.environ['MULTIPASS'] = 'true' # TODO better way to do this.

class TestCluster(Framework):

INIT_FLAG = 'control'

def _compute_node(self, channel='dangerous'):
"""Make a compute node.

TODO: refactor framework so that we can fold a lot of this
into the parent framework. There's a lot of dupe code here.

"""
machine = petname.generate()
prefix = ['multipass', 'exec', machine, '--']

check('multipass', 'launch', '--cpus', '2', '--mem', '8G',
self.DISTRO, '--name', machine)

check('multipass', 'copy-files', self.SNAP, '{}:'.format(machine))
check(*prefix, 'sudo', 'snap', 'install', '--classic',
'--{}'.format(channel), self.SNAP)

return machine, prefix

def test_cluster(self):

# After the setUp step, we should have a control node running
@@ -54,18 +32,26 @@ class TestCluster(Framework):
# address.

openstack = '/snap/bin/microstack.openstack'
control_host = self.get_host()
control_host.install()
control_host.init(flag='control')

cluster_password = check_output(*self.PREFIX, 'sudo', 'snap',
control_prefix = control_host.prefix
cluster_password = check_output(*control_prefix, 'sudo', 'snap',
'get', 'microstack',
'config.cluster.password')
control_ip = check_output(*self.PREFIX, 'sudo', 'snap',
control_ip = check_output(*control_prefix, 'sudo', 'snap',
'get', 'microstack',
'config.network.control-ip')

self.assertTrue(cluster_password)
self.assertTrue(control_ip)

compute_machine, compute_prefix = self._compute_node()
compute_host = self.add_host()
compute_host.install()

compute_machine = compute_host.machine
compute_prefix = compute_host.prefix

# TODO add the following to args for init
check(*compute_prefix, 'sudo', 'snap', 'set', 'microstack',
@@ -110,7 +96,7 @@ class TestCluster(Framework):
max_pings = 60 # ~1 minutes
# Ping the machine from the control node (we don't have
# networking wired up for the other nodes).
while not call(*self.PREFIX, 'ping', '-c1', '-w1', ip):
while not call(*control_prefix, 'ping', '-c1', '-w1', ip):
pings += 1
if pings > max_pings:
self.assertFalse(

+ 7
- 9
tests/test_control.py View File

@@ -2,12 +2,7 @@
"""
control_test.py

This is a test to verify that a control node gets setup properly. We verify:

1) We can install the snap.
2) Nova services are not running
3) Other essential services are running
4) TODO: the horizon dashboard works.
This is a test to verify that a control node gets setup properly.

"""

@@ -23,17 +18,19 @@ from tests.framework import Framework, check, check_output # noqa E402

class TestControlNode(Framework):

INIT_FLAG = 'control'

def test_control_node(self):
"""A control node has all services running, so this shouldn't be any
different than our standard setup.

"""

host = self.get_host()
host.install()
host.init(flag='control')

print("Checking output of services ...")
services = check_output(
*self.PREFIX, 'systemctl', 'status', 'snap.microstack.*',
*host.prefix, 'systemctl', 'status', 'snap.microstack.*',
'--no-page')

print("services: @@@")
@@ -41,6 +38,7 @@ class TestControlNode(Framework):

self.assertTrue('neutron-' in services)
self.assertTrue('keystone-' in services)
self.assertTrue('nova-' in services)

self.passed = True


+ 78
- 0
tests/test_refresh.py View File

@@ -0,0 +1,78 @@
#!/usr/bin/env python
"""
refresh_test.py

Verify that existing installs can refresh to our newly built snap.

"""
import json
import os
import sys
import unittest

sys.path.append(os.getcwd())

from tests.framework import Framework, check, check_output, call # noqa E402


class TestRefresh(Framework):
"""Refresh from beta and from edge."""

def test_refresh_from_beta(self):
self._refresh_from('beta')
self.passed = True

def test_refresh_from_edge(self):
self._refresh_from('edge')
self.passed = True

def _refresh_from(self, refresh_from='beta'):
"""Refresh test

Like the basic test, but we refresh first.

"""
print("Installing and verfying {} ...".format(refresh_from))
host = self.get_host()
host.install(snap="microstack", channel=refresh_from)
host.init()
prefix = host.prefix

check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'breakfast', '--retry')

if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')

print("Upgrading ...")
host.install() # Install compiled snap
# Should not need to re-init

print("Verifying that refresh completed successfully ...")

# Check our existing instance, starting it if necessary.
if json.loads(check_output(*prefix, '/snap/bin/microstack.openstack',
'server', 'show', 'breakfast',
'--format', 'json'))['status'] == 'SHUTOFF':
print("Starting breakfast (TODO: auto start.)")
check(*prefix, '/snap/bin/microstack.openstack', 'server', 'start',
'breakfast')

# Launch another instance
check(*prefix, '/snap/bin/microstack.launch', 'cirros',
'--name', 'lunch', '--retry')

# Verify networking
if 'multipass' in prefix:
self.verify_instance_networking(host, 'breakfast')
self.verify_instance_networking(host, 'lunch')

# Verify GUI
self.verify_gui(host)


if __name__ == '__main__':
# Run our tests, ignoring deprecation warnings and warnings about
# unclosed sockets. (TODO: setup a selenium server so that we can
# move from PhantomJS, which is deprecated, to to Selenium headless.)
unittest.main(warnings='ignore')

+ 10
- 0
tox.ini View File

@@ -49,6 +49,16 @@ commands =
setenv =
MULTIPASS=true

[testenv:refresh]
# Verify that we can refresh MicroStack
setenv =
MULTIPASS=true

commands =
{toxinidir}/tools/basic_setup.sh
flake8 {toxinidir}/tests/
{toxinidir}/tests/test_refresh.py

[testenv:xenial]
# Run basic tests, under xenial.
setenv =

Loading…
Cancel
Save