Add some basic functional tests

Adds the required infraestructure to run functional tests:

- With multiple backends, giving meaningful name to the tests
- With externally defined backends configuration
- Automatic cleanup of attachments, connections, volumes, and snapshots.

Additionally provides a basic LVM configuration for the functional tests
and a script to set things up in the system before running the actual
tests.

The tests have been validated using a configuration that included 3
backends:

- LVM
- XtremIO
- Kaminario

requirements_dev.txt

@@ -1,3 +1,5 @@
+unittest2
+pyyaml
 pip==8.1.2
 bumpversion==0.5.3
 wheel==0.29.0

setup.py

@@ -79,6 +79,6 @@ setup(
         'Programming Language :: Python :: 3.4',
         'Programming Language :: Python :: 3.5',
     ],
-    test_suite='tests',
+    test_suite='unittest2.collector',
     tests_require=test_requirements,
 )

tests/functional/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tests/functional/base_tests.py

@@ -0,0 +1,158 @@
+# Copyright (c) 2018, Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import functools
+import os
+import subprocess
+
+import unittest2
+import yaml
+
+import cinderlib
+
+
+def set_backend(func, new_name, backend_name):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        self.backend = cinderlib.Backend.backends[backend_name]
+        return func(self, *args, **kwargs)
+    wrapper.__name__ = new_name
+    wrapper.__wrapped__ = func
+    return wrapper
+
+
+def test_all_backends(cls):
+    config = BaseFunctTestCase.ensure_config_loaded()
+    for fname, func in cls.__dict__.items():
+        if fname.startswith('test_'):
+            for backend in config['backends']:
+                bname = backend['volume_backend_name']
+                test_name = '%s_on_%s' % (fname, bname)
+                setattr(cls, test_name, set_backend(func, test_name, bname))
+            delattr(cls, fname)
+    return cls
+
+
+class BaseFunctTestCase(unittest2.TestCase):
+    DEFAULTS = {'logs': False, 'venv_sudo': False}
+    FNULL = open(os.devnull, 'w')
+    CONFIG_FILE = os.environ.get('CL_FTEST_CFG', 'tests/functional/lvm.yaml')
+    tests_config = None
+
+    @classmethod
+    def ensure_config_loaded(cls):
+        if not cls.tests_config:
+            # Read backend configuration file
+            with open(cls.CONFIG_FILE, 'r') as f:
+                cls.tests_config = yaml.load(f)
+            # Set configuration default values
+            for k, v in cls.DEFAULTS.items():
+                cls.tests_config.setdefault(k, v)
+        return cls.tests_config
+
+    @classmethod
+    def setUpClass(cls):
+        config = cls.ensure_config_loaded()
+
+        if config['venv_sudo']:
+            # NOTE(geguileo): For some drivers need to use a custom sudo script
+            # to find virtualenv commands (ie: cinder-rtstool).
+            path = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+            cls.root_helper = os.path.join(path, 'virtualenv-sudo.sh')
+        else:
+            cls.root_helper = 'sudo'
+        cinderlib.setup(root_helper=cls.root_helper,
+                        disable_logs=not config['logs'])
+
+        # Initialize backends
+        cls.backends = [cinderlib.Backend(**cfg) for cfg in
+                        config['backends']]
+
+        # Set current backend, by default is the first
+        cls.backend = cls.backends[0]
+
+    @classmethod
+    def tearDownClass(cls):
+        errors = []
+        # Do the cleanup of the resources the tests haven't cleaned up already
+        for backend in cls.backends:
+            # For each of the volumes that haven't been deleted delete the
+            # snapshots that are still there and then the volume.
+            # NOTE(geguileo): Don't use volumes and snapshots iterables since
+            # they are modified when deleting.
+            for vol in list(backend.volumes):
+                for snap in list(vol.snapshots):
+                    try:
+                        snap.delete()
+                    except Exception as exc:
+                        errors.append('Error deleting snapshot %s from volume '
+                                      '%s: %s' % (snap.id, vol.id, exc))
+                # Detach if locally attached
+                if vol.local_attach:
+                    try:
+                        vol.detach()
+                    except Exception as exc:
+                        errors.append('Error detaching %s for volume %s %s: '
+                                      '%s' % (vol.local_attach.path, vol.id,
+                                              exc))
+
+                # Disconnect any existing connections
+                for conn in vol.connections:
+                    try:
+                        conn.disconnect()
+                    except Exception as exc:
+                        errors.append('Error disconnecting volume %s: %s' %
+                                      (vol.id, exc))
+
+                try:
+                    vol.delete()
+                except Exception as exc:
+                    errors.append('Error deleting volume %s: %s' %
+                                  (vol.id, exc))
+        if errors:
+            raise Exception('Errors on test cleanup: %s' % '\n\t'.join(errors))
+
+    def _root_execute(self, *args, **kwargs):
+        cmd = [self.root_helper]
+        cmd.extend(args)
+        cmd.extend("%s=%s" % (k, v) for k, v in kwargs.items())
+        return subprocess.check_output(cmd, stderr=self.FNULL)
+
+    def _create_vol(self, backend=None, **kwargs):
+        if not backend:
+            backend = self.backend
+
+        vol_size = kwargs.setdefault('size', 1)
+        name = kwargs.setdefault('name', backend.id)
+
+        vol = backend.create_volume(**kwargs)
+
+        self.assertEqual('available', vol.status)
+        self.assertEqual(vol_size, vol.size)
+        self.assertEqual(name, vol.display_name)
+        self.assertIn(vol, backend.volumes)
+        return vol
+
+    def _create_snap(self, vol, **kwargs):
+        name = kwargs.setdefault('name', vol.id)
+
+        snap = vol.create_snapshot(name=vol.id)
+
+        self.assertEqual('available', snap.status)
+        self.assertEqual(vol.size, snap.volume_size)
+        self.assertEqual(name, snap.display_name)
+
+        self.assertIn(snap, vol.snapshots)
+        return snap

tests/functional/lvm-prepare.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+
+# Must be run as root
+
+dd if=/dev/zero of=cinder-volumes bs=1048576 seek=22527 count=1
+lodevice=`losetup --show -f ./cinder-volumes`
+pvcreate $lodevice
+vgcreate cinder-volumes $lodevice
+vgscan --cache

tests/functional/lvm.yaml

@@ -0,0 +1,19 @@
+# For Fedora, CentOS, RHEL we require the targetcli package.
+# For Ubuntu we require lio-utils or changing the target iscsi_helper
+#
+
+# Logs are way too verbose, so we disable them
+logs: false
+
+# LVM backend uses cinder-rtstool command that is installed by Cinder in the
+# virtual environment, so we need the custom sudo command that inherits the
+# virtualenv binaries PATH
+venv_sudo: true
+
+# We only define one backend
+backends:
+    - volume_backend_name: lvm
+      volume_driver: cinder.volume.drivers.lvm.LVMVolumeDriver
+      volume_group: cinder-volumes
+      iscsi_protocol: iscsi
+      iscsi_helper: lioadm

tests/functional/tests_basic.py

@@ -0,0 +1,155 @@
+# Copyright (c) 2018, Red Hat, Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import tempfile
+
+import base_tests
+
+
+@base_tests.test_all_backends
+class BackendFunctBasic(base_tests.BaseFunctTestCase):
+
+    def test_stats(self):
+        stats = self.backend.stats()
+        self.assertIn('vendor_name', stats)
+        self.assertIn('volume_backend_name', stats)
+        pools_info = stats.get('pools', [stats])
+        for pool_info in pools_info:
+            self.assertIn('free_capacity_gb', pool_info)
+            self.assertIn('total_capacity_gb', pool_info)
+
+    def test_create_volume(self):
+        self._create_vol(self.backend)
+        # We are not testing delete, so leave the deletion to the tearDown
+
+    def test_create_delete_volume(self):
+        vol = self._create_vol(self.backend)
+
+        vol.delete()
+        self.assertEqual('deleted', vol.status)
+        self.assertTrue(vol.deleted)
+        self.assertNotIn(vol, self.backend.volumes)
+
+        # Confirm idempotency of the operation by deleting it again
+        vol._ovo.status = 'error'
+        vol._ovo.deleted = False
+        vol.delete()
+        self.assertEqual('deleted', vol.status)
+        self.assertTrue(vol.deleted)
+
+    def test_create_snapshot(self):
+        vol = self._create_vol(self.backend)
+        self._create_snap(vol)
+        # We are not testing delete, so leave the deletion to the tearDown
+
+    def test_create_delete_snapshot(self):
+        vol = self._create_vol(self.backend)
+        snap = self._create_snap(vol)
+
+        snap.delete()
+        self.assertEqual('deleted', snap.status)
+        self.assertTrue(snap.deleted)
+        self.assertNotIn(snap, vol.snapshots)
+
+        # Confirm idempotency of the operation by deleting it again
+        snap._ovo.status = 'error'
+        snap._ovo.deleted = False
+        snap.delete()
+        self.assertEqual('deleted', snap.status)
+        self.assertTrue(snap.deleted)
+
+    def test_attach_volume(self):
+        vol = self._create_vol(self.backend)
+
+        attach = vol.attach()
+        path = attach.path
+
+        self.assertIs(attach, vol.local_attach)
+        self.assertIn(attach, vol.connections)
+
+        self.assertTrue(os.path.exists(path))
+        # We are not testing detach, so leave it to the tearDown
+
+    def test_attach_detach_volume(self):
+        vol = self._create_vol(self.backend)
+
+        attach = vol.attach()
+        self.assertIs(attach, vol.local_attach)
+        self.assertIn(attach, vol.connections)
+
+        vol.detach()
+        self.assertIsNone(vol.local_attach)
+        self.assertNotIn(attach, vol.connections)
+
+    def test_attach_detach_volume_via_attachment(self):
+        vol = self._create_vol(self.backend)
+
+        attach = vol.attach()
+        self.assertTrue(attach.attached)
+        path = attach.path
+
+        self.assertTrue(os.path.exists(path))
+
+        attach.detach()
+        self.assertFalse(attach.attached)
+        self.assertIsNone(vol.local_attach)
+
+        # We haven't disconnected the volume, just detached it
+        self.assertIn(attach, vol.connections)
+
+        attach.disconnect()
+        self.assertNotIn(attach, vol.connections)
+
+    def test_disk_io(self):
+        data = '0123456789' * 100
+
+        vol = self._create_vol(self.backend)
+
+        attach = vol.attach()
+
+        # TODO(geguileo: This will not work on Windows, for that we need to
+        # pass delete=False and do the manual deletion ourselves.
+        with tempfile.NamedTemporaryFile() as f:
+            f.write(data)
+            f.flush()
+            self._root_execute('dd', 'if=' + f.name, of=attach.path)
+
+        # Detach without removing the mapping of the volume since it's faster
+        attach.detach()
+
+        # Reattach, using old mapping, to validate data is there
+        attach.attach()
+        stdout = self._root_execute('dd', 'if=' + attach.path, count=1,
+                                    ibs=len(data))
+
+        self.assertEqual(data, stdout)
+        vol.detach()
+
+    def test_connect_disconnect_volume(self):
+        # TODO(geguileo): Implement the test
+        pass
+
+    def test_connect_disconnect_multiple_volumes(self):
+        # TODO(geguileo): Implement the test
+        pass
+
+    def test_connect_disconnect_multiple_times(self):
+        # TODO(geguileo): Implement the test
+        pass
+
+    def test_stats_with_creation(self):
+        # TODO(geguileo): Implement the test
+        pass

tests/functional/virtualenv-sudo.sh

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+# Script to ensure that calling commands added in the virtualenv with sudo will
+# be able to find them during the functional tests, ie: cinder-rtstool
+
+params=()
+for arg in "$@"; do params+=("\"$arg\""); done
+params="${params[@]}"
+sudo -E --preserve-env=PATH /bin/bash -c "$params"

tests/unit/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-

tests/unit/test_cinderlib.py

@@ -8,12 +8,12 @@ test_cinderlib
 Tests for `cinderlib` module.
 """
 
-import unittest
+import unittest2
 
-from cinderlib import cinderlib
+import cinderlib
 
 
-class TestCinderlib(unittest.TestCase):
+class TestCinderlib(unittest2.TestCase):
 
     def setUp(self):
         pass

tox.ini

@@ -1,5 +1,7 @@
 [tox]
-envlist = py26, py27, py33, py34, py35, flake8
+envlist = py27, py33, py34, py35, flake8
+skipsdist=True
+setenv = VIRTUAL_ENV={envdir}
 
 [testenv:flake8]
 basepython=python
@@ -9,8 +11,22 @@ deps=
     -r{toxinidir}/requirements_docs.txt
 
 [testenv]
+usedevelop=True
+install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt?h=stable/pike} {opts} {packages}
 setenv =
     PYTHONPATH = {toxinidir}:{toxinidir}/cinderlib
 deps= -r{toxinidir}/requirements_dev.txt
 
-commands = python setup.py test
+commands =
+    unit2 discover -s tests/unit []
+
+[testenv:functional]
+usedevelop=True
+# Workaround for https://github.com/tox-dev/tox/issues/425
+basepython=python2.7
+envdir = {toxworkdir}/py27
+
+# Pass on the location of the backend configuration to the tests
+setenv = CL_FTEST_CFG = {env:CL_FTEST_CFG:tests/functional/lvm.yaml}
+commands =
+    unit2 discover -v -s tests/functional []