Add functional tests for swift

This change adds functional tests for swift.

Co-Authored-By: Louis Taylor <louis@kragniz.eu>

Change-Id: Ibdd17eebb49a5d96c5cfb02c0ec6bac4826aac7f

.gitignore

@@ -27,3 +27,6 @@ covhtml/
 
 # Files created by releasenotes build
 releasenotes/build
+
+#Files created by functional tests
+functional_testing.conf

.testr.conf

@@ -2,7 +2,7 @@
 test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} \
              OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} \
              OS_TEST_TIMEOUT=${OS_TEST_TIMEOUT:-160} \
-             ${PYTHON:-python} -m subunit.run discover -t ./ ./glance_store/tests $LISTOPT $IDOPTION
+             ${PYTHON:-python} -m subunit.run discover -t ./ ${OS_TEST_PATH:-./glance_store/tests/unit} $LISTOPT $IDOPTION
 
 test_id_option=--load-list $IDFILE
 test_list_option=--list

functional_testing.conf.sample

@@ -0,0 +1,9 @@
+[tests]
+stores = file,swift
+
+[admin]
+user = admin:admin
+key = secretadmin
+auth_version = 2
+auth_address = http://localhost:35357/v2.0
+region = RegionOne

glance_store/tests/functional/base.py

@@ -13,6 +13,10 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+try:
+    import configparser as ConfigParser
+except ImportError:
+    from six.moves import configparser as ConfigParser
 from io import BytesIO
 
 import glance_store
@@ -21,8 +25,6 @@ import testtools
 
 CONF = cfg.CONF
 
-STORES = ['file', ]
-
 UUID1 = '961973d8-3360-4364-919e-2c197825dbb4'
 UUID2 = 'e03cf3b1-3070-4497-a37d-9703edfb615b'
 UUID3 = '0d7f89b2-e236-45e9-b081-561cd3102e92'
@@ -35,16 +37,23 @@ class Base(testtools.TestCase):
     def __init__(self, driver_name, *args, **kwargs):
         super(Base, self).__init__(*args, **kwargs)
         self.driver_name = driver_name
+        self.config = ConfigParser.RawConfigParser()
+        self.config.read('functional_testing.conf')
 
         glance_store.register_opts(CONF)
 
     def setUp(self):
         super(Base, self).setUp()
 
-        if self.driver_name not in STORES:
+        stores = self.config.get('tests', 'stores').split(',')
+        if self.driver_name not in stores:
             self.skipTest('Not running %s store tests' % self.driver_name)
 
         CONF.set_override('stores', [self.driver_name], group='glance_store')
+        CONF.set_override('default_store',
+                          [self.driver_name],
+                          group='glance_store'
+                          )
         glance_store.create_stores()
         self.store = glance_store.backend._load_store(CONF, self.driver_name)
         self.store.configure()

glance_store/tests/functional/hooks/gate_hook.sh

@@ -14,9 +14,18 @@
 
 # This script is executed inside gate_hook function in devstack gate.
 
-GLANCE_STORE_DRIVER=$1
+# NOTE(NiallBunting) In future these tests should be split up. This means
+# based on the store passed to this script in $1, particular services can
+# be enabled.
+GLANCE_STORE_DRIVER=swift
 
-ENABLED_SERVICES="mysql,key,glance,"
+ENABLED_SERVICES+=",key,glance"
+
+case $GLANCE_STORE_DRIVER in
+    swift)
+        ENABLED_SERVICES+=",s-proxy,s-account,s-container,s-object,"
+        ;;
+esac
 
 export DEVSTACK_LOCAL_CONFIG
 export DEVSTACK_GATE_INSTALL_TESTONLY=1

glance_store/tests/functional/hooks/post_test_hook.sh

@@ -14,19 +14,47 @@
 
 # This script is executed inside post_test_hook function in devstack gate.
 
+set -xe
+
+export GLANCE_STORE_DIR="$BASE/new/glance_store"
+SCRIPTS_DIR="/usr/os-testr-env/bin/"
+venv=${1:-"dsvm-functional"}
+
+function generate_test_logs {
+    local path="$1"
+    # Compress all $path/*.txt files and move the directories holding those
+    # files to /opt/stack/logs. Files with .log suffix have their
+    # suffix changed to .txt (so browsers will know to open the compressed
+    # files and not download them).
+    if [ -d "$path" ]
+    then
+        sudo find $path -iname "*.log" -type f -exec mv {} {}.txt \; -exec gzip -9 {}.txt \;
+        sudo mv $path/* /opt/stack/logs/
+    fi
+}
+
 function generate_testr_results {
     if [ -f .testrepository/0 ]; then
-        sudo .tox/functional/bin/testr last --subunit > $WORKSPACE/testrepository.subunit
-        sudo mv $WORKSPACE/testrepository.subunit $BASE/logs/testrepository.subunit
-        sudo .tox/functional/bin/python /usr/local/jenkins/slave_scripts/subunit2html.py $BASE/logs/testrepository.subunit $BASE/logs/testr_results.html
-        sudo gzip -9 $BASE/logs/testrepository.subunit
-        sudo gzip -9 $BASE/logs/testr_results.html
-        sudo chown jenkins:jenkins $BASE/logs/testrepository.subunit.gz $BASE/logs/testr_results.html.gz
-        sudo chmod a+r $BASE/logs/testrepository.subunit.gz $BASE/logs/testr_results.html.gz
+        # Give job user rights to access tox logs
+        sudo -H -u "$owner" chmod o+rw .
+        sudo -H -u "$owner" chmod o+rw -R .testrepository
+
+        if [[ -f ".testrepository/0" ]] ; then
+            "subunit-1to2" < .testrepository/0 > ./testrepository.subunit
+            $SCRIPTS_DIR/subunit2html ./testrepository.subunit testr_results.html
+            gzip -9 ./testrepository.subunit
+            gzip -9 ./testr_results.html
+            sudo mv ./*.gz /opt/stack/logs/
+        fi
+
+        if [[ "$venv" == dsvm-functional* ]] ; then
+            generate_test_logs "/tmp/${venv}-logs"
+        fi
+
     fi
 }
 
-export GLANCE_STORE_DIR="$BASE/new/glance_store"
+owner=jenkins
 
 # Get admin credentials
 cd $BASE/new/devstack
@@ -35,13 +63,18 @@ source openrc admin admin
 # Go to the glance_store dir
 cd $GLANCE_STORE_DIR
 
-sudo chown -R jenkins:stack $GLANCE_STORE_DIR
+sudo chown -R $owner:stack $GLANCE_STORE_DIR
+
+sudo cp $GLANCE_STORE_DIR/functional_testing.conf.sample $GLANCE_STORE_DIR/functional_testing.conf
+
+# Set admin creds
+iniset $GLANCE_STORE_DIR/functional_testing.conf admin key $ADMIN_PASSWORD
 
 # Run tests
 echo "Running glance_store functional test suite"
 set +e
 # Preserve env for OS_ credentials
-sudo -E -H -u jenkins tox -efunctional
+sudo -E -H -u jenkins tox -e functional
 EXIT_CODE=$?
 set -e
 

glance_store/tests/functional/test_functional.py

@@ -15,10 +15,13 @@
 
 
 import logging
+import random
 import shutil
 import tempfile
+import time
 
 from oslo_config import cfg
+import swiftclient
 
 from glance_store.tests.functional import base
 
@@ -42,3 +45,67 @@ class TestFilesystem(base.BaseFunctionalTests):
     def tearDown(self):
         shutil.rmtree(self.tmp_image_dir)
         super(TestFilesystem, self).tearDown()
+
+
+class TestSwift(base.BaseFunctionalTests):
+
+    def __init__(self, *args, **kwargs):
+        super(TestSwift, self).__init__('swift', *args, **kwargs)
+
+        self.auth = self.config.get('admin', 'auth_address')
+        user = self.config.get('admin', 'user')
+        self.key = self.config.get('admin', 'key')
+        self.region = self.config.get('admin', 'region')
+
+        self.tenant, self.username = user.split(':')
+
+        CONF.set_override('swift_store_user',
+                          user,
+                          group='glance_store')
+        CONF.set_override('swift_store_auth_address',
+                          self.auth,
+                          group='glance_store')
+        CONF.set_override('swift_store_key',
+                          self.key,
+                          group='glance_store')
+        CONF.set_override('swift_store_create_container_on_put',
+                          True,
+                          group='glance_store')
+        CONF.set_override('swift_store_region',
+                          self.region,
+                          group='glance_store')
+        CONF.set_override('swift_store_create_container_on_put',
+                          True,
+                          group='glance_store')
+
+    def setUp(self):
+        self.container = ("glance_store_container_" +
+                          str(int(random.random() * 1000)))
+
+        CONF.set_override('swift_store_container',
+                          self.container,
+                          group='glance_store')
+
+        super(TestSwift, self).setUp()
+
+    def tearDown(self):
+        for x in range(1, 4):
+            time.sleep(x)
+            try:
+                swift = swiftclient.client.Connection(auth_version='2',
+                                                      user=self.username,
+                                                      key=self.key,
+                                                      tenant_name=self.tenant,
+                                                      authurl=self.auth)
+                _, objects = swift.get_container(self.container)
+                for obj in objects:
+                    swift.delete_object(self.container, obj.get('name'))
+                swift.delete_container(self.container)
+            except Exception:
+                if x < 3:
+                    pass
+                else:
+                    raise
+            else:
+                break
+        super(TestSwift, self).tearDown()

tox.ini

@@ -10,6 +10,7 @@ install_command = pip install --allow-all-external --allow-insecure netaddr -U {
 deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
        .[s3,vmware,swift,cinder]
+passenv = OS_TEST_*
 commands = ostestr --slowest {posargs}
 
 [testenv:docs]
@@ -30,8 +31,8 @@ commands = python setup.py testr --coverage --testr-args='^(?!.*test.*coverage).
 commands = {posargs}
 
 [testenv:functional]
+sitepackages = True
 setenv = OS_TEST_PATH=./glance_store/tests/functional
-passenv = OS_*
 commands = python setup.py testr --slowest --testr-args='glance_store.tests.functional'
 
 [flake8]