Add functional tests to tox

This adds functional tests to tox, along with a number of prerequisite
steps. Since this is primarily a bash project, pbr usage has been avoided.
Tests are to be written in python and put in the tests directory.

Running tox -e setupenv will execute a setup script
(tests/setup_docker.sh) to ensure the running instance of docker meets
the minimum version requirement.

Running tox -e images will execute the image building script
(tools/build-all-docker-images) and will parse the resulting output to report
failures.

Running tox -e startenv will generate the environment file, run
"tools/kolla start", and run first time initialization (eventually).

Running tox -e functional is for actually testing the deployed OpenStack
environment via a series of tests utilizing the client APIs.

Change-Id: Iff6dfdca43f0c44d471e7540a7836e56a0de4507

.testr.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=python -m subunit.run discover tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

requirements.txt

@@ -0,0 +1 @@
+-e git://github.com/docker/compose#egg=docker-compose

test-requirements.txt

@@ -1 +1,13 @@
 PyYAML
+python-barbicanclient>=3.0.1
+python-ceilometerclient>=1.0.6
+python-cinderclient>=1.1.0
+python-glanceclient>=0.15.0
+python-heatclient>=0.3.0
+python-keystoneclient>=1.1.0
+python-neutronclient>=2.3.11,<3
+python-novaclient>=2.18.0,!=2.21.0
+python-swiftclient>=2.2.0
+testrepository>=0.0.18
+testscenarios>=0.4
+testtools>=0.9.36,!=1.2.0

tests/clients.py

@@ -0,0 +1,68 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+import logging
+from keystoneclient.v2_0 import client as ksclient
+
+logging.basicConfig(level=logging.WARNING)
+LOG = logging.getLogger(__name__)
+
+class OpenStackClients(object):
+
+    def __init__(self):
+        self._connected_clients = {}
+        self._supported_clients = self.__class__.__subclasses__()
+        self.client = None
+
+    def get_client(self, name):
+        if name in self._connected_clients:
+            return self._connected_clients[name]
+        try:
+            aclass = next(s for s in self._supported_clients if name in
+                          s.__name__)
+            sclient = aclass()
+            connected_client = sclient.create()
+            self._connected_clients[name] = connected_client
+            return connected_client
+
+        except StopIteration:
+            LOG.warn("Requested client %s not found", name)
+            raise
+
+    def create(self):
+        pass
+
+
+class KeystoneClient(OpenStackClients):
+
+    def __init__(self):
+        super(KeystoneClient, self).__init__()
+        # TODO: this shouldn't be hard coded
+        self.creds = {'auth_url': 'http://10.0.0.4:5000/v2.0',
+                      'username': 'admin',
+                      'password': 'steakfordinner',
+                      'tenant_name': 'admin'}
+
+    def create(self):
+        if self.client is None:
+            self.client = ksclient.Client(**self.creds)
+        return self.client
+
+
+if __name__ == '__main__':
+    # TODO: mox this
+    client_mgr = OpenStackClients()
+    ks = client_mgr.get_client('KeystoneClient')
+    LOG.info(ks)
+    ks2 = client_mgr.get_client('KeystoneClient')
+    LOG.info(ks2)

tests/setup_docker.sh

@@ -50,12 +50,15 @@ function start_docker() {
 
 function create_group() {
     getent group docker
-    if [ $? -eq 2 ]; then # 2: key could not be found in database
+    result=$?
+    if [ $result -eq 0 ]; then # 0: key already exists, nothing to do
+        return
+    elif [ $result -eq 2 ]; then # 2: key could not be found in database
         groupadd docker
         chown root:docker /var/run/docker.sock
         usermod -a -G docker ${SUDO_USER:-$USER}
     else
-        echo Unexpected failure: $?
+        echo Unexpected failure: $result
         exit
     fi
 }

tests/test_images.py

@@ -0,0 +1,42 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import testtools
+from subprocess import check_output
+
+class ImagesTest(testtools.TestCase):
+    def setUp(self):
+        super(ImagesTest, self).setUp()
+
+    def test_builds(self):
+        build_output = check_output(["tools/build-all-docker-images",
+                                    "--release",
+                                    "--pull",
+                                    "--testmode"])
+
+        # these are images that are known to not build properly
+        excluded_images = ["kollaglue/centos-rdo-swift-proxy-server",
+                           "kollaglue/centos-rdo-swift-container",
+                           "kollaglue/centos-rdo-swift-base",
+                           "kollaglue/centos-rdo-swift-account",
+                           "kollaglue/centos-rdo-swift-object",
+                           "kollaglue/centos-rdo-barbican",
+                           "kollaglue/fedora-rdo-base",
+                           "kollaglue/centos-rdo-rhel-osp-base"]
+
+        results = eval(build_output.splitlines()[-1])
+
+        for image, result in results.iteritems():
+            if image in excluded_images:
+                self.assertEqual(result, 'fail')
+            else:
+                self.assertNotEqual(result, 'fail')

tests/test_keystone.py

@@ -0,0 +1,25 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import testtools
+from clients import OpenStackClients
+
+
+class KeystoneTest(testtools.TestCase):
+    def setUp(self):
+        super(KeystoneTest, self).setUp()
+        self.kc = OpenStackClients().get_client('KeystoneClient')
+
+    def test_tenants(self):
+        result = self.kc.tenants.list()
+        # only admin tenant
+        self.assertEqual(1, len(result))

tools/build-all-docker-images

@@ -112,6 +112,14 @@ function print_summary {
     done
 }
 
+function print_parseable_summary {
+    printf "{"
+    for image in "${!status[@]}"; do
+        printf "'$image':'${status[$image]}',"
+    done
+    printf "}"
+}
+
 function interrupted {
     info "Interrupted..."
     print_summary
@@ -137,7 +145,7 @@ trap 'interrupted' INT
 
 
 ARGS=$@
-PARSED_ARGS=$(getopt -q -o hr:n: -l help,namespace:,private-registry:,from:,to: -- "$@")
+PARSED_ARGS=$(getopt -q -o hr:n: -l help,namespace:,private-registry:,from:,to:,testmode -- "$@")
 
 eval set -- "$PARSED_ARGS"
 
@@ -170,6 +178,11 @@ while :; do
                 ARGS=${ARGS/\-\-to*$TO/}
                 ;;
 
+    (--testmode)
+                TESTMODE=1
+                ARGS=${ARGS/\-\-testmode/}
+                ;;
+
     (--)        break
                 ;;
 
@@ -189,4 +202,5 @@ for image in "${!img_dirs[@]}"; do
 done
 
 print_summary
+[ -n "$TESTMODE" ] && print_parseable_summary
 rm -rf $WORKDIR

tox.ini

@@ -1,13 +1,52 @@
 [tox]
-skipsdist = True
-envlist = pep8
 minversion = 1.6
+skipsdist = True
+envlist = functional
 
 [testenv]
-deps = -r{toxinidir}/test-requirements.txt
+install_command = pip install {opts} {packages}
 
 [testenv:pep8]
+deps = PyYAML
 commands =
     {toxinidir}/tools/validate-all-json.sh
     {toxinidir}/tools/validate-all-yaml.sh
     {toxinidir}/tools/validate-all-maintainer.sh
+
+[testenv:bashate]
+deps = bashate
+whitelist_externals = bash
+# tox improperly interprets # and {1} in regex, so match on [[:punct:]]+
+commands =
+    bash -c "files=`egrep -rlI '^[[:punct:]]+!/(bin/|/usr/bin/env )(ba)?sh' .` && bashate $files"
+
+[testenv:setupenv]
+whitelist_externals = bash
+commands = bash -c tests/setup_docker.sh
+
+[testenv:images]
+deps = -r{toxinidir}/test-requirements.txt
+whitelist_externals = find
+                      bash
+commands =
+   find . -type f -name "*.pyc" -delete
+   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
+   testr run ^(test_images).*
+
+[testenv:startenv]
+whitelist_externals = bash
+commands =
+    bash -c tools/genenv
+    sudo tools/kolla start
+# this can be improved after https://review.openstack.org/#/c/180729/
+# tools/test-deploy
+
+[testenv:functional]
+deps = -r{toxinidir}/requirements.txt
+       -r{toxinidir}/test-requirements.txt
+whitelist_externals = find
+                      bash
+commands =
+   find . -type f -name "*.pyc" -delete
+   bash -c "if [ ! -d .testrepository ]; then testr init; fi"
+   testr run ^(?!test_images).*