Implement testing using tempest

Rather than the previous basic functional tests, this
implements the use of tempest to exercise the API's
properly to validate that they're working properly.

Included is the normalisation of the way the upgrade
test is implemented to make it exactly in line with
the way the original API upgrade test was implemented
in the keystone repository. This keeps it consistent.

Change-Id: Id491d018beb30bce748240b50e9d66dcc1a249fa

tests/ansible-role-requirements.yml

@@ -46,6 +46,10 @@
   src: https://git.openstack.org/openstack/openstack-ansible-ceph_client
   scm: git
   version: master
+- name: os_tempest
+  src: https://git.openstack.org/openstack/openstack-ansible-os_tempest
+  scm: git
+  version: master
 - name: os_previous_glance
   src: https://git.openstack.org/openstack/openstack-ansible-os_glance
   scm: git

tests/inventory

@@ -24,6 +24,9 @@ memcached_all
 [keystone_all]
 openstack1
 
+[utility_all]
+openstack1
+
 [glance_api]
 openstack1
 

tests/os_glance-overrides.yml

@@ -1,5 +1,5 @@
 ---
-# Copyright 2017, Rackspace US, Inc.
+# Copyright 2016, Rackspace US, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,8 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- include: test-glance-resources-upgrade.yml
+tempest_run: yes
 
-- include: common/test-install-glance.yml
+tempest_test_whitelist:
+  - tempest.api.image
+
+tempest_image_api_v1_enabled: "{{ glance_enable_v1_api | default(False) }}"
+tempest_image_api_v2_enabled: "{{ glance_enable_v2_api | default(True) }}"
 
-- include: test-glance-resources-results.yml

tests/test-glance-functional.yml

@@ -1,62 +0,0 @@
----
-# Copyright 2015, Rackspace US, Inc.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-
-- name: Playbook for testing glance
-  hosts: glance_all
-  user: root
-  gather_facts: true
-  tasks:
-    # Packages need to be installed outside
-    # of venv to be usable by Ansible
-    - name: Install testing pip packages
-      pip:
-        name: "shade"
-
-    - name: Check the glance-api
-      uri:
-        url: "http://localhost:9292"
-        status_code: 300
-
-    - name: Check the glance-registry
-      uri:
-        url: "http://localhost:9191"
-        status_code: 401
-      when:
-        - (glance_enable_v1_api | default(False)) | bool or (glance_enable_v2_registry | default(False)) | bool
-
-    - name: Download the Cirros image
-      get_url:
-        url: "http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-uec.tar.gz"
-        dest: "/var/tmp/cirros.tar.gz"
-
-    - name: Upload the Cirros image
-      os_image:
-        cloud: default
-        endpoint_type: internal
-        validate_certs: "{{ keystone_service_internaluri_insecure | ternary(false, true) }}"
-        name: cirros
-        filename: "/var/tmp/cirros.tar.gz"
-        container_format: bare
-        disk_format: qcow2
-        is_public: True
-      register: cirros_image_create
-      until: cirros_image_create | success
-      retries: 5
-      delay: 15
-
-  vars_files:
-    - common/test-vars.yml
-

tests/test-glance-upgrades.sh

@@ -37,7 +37,7 @@ export WORKING_DIR=${WORKING_DIR:-$(pwd)}
 export ROLE_NAME=${ROLE_NAME:-''}
 
 export ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-"-vvv"}
-export TEST_PLAYBOOK=${TEST_PLAYBOOK:-$WORKING_DIR/tests/test-upgrade.yml}
+export TEST_PLAYBOOK=${TEST_PLAYBOOK:-$WORKING_DIR/tests/test-upgrade-pre.yml}
 export TEST_CHECK_MODE=${TEST_CHECK_MODE:-false}
 export TEST_IDEMPOTENCE=${TEST_IDEMPOTENCE:-false}
 
@@ -81,21 +81,15 @@ trap gate_job_exit_tasks EXIT
 
 # Prepare environment for the initial deploy of previous Glance
 # No upgrading or testing is done yet.
+export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-upgrade-pre.yml"
 export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-glance-install.log"
 
 # Execute the setup of previous Glance
 execute_ansible_playbook
 
-# Prepare environment for the upgrade of Glance
-export TEST_PLAYBOOK="${WORKING_DIR}/tests/benchmark-upgrade.yml"
+# Prepare the environment for the upgrade of Glance
+export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-upgrade-post.yml"
 export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-glance-upgrade.log"
 
-# Excute the upgrade of Glance
-execute_ansible_playbook
-
-# Prepare the environment for the testing of upgraded Glance
-export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-glance-functional.yml"
-export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-glance-upgrade-test.log"
-
-# Execute testing of upgraded Glance
+# Execute the upgrade of Glance including testing/benchmarking
 execute_ansible_playbook

tests/test-upgrade-post.yml

@@ -0,0 +1,29 @@
+---
+# Copyright 2017, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This playbook handles the installation of current Glance
+# Benchmarking, and tempest testing.
+
+# Install upgrade benchmarking
+- include: test-benchmark-glance-upgrade.yml
+
+# Install Glance
+- include: common/test-install-glance.yml
+
+# Install and execute Tempest
+- include: common/test-install-tempest.yml
+
+# Test upgrade benchmarking results
+- include: test-benchmark-glance-upgrade-results.yml

tests/test.yml

@@ -25,4 +25,5 @@
 # Install Glance
 - include: common/test-install-glance.yml
 
-- include: test-glance-functional.yml
+# Install and execute Tempest
+- include: common/test-install-tempest.yml

tox.ini

@@ -108,7 +108,6 @@ deps =
     {[testenv:ansible]deps}
 setenv =
     {[testenv]setenv}
-    TEST_PLAYBOOK={toxinidir}/tests/test-upgrade.yml
     CLONE_UPGRADE_TESTS=yes
 commands =
     bash -c "{toxinidir}/tests/tests-repo-clone.sh"