Updated from OpenStack Ansible Tests

Change-Id: Id33c46798385b72c967e3a5118e252d43c5f1296

.gitignore

@@ -1,7 +1,3 @@
-# Override Files #
-rpc_deployment/playbooks/lab_plays
-rpc_deployment/vars/overrides/*.yml
-
 # Compiled source #
 ###################
 *.com
@@ -33,6 +29,7 @@ doc/build/
 *.log
 *.sql
 *.sqlite
+logs/*
 
 # OS generated files #
 ######################
@@ -62,3 +59,13 @@ ChangeLog
 
 # Files created by releasenotes build
 releasenotes/build
+
+# Test temp files
+tests/common
+tests/*.retry
+
+# Vagrant artifacts
+.vagrant
+
+# Git clones
+openstack-ansible-ops

bindep.txt

@@ -3,24 +3,54 @@
 #
 # See the following for details:
 #  - http://docs.openstack.org/infra/bindep/
-#  - https://git.openstack.org/openstack-infra/bindep
+#  - https://git.openstack.org/cgit/openstack-infra/bindep
 #
 # Even if the role does not make use of this facility, it
 # is better to have this file empty, otherwise OpenStack-CI
 # will fall back to installing its default packages which
 # will potentially be detrimental to the tests executed.
+#
+# Note:
+# This file is maintained in the openstack-ansible-tests repository.
+# https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/bindep.txt
+# If you need to remove or add extra dependencies, you should modify
+# the central file instead and once your change is accepted then update
+# this file as well. The purpose of this file is to ensure that Python and
+# Ansible have all their necessary binary requirements on the test host before
+# tox executes. Any binary requirements needed by services/roles should be
+# installed by those roles in their applicable package install tasks, not through
+# using this file.
+#
 
-# OpenStack-CI's Jenkins needs curl
+# Base requirements for Ubuntu
-# TODO(odyssey4me) remove this once https://review.openstack.org/288634 has merged
+build-essential   [platform:dpkg]
-# and the disk images are rebuilt and redeployed.
+git-core          [platform:dpkg]
-curl
-wget
-
-# Requirements for Paramiko 2.0
 libssl-dev        [platform:dpkg]
 libffi-dev        [platform:dpkg]
+python2.7         [platform:dpkg]
+python-dev        [platform:dpkg]
+
+# Base requirements for RPM distros
+gcc               [platform:rpm]
+gcc-c++           [platform:rpm]
+git               [platform:rpm]
 libffi-devel      [platform:rpm]
 openssl-devel     [platform:rpm]
+python-devel      [platform:rpm]
 
-# For selinux
+# For SELinux
-libselinux-python [platform:rpm]
+libselinux-python [platform:redhat]
+libsemanage-python [platform:redhat]
+
+# For SSL SNI support
+python-pyasn1               [platform:dpkg platform:suse]
+python-openssl              [platform:dpkg platform:suse]
+python-ndg-httpsclient      [platform:ubuntu]
+python2-pyasn1              [platform:redhat]
+python2-pyOpenSSL           [platform:redhat]
+python-pyOpenSSL            [platform:suse]
+python-ndg_httpsclient      [platform:redhat]
+python-ndg-httpsclient      [platform:suse]
+
+# Required for compressing collected log files in CI
+gzip

run_tests.sh

@@ -12,37 +12,58 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# Note:
+# This file is maintained in the openstack-ansible-tests repository.
+# https://git.openstack.org/cgit/openstack/openstack-ansible-tests/tree/run_tests.sh
+# If you need to modify this file, update the one in the openstack-ansible-tests
+# repository and then update this file as well. The purpose of this file is to
+# prepare the host and then execute all the tox tests.
+#
 
-set -euov
+## Shell Opts ----------------------------------------------------------------
+set -xeu
 
-FUNCTIONAL_TEST=${FUNCTIONAL_TEST:-true}
+## Vars ----------------------------------------------------------------------
 
-# Prepare Ubuntu 14.04 and 16.04 hosts
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
-if [ "$(which apt-get)" ]; then
-  apt-get install -y build-essential python2.7 python-dev git-core libssl-dev libffi-dev
-fi
 
-# Prepare CentOS and Red Hat Enterprise Linux 7 hosts
+## Main ----------------------------------------------------------------------
-if [ "$(which yum)" ]; then
-  yum -y install libffi-devel openssl-devel git python-devel "@Development Tools"
-fi
 
-# Download and install pip
+source /etc/os-release || source /usr/lib/os-release
-if [ ! "$(which pip)" ]; then
-  curl --silent --show-error --retry 5 \
-    https://bootstrap.pypa.io/get-pip.py | sudo python2.7
-fi
 
-# install tox
+install_pkg_deps() {
-pip install tox
+    pkg_deps="git"
 
-# run through each tox env and execute the test
+    case ${ID,,} in
-for tox_env in $(awk -F= '/envlist/ {print $2}' tox.ini | sed 's/,/ /g'); do
+        *suse*) pkg_mgr_cmd="zypper -n in" ;;
-  if [ "${tox_env}" == "ansible-functional" ]; then
+        centos|rhel) pkg_mgr_cmd="yum install -y" ;;
-    if ${FUNCTIONAL_TEST}; then
+        fedora) pkg_mgr_cmd="dnf -y install" ;;
-      tox -e ${tox_env}
+        ubuntu|debian) pkg_mgr_cmd="apt-get install -y" ;;
+        *) echo "unsupported distribution: ${ID,,}"; exit 1 ;;
+    esac
+
+    eval sudo $pkg_mgr_cmd $pkg_deps
+}
+
+git_clone_repo() {
+    if [[ ! -d tests/common ]]; then
+        # The tests repo doesn't need a clone, we can just
+        # symlink it.
+        if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
+            ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common
+        else
+            git clone \
+                https://git.openstack.org/openstack/openstack-ansible-tests \
+                tests/common
+        fi
     fi
-  else
+}
-    tox -e ${tox_env}
+
-  fi
+install_pkg_deps
-done
+
+git_clone_repo
+
+# start executing the main test script
+source tests/common/run_tests_common.sh
+

tests/tests-repo-clone.sh

@@ -34,28 +34,39 @@ export TESTING_HOME=${TESTING_HOME:-$HOME}
 export WORKING_DIR=${WORKING_DIR:-$(pwd)}
 export CLONE_UPGRADE_TESTS=${CLONE_UPGRADE_TESTS:-no}
 
-## Main ----------------------------------------------------------------------
+## Functions -----------------------------------------------------------------
+
+function create_tests_clonemap {
 
 # Prepare the clonemap for zuul-cloner to use
-# This is placed here instead of inside the conditional
-# to prevent indentation problems.
 cat > ${TESTING_HOME}/tests-clonemap.yaml << EOF
 clonemap:
   - name: openstack/openstack-ansible-tests
     dest: ${WORKING_DIR}/tests/common
 EOF
 
+}
+
+## Main ----------------------------------------------------------------------
+
 # If zuul-cloner is present, use it so that we
 # also include any dependent patches from the
 # tests repo noted in the commit message.
 if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
 
+    # Prepare the clonemap for zuul-cloner to use
+    create_tests_clonemap
+
+    # Execute the clone
     /usr/zuul-env/bin/zuul-cloner \
         --cache-dir /opt/git \
         --map ${TESTING_HOME}/tests-clonemap.yaml \
         git://git.openstack.org \
         openstack/openstack-ansible-tests
 
+    # Clean up the clonemap.
+    rm -f ${TESTING_HOME}/tests-clonemap.yaml
+
 # Alternatively, use a simple git-clone. We do
 # not re-clone if the directory exists already
 # to prevent overwriting any local changes which
@@ -73,9 +84,6 @@ elif [[ ! -d tests/common ]]; then
     fi
 fi
 
-# Clean up the clonemap.
-rm -f ${TESTING_HOME}/tests-clonemap.yaml
-
 # If this test set includes an upgrade test, the
 # previous stable release tests repo must also be
 # cloned.