Port in tox jobs from openstack-zuul-jobs

The original idea was that we'd have a "python27" job. But it turns out
that "python27" isn't actually as universal as you might think.
"tox-py27" on the other hand is a thing that can and should always work
for people who want to run tox with the py27 env.

Port in the jobs we had in openstack-zuul-jobs.

Also, copy in the text of the old shell scripts so that we can iterate on
them and make them better piecemeal.

These are named zuul-tox not tox because we already have tox jobs
defined elsewhere. We'll need to do the naming dance while we work on
them.

Change-Id: I813f3f2ae138c07918556bc81655518023527131

.zuul.yaml

@@ -6,34 +6,110 @@
 
       This performs basic host and general project setup tasks common
       to all types of unit test jobs.
-    pre-run: unittests/pre
+    roles:
+      - zuul: openstack-infra/zuul-jobs
+    pre-run: playbooks/unittests/pre
 
 - job:
-    name: unittests-python
+    name: zuul-tox
     parent: unittests
     description: |
-      Perform setup common to python unit test jobs.
+      Base job containing setup and teardown for tox-based test jobs.
 
       This performs basic host and general project setup tasks common
-      to all python unit test jobs.
+      to all tox unit test jobs.
-    run: unittests/python
+
+      Responds to two variables:
+
+      tox_environment
+        the environment to pass on the command line
+
+      tox_command_line
+        an optional command line
+    run: playbooks/tox/run
+    pre-run: playbooks/tox/pre
+    post-run: playbooks/tox/post
 
 - job:
-    name: python27
+    name: zuul-tox-py27
-    parent: unittests-python
+    parent: zuul-tox
     description: |
       Run unit tests for a Python project under cPython version 2.7.
 
       Uses tox with the "py27" environment.
     vars:
-      zuul_python_version: 27
+      tox_environment: py27
 
 - job:
-    name: python35
+    name: zuul-tox-py34
-    parent: unittests-python
+    parent: zuul-tox
+    description: |
+      Run unit tests for a Python project under cPython version 3.4.
+
+      Uses tox with the "py34" environment.
+    vars:
+      tox_environment: py34
+
+- job:
+    name: zuul-tox-py35
+    parent: zuul-tox
     description: |
       Run unit tests for a Python project under cPython version 3.5.
 
       Uses tox with the "py35" environment.
     vars:
-      zuul_python_version: 35
+      tox_environment: py35
+
+- job:
+    name: zuul-tox-docs
+    parent: zuul-tox
+    # NOTE: This is not the OpenStack docs job, this is a job to run a tox
+    #       environment called "docs". OpenStack will want a different job
+    #       in openstack-infra/zuul-jobs
+    description: |
+      Run documentation unit tests.
+
+      Uses tox with the "docs" environment.
+    vars:
+      tox_environment: docs
+    post-run: playbooks/tox/docs-post
+
+- job:
+    name: zuul-tox-linters
+    parent: zuul-tox
+    description: |
+      Runs code linting tests.
+
+      Uses tox with the "linters" environment.
+    run: playbooks/tox/linters
+
+- job:
+    name: zuul-tox-cover
+    parent: zuul-tox
+    voting: false
+    description: |
+      Run code coverage tests.
+
+      Uses tox with the "cover" environment.
+    vars:
+      tox_environment: cover
+
+- job:
+    name: zuul-tox-tarball
+    parent: zuul-tox
+    voting: false
+    description: |
+      Generate a python source tarball and a binary wheel
+
+      Uses tox with the "venv" environment.
+    vars:
+      tox_environment: venv
+    run: playbooks/tox/tarball
+    post-run: playbooks/tox/tarball-post
+
+- project:
+    name: openstack-infra/zuul-jobs
+    check:
+      jobs:
+        - zuul-tox-linters
+        - tox-linters

bindep.txt

@@ -1,4 +1,12 @@
 # This is a cross-platform list tracking distribution packages needed by tests;
 # see http://docs.openstack.org/infra/bindep/ for additional information.
 
-# Nothing here yet.
+# Need to build cryptography for installing ansible-lint
+build-essential [platform:dpkg]
+gcc [platform:rpm]
+libssl-dev [platform:dpkg]
+openssl-devel [platform:rpm]
+libffi-dev [platform:dpkg]
+libffi-devel [platform:rpm]
+python-dev [platform:dpkg]
+python-devel [platform:rpm]

playbooks/tox/docs-post.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - fetch-sphinx-output

playbooks/tox/linters.yaml

@@ -0,0 +1,14 @@
+- hosts: all
+  pre_tasks:
+    - shell: tox -l
+      args:
+        chdir: "src/{{ zuul.project.canonical_name }}"
+      register: envlist
+    - set_fact:
+        tox_environment: 'pep8'
+      when: envlist.stdout.find('pep8') != -1
+    - set_fact:
+        tox_environment: 'linters'
+      when: envlist.stdout.find('linters') != -1
+  roles:
+    - tox

playbooks/tox/post.yaml

@@ -0,0 +1,22 @@
+- hosts: all
+  tasks:
+    - name: Find tox directories to synchrionize
+      find:
+        file_type: directory
+        paths: "src/{{ zuul.project.canonical_name }}/.tox"
+        # NOTE(pabelanger): The .tox/log folder is empty, ignore it.
+        patterns: ^(?!log).*$
+        use_regex: yes
+      register: result
+
+    - name: Ensure local tox dir
+      file:
+        path: "{{ zuul.executor.log_root }}/tox"
+        state: directory
+
+    - name: Collect tox logs
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}/tox"
+        mode: pull
+        src: "{{ item.path }}/log/"
+      with_items: "{{ result.files }}"

playbooks/tox/pre.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  tasks:
+    - name: Ensure tox is installed
+      shell: type tox || pip install --user tox

playbooks/tox/run.yaml

@@ -0,0 +1,3 @@
+- hosts: all
+  roles:
+    - tox

playbooks/tox/tarball-post.yaml

@@ -0,0 +1,10 @@
+- hosts: all
+  tasks:
+    - name: Collect tarball artifacts.
+      synchronize:
+        dest: "{{ zuul.executor.src_root }}/tarballs"
+        mode: pull
+        src: "src/{{ zuul.project.canonical_name }}/dist/{{ item }}"
+      with_items:
+        - "*.tar.gz"
+        - "*.whl"

playbooks/tox/tarball.yaml

@@ -0,0 +1,4 @@
+- hosts: all
+  roles:
+    - run-tarball
+    - run-wheel

playbooks/unittests/post.yaml

@@ -0,0 +1,12 @@
+- hosts: all
+  tasks::
+    - name: Collect test-results
+      synchronize:
+        dest: "{{ zuul.executor.log_root }}"
+        mode: pull
+        rsync_opts:
+          - "--ignore-missing-args"
+        src: "src/{{ zuul.project.canonical_name }}/{{ item }}"
+      with_items:
+        - "*testr_results.html.gz"
+        - "*testrepository.subunit.gz"

playbooks/unittests/pre.yaml

@@ -1,5 +1,11 @@
-hosts: all
+- hosts: all
-roles:
+  # We're gathering facts here so that we can emit things in the debug-ansible
-  - extra-test-setup
+  # role. Don't copy-pasta this. Zuul runs ansible with gathering = explicit
-  - run-bindep
+  gather_facts: true
-  - revoke-sudo
+  gather_subset:
+    - "!all"
+  roles:
+    - debug-ansible
+    - bindep
+    - test-setup
+    - zuul-revoke-sudo

playbooks/unittests/python.yaml

@@ -1,2 +0,0 @@
-hosts: all
-roles: []

roles/bindep/README.rst

@@ -0,0 +1,6 @@
+Installs distro packages using bindep tool
+
+Looks for a ``bindep.txt`` in a project's source directory, or failing
+that a ``other-requirements.txt``. If one exists, run ``bindep`` on the
+file to produce a list of required distro packages that do not exist and
+then install the missing packages.

roles/bindep/tasks/main.yaml

@@ -0,0 +1,68 @@
+---
+- name: Install distro packages from bindep
+  args:
+    chdir: "src/{{ zuul.project.canonical_name }}"
+    executable: /bin/bash
+  shell: |
+    # set a default path to the preinstalled bindep entrypoint
+    export BINDEP=${BINDEP:-/usr/bindep-env/bin/bindep}
+
+    function is_fedora {
+        [ -f /usr/bin/yum ] && cat /etc/*release | grep -q -e "Fedora"
+    }
+
+    YUM=yum
+    if is_fedora; then
+        YUM=dnf
+    fi
+
+    # figure out which bindep list to use
+    if [ -n "$PACKAGES" ] ; then
+        # already set in the calling environment
+        :
+    elif [ -e bindep.txt ] ; then
+        # project has its own bindep list
+        export PACKAGES=bindep.txt
+    elif [ -e other-requirements.txt ] ; then
+        # project has its own bindep list
+        export PACKAGES=other-requirements.txt
+    else
+        # use the bindep fallback list preinstalled on the worker
+        export PACKAGES=/usr/local/jenkins/common_data/bindep-fallback.txt
+    fi
+
+    # an install loop, retrying to check that all requested packages are
+    # obtained
+    try=0
+    # Install test profile using bindep
+    until $BINDEP -b -f $PACKAGES test; do
+        if [ $try -gt 2 ] ; then
+            set +x
+            echo -e "\nERROR: These requested packages were not installed:\n" \
+                "\n`$BINDEP -b -f $PACKAGES test`\n" 1>&2
+            set -x
+            exit 1
+        fi
+
+        # do not abort inside the loop, we check for the desired outcome
+        set +e
+        if apt-get -v >/dev/null 2>&1 ; then
+            sudo apt-get -qq update
+            sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \
+                apt-get -q --option "Dpkg::Options::=--force-confold" \
+                --assume-yes install `$BINDEP -b -f $PACKAGES test`
+        elif emerge --version >/dev/null 2>&1 ; then
+            sudo emerge -uDNq --jobs=4 @world
+            sudo PATH=/usr/sbin:/sbin:$PATH emerge -q --jobs=4 \
+                `$BINDEP -b -f $PACKAGES test`
+        elif zypper --version >/dev/null 2>&1 ; then
+            sudo PATH=/usr/sbin:/sbin:$PATH zypper --non-interactive install \
+                `$BINDEP -b -f $PACKAGES test`
+        else
+            sudo PATH=/usr/sbin:/sbin:$PATH $YUM install -y \
+                `$BINDEP -b -f $PACKAGES test`
+        fi
+        set -e
+
+        try=$(( $try+1 ))
+    done

roles/debug-ansible/README.rst

@@ -0,0 +1,3 @@
+Output all of the Ansible variables for the host
+
+This is unsafe to run in Trusted jobs as it will write any secrets to the log.

roles/debug-ansible/tasks/main.yaml

@@ -0,0 +1,5 @@
+- name: Write out all ansible variables/facts known for the host
+  delegate_to: localhost
+  template:
+    dest: "{{ zuul.executor.log_root }}/ansible-hostvars.{{ inventory_hostname }}.yaml"
+    src: templates/ansible-hostvars.j2

roles/debug-ansible/templates/ansible-hostvars.j2

@@ -0,0 +1 @@
+{{ hostvars[inventory_hostname] | to_nice_yaml(indent=2) }}

roles/fetch-sphinx-output/README.rst

@@ -0,0 +1 @@
+Collect output from a sphinx build

roles/fetch-sphinx-output/tasks/main.yaml

@@ -0,0 +1,6 @@
+- name: Collect sphinx build html
+  synchronize:
+    dest: "{{ zuul.executor.log_root }}"
+    mode: pull
+    src: "src/{{ zuul.project.canonical_name }}/doc/build/html"
+  no_log: true

roles/run-cover/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+tox_environment: cover

roles/run-cover/tasks/main.yaml

@@ -0,0 +1,30 @@
+- name: Run code coverage
+  args:
+    chdir: "src/{{ zuul.project.canonical_name }}"
+  shell: |
+    #!/bin/bash -xe
+
+    # Run coverage via tox. Also, run pbr freeze on the
+    # resulting environment at the end so that we have a record of exactly
+    # what packages we ended up testing.
+
+    export NOSE_COVER_HTML=1
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+
+    venv={{ tox_environment }}
+
+    # Workaround the combo of tox running setup.py outside of virtualenv
+    # and RHEL having an old distribute. The next line can be removed
+    # when either get fixed.
+    python setup.py --version
+
+    tox -e$venv
+    result=$?
+    [ -e .tox/$venv/bin/pbr ] && freezecmd=pbr || freezecmd=pip
+
+    echo "Begin $freezecmd freeze output from test virtualenv:"
+    echo "======================================================================"
+    .tox/${venv}/bin/${freezecmd} freeze
+    echo "======================================================================"
+
+    exit $result

roles/run-tarball/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+tox_environment: venv

roles/run-tarball/tasks/main.yaml

@@ -0,0 +1,37 @@
+- name: Build a tarball
+  args:
+    chdir: "src/{{ zuul.project.canonical_name }}"
+  shell: |
+    #!/bin/bash -xe
+
+    # Copyright 2013 OpenStack Foundation
+    #
+    # Licensed under the Apache License, Version 2.0 (the "License"); you may
+    # not use this file except in compliance with the License. You may obtain
+    # a copy of the License at
+    #
+    #      http://www.apache.org/licenses/LICENSE-2.0
+    #
+    # Unless required by applicable law or agreed to in writing, software
+    # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+    # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+    # License for the specific language governing permissions and limitations
+    # under the License.
+
+    venv={{ tox_environment }}
+
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+
+    rm -f dist/*.tar.gz
+    tox -e$venv python setup.py sdist
+
+    FILES=dist/*.tar.gz
+    for f in $FILES; do
+        echo "SHA1sum for $f:"
+        sha1sum $f | awk '{print $1}' > $f.sha1
+        cat $f.sha1
+
+        echo "MD5sum for $f:"
+        md5sum $f  | awk '{print $1}' > $f.md5
+        cat $f.md5
+    done

roles/run-wheel/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+tox_environment: venv

roles/run-wheel/tasks/main.yaml

@@ -0,0 +1,36 @@
+- name: Build a wheel
+  args:
+    chdir: "src/{{ zuul.project.canonical_name }}"
+  shell: |
+    #!/bin/bash -xe
+
+    # Copyright 2013 OpenStack Foundation
+    #
+    # Licensed under the Apache License, Version 2.0 (the "License"); you may
+    # not use this file except in compliance with the License. You may obtain
+    # a copy of the License at
+    #
+    #      http://www.apache.org/licenses/LICENSE-2.0
+    #
+    # Unless required by applicable law or agreed to in writing, software
+    # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+    # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+    # License for the specific language governing permissions and limitations
+    # under the License.
+
+    venv={{ tox_environment }}
+
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+
+    rm -f dist/*.whl
+    tox -e$venv pip install wheel
+    tox -e$venv python setup.py bdist_wheel
+
+    FILES=dist/*.whl
+    for f in $FILES; do
+        echo -n "SHA1sum for $f: "
+        sha1sum $f | awk '{print $1}' | tee $f.sha1
+
+        echo -n "MD5sum for $f: "
+        md5sum $f  | awk '{print $1}' | tee $f.md5
+    done

roles/tox/defaults/main.yaml

@@ -0,0 +1,2 @@
+---
+tox_environment: venv

roles/tox/tasks/main.yaml

@@ -0,0 +1,140 @@
+- name: Run tox
+  args:
+    chdir: "src/{{ zuul.project.canonical_name }}"
+    executable: /bin/bash
+  shell: |
+    # If a bundle file is present, call tox with the jenkins version of
+    # the test environment so it is used.  Otherwise, use the normal
+    # (non-bundle) test environment.  Also, run pbr freeze on the
+    # resulting environment at the end so that we have a record of exactly
+    # what packages we ended up testing.
+
+    venv={{ tox_environment }}
+
+    if [[ -z "$venv" ]]; then
+        echo "Usage: $?"
+        echo
+        echo "VENV: The tox environment to run (eg 'python27')"
+        exit 1
+    fi
+
+    function freeze_venv {
+        [ -e $bin_path/pbr ] && freezecmd=pbr || freezecmd=pip
+
+        echo "Begin $freezecmd freeze output from test virtualenv:"
+        echo "======================================================================"
+        ${bin_path}/${freezecmd} freeze | sort -f
+        echo "======================================================================"
+    }
+
+    function process_testr_artifacts {
+        if [ ! -d ".testrepository" ] ; then
+            return
+        fi
+
+        if [ -f ".testrepository/0.2" ] ; then
+            cp .testrepository/0.2 ./testrepository.subunit
+        elif [ -f ".testrepository/0" ] ; then
+            $bin_path/testr last --subunit > ./testrepository.subunit
+        fi
+        /usr/os-testr-env/bin/subunit2html ./testrepository.subunit testr_results.html
+        SUBUNIT_SIZE=$(du -k ./testrepository.subunit | awk '{print $1}')
+        gzip -9 ./testrepository.subunit
+        gzip -9 ./testr_results.html
+
+        if [[ "$SUBUNIT_SIZE" -gt 50000 ]]; then
+            echo
+            echo "testrepository.subunit was > 50 MB of uncompressed data!!!"
+            echo "Something is causing tests for this project to log significant amounts"
+            echo "of data. This may be writers to python logging, stdout, or stderr."
+            echo "Failing this test as a result"
+            echo
+            exit 1
+        fi
+
+        rancount=$($bin_path/testr last | sed -ne 's/Ran \([0-9]\+\).*tests in.*/\1/p')
+        if [ -z "$rancount" ] || [ "$rancount" -eq "0" ] ; then
+            echo
+            echo "Zero tests were run. At least one test should have been run."
+            echo "Failing this test as a result"
+            echo
+            exit 1
+        fi
+    }
+
+    function check_sudo_usage {
+        sudo $script_path/jenkins-sudo-grep.sh post
+        sudoresult=$?
+
+        if [ $sudoresult -ne "0" ]; then
+            echo
+            echo "This test has failed because it attempted to execute commands"
+            echo "with sudo.  See above for the exact commands used."
+            echo
+            exit 1
+        fi
+    }
+
+    function check_oom {
+        $script_path/jenkins-oom-grep.sh post
+        oomresult=$?
+
+        if [ $oomresult -ne "0" ]; then
+            echo
+            echo "This test has failed because it attempted to exceed configured"
+            echo "memory limits and was killed prior to completion.  See above"
+            echo "for related kernel messages."
+            echo
+            exit 1
+        fi
+    }
+
+    function check_nose_html {
+        htmlreport=$(find . -name $NOSE_HTML_OUT_FILE)
+        if [ -f "$htmlreport" ]; then
+            passcount=$(grep -c 'tr class=.passClass' $htmlreport)
+            if [ $passcount -eq "0" ]; then
+                echo
+                echo "Zero tests passed, which probably means there was an error"
+                echo "parsing one of the python files, or that some other failure"
+                echo "during test setup prevented a sane run."
+                echo
+                exit 1
+            fi
+        fi
+    }
+
+    script_path=/usr/local/jenkins/slave_scripts
+    bin_path=.tox/$venv/bin
+
+    export PYTHON=$bin_path/python
+    export NOSE_WITH_XUNIT=1
+    export NOSE_WITH_HTML_OUTPUT=1
+    export NOSE_HTML_OUT_FILE='nose_results.html'
+    if [[ -z "$TMPDIR" ]]; then
+        export TMPDIR=$(/bin/mktemp -d)
+    fi
+    export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
+    trap "rm -rf $TMPDIR" EXIT
+
+    cat /etc/image-hostname.txt
+
+    $script_path/jenkins-oom-grep.sh pre
+
+    sudo $script_path/jenkins-sudo-grep.sh pre
+
+    tox -vv -e$venv
+    result=$?
+
+    freeze_venv
+    process_testr_artifacts
+    check_sudo_usage
+    check_oom
+    check_nose_html
+
+    # Rename tox .log files to .log.txt so that Apache serves them as
+    # text/plain files since it can handle .txt special - and .log is
+    # unknown and therefore served as binary.
+    find .tox -type f -name "*.log" -exec mv {} {}.txt \;
+
+    exit $result