enable an xfs tmpdir for swift tests

Builder added:
 - mount-xfs-tmp-space: takes a path, creates a 1G file there,
   and loopback-mounts it to a passed in path
 - tox-with-tmpdir-passed-in: a copy of the tox job, but with an
   extra {new_tmpdir_loc} value passed in and TMPDIR set

projects.yaml changes (for swift):
 - replaced 'gate-{name}-tox-{envlist}-{node}' for
   'gate-{name}-tox-xfs-tmp-{envlist}-{node}' and placed
   'py27' and 'py34' envlist entries under it

scripts updated:
 - run-tox.sh: now only sets TMPDIR if it's currently unset

job-template updates:
 - create 'gate-{name}-tox-xfs-tmp-{envlist}-{node}' that is a copy
   of 'gate-{name}-tox-{envlist}' but calls 'mount-xfs-tmp-space'
   before dropping sudo. Then it calls 'tox-with-tmpdir-passed-in'
   instead of 'tox'

zuul/layouy.yaml updates:
 - added the "with xfs" tox jobs to swift

Change-Id: I3339de0d57726b339dfc6638d225e49d097f7b63

jenkins/jobs/macros.yaml

@@ -776,6 +776,14 @@
     builders:
       - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {envlist}"
 
+- builder:
+    name: tox-with-tmpdir-passed-in
+    builders:
+      - shell: |
+          #!/bin/bash
+          export TMPDIR={new_tmpdir_loc}
+          exit /usr/local/jenkins/slave_scripts/run-tox.sh {envlist}"
+
 - builder:
     name: assert-no-extra-files
     builders:
@@ -1069,6 +1077,20 @@
           #Make 'elasticsearch' binary callable from within functional tests
           sudo ln -s /usr/share/elasticsearch/bin/elasticsearch /usr/local/bin/elasticsearch
 
+- builder:
+    name: mount-xfs-tmp-space
+    builders:
+      - shell: |
+          #!/bin/bash -xe
+          # create a large-ish file that we'll mount as a loopback
+          truncate -s 1GB {xfs_file}
+          # format the new file as xfs
+          mkfs.xfs {xfs_file}
+          # loopback mount the file
+          mkdir -p {xfs_mount_point}
+          sudo mount -o loop,noatime,nodiratime {xfs_file} {xfs_mount_point}
+          sudo chmod 777 {xfs_mount_point}
+
 # ======================================================================
 
 - wrapper:

jenkins/jobs/projects.yaml

@@ -14215,18 +14215,16 @@
           service: object-storage
       - install-guide-jobs:
           service: object-storage
-      - 'gate-{name}-tox-{envlist}-{node}':
+      - 'gate-{name}-tox-xfs-tmp-{envlist}-{node}':
           envlist:
+            - py27
+            - py34
             - bandit
             - func
             - func-in-process-fast-post
-          node:
-            - ubuntu-trusty
-            - ubuntu-xenial
-      - 'gate-{name}-tox-{envlist}-{node}':
-          envlist:
             - func-in-process-encryption
           node:
+            - ubuntu-trusty
             - ubuntu-xenial
       - '{pipeline}-swift-dsvm-functional{special}-{node}{job-suffix}':
           pipeline: gate

jenkins/jobs/python-jobs.yaml

@@ -506,6 +506,34 @@
       - test-results
       - console-log
 
+- job-template:
+    name: 'gate-{name}-tox-xfs-tmp-{envlist}-{node}'
+    description: 'Same job as gate-{name}-tox-{envlist}, but with TMPDIR set to an xfs mount point.'
+    node: '{node}'
+
+    wrappers:
+      - build-timeout:
+          timeout: 40
+      - timestamps
+
+    builders:
+      - print-template-name:
+          template-name: "{template-name}"
+      - zuul-git-prep-upper-constraints
+      - install-distro-packages
+      - mount-xfs-tmp-space:
+          xfs_mount_point: "$HOME/xfstmp"
+          xfs_file: "$HOME/1G_xfs_file"
+      - revoke-sudo
+      - tox-with-tmpdir-passed-in:
+          new_tmpdir_loc: "$HOME/xfstmp"
+          envlist: '{envlist}'
+      - assert-no-extra-files
+
+    publishers:
+      - test-results
+      - console-log
+
 - job-template:
     name: 'gate-{name}-tox-db-{envlist}-{node}'
     description: 'Same job as gate-{name}-tox-db-{envlist} but on node {node}'

jenkins/scripts/run-tox.sh

@@ -113,7 +113,9 @@ export PYTHON=$bin_path/python
 export NOSE_WITH_XUNIT=1
 export NOSE_WITH_HTML_OUTPUT=1
 export NOSE_HTML_OUT_FILE='nose_results.html'
-export TMPDIR=$(/bin/mktemp -d)
+if [[ -z "$TMPDIR" ]]; then
+    export TMPDIR=$(/bin/mktemp -d)
+fi
 export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt
 trap "rm -rf $TMPDIR" EXIT
 

zuul/layout.yaml

@@ -3181,7 +3181,7 @@ jobs:
   - name: gate-keystone-python34-db
     branch: ^(?!stable/liberty).*$
 
-  - name: ^gate-swift-tox-bandit-ubuntu-(trusty|xenial)$
+  - name: ^gate-swift-tox-xfs-tmp-bandit-ubuntu-(trusty|xenial)$
     voting: false
 
   - name: gate-swift-python34
@@ -16832,11 +16832,15 @@ projects:
       - name: api-ref-jobs
       - name: install-guide-jobs
     check:
-      - gate-swift-tox-func-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-py27-ubuntu-trusty
-      - gate-swift-tox-func-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-py34-ubuntu-trusty
-      - gate-swift-tox-func-in-process-fast-post-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-py27-ubuntu-xenial
-      - gate-swift-tox-func-in-process-fast-post-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-py34-ubuntu-xenial
-      - gate-swift-tox-func-in-process-encryption-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-func-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-in-process-encryption-ubuntu-xenial
       - gate-tempest-dsvm-full-ubuntu-trusty
       - gate-tempest-dsvm-full-ubuntu-xenial
       - gate-tempest-dsvm-neutron-identity-v3-only-full-ubuntu-xenial-nv
@@ -16844,15 +16848,19 @@ projects:
       - gate-grenade-dsvm-ubuntu-xenial
       - gate-swift-dsvm-functional-ubuntu-trusty
       - gate-swift-dsvm-functional-ubuntu-xenial
-      - gate-swift-tox-bandit-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-bandit-ubuntu-trusty
-      - gate-swift-tox-bandit-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-bandit-ubuntu-xenial
       - gate-swift-dsvm-functional-identity-v3-only-ubuntu-xenial-nv
     gate:
-      - gate-swift-tox-func-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-py27-ubuntu-trusty
-      - gate-swift-tox-func-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-py34-ubuntu-trusty
-      - gate-swift-tox-func-in-process-fast-post-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-py27-ubuntu-xenial
-      - gate-swift-tox-func-in-process-fast-post-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-py34-ubuntu-xenial
-      - gate-swift-tox-func-in-process-encryption-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-func-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-trusty
+      - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-xenial
+      - gate-swift-tox-xfs-tmp-func-in-process-encryption-ubuntu-xenial
       - gate-tempest-dsvm-full-ubuntu-trusty
       - gate-tempest-dsvm-full-ubuntu-xenial
       - gate-grenade-dsvm-ubuntu-trusty