From f19f2fc381b8c0dc5f7c8402efa488007d8ed1c1 Mon Sep 17 00:00:00 2001 From: John Dickinson Date: Mon, 7 Nov 2016 11:43:39 -0800 Subject: [PATCH] enable an xfs tmpdir for swift tests Builder added: - mount-xfs-tmp-space: takes a path, creates a 1G file there, and loopback-mounts it to a passed in path - tox-with-tmpdir-passed-in: a copy of the tox job, but with an extra {new_tmpdir_loc} value passed in and TMPDIR set projects.yaml changes (for swift): - replaced 'gate-{name}-tox-{envlist}-{node}' for 'gate-{name}-tox-xfs-tmp-{envlist}-{node}' and placed 'py27' and 'py34' envlist entries under it scripts updated: - run-tox.sh: now only sets TMPDIR if it's currently unset job-template updates: - create 'gate-{name}-tox-xfs-tmp-{envlist}-{node}' that is a copy of 'gate-{name}-tox-{envlist}' but calls 'mount-xfs-tmp-space' before dropping sudo. Then it calls 'tox-with-tmpdir-passed-in' instead of 'tox' zuul/layouy.yaml updates: - added the "with xfs" tox jobs to swift Change-Id: I3339de0d57726b339dfc6638d225e49d097f7b63 --- jenkins/jobs/macros.yaml | 22 ++++++++++++++++++++++ jenkins/jobs/projects.yaml | 10 ++++------ jenkins/jobs/python-jobs.yaml | 28 ++++++++++++++++++++++++++++ jenkins/scripts/run-tox.sh | 4 +++- zuul/layout.yaml | 34 +++++++++++++++++++++------------- 5 files changed, 78 insertions(+), 20 deletions(-) diff --git a/jenkins/jobs/macros.yaml b/jenkins/jobs/macros.yaml index 419ba6dbe4..24a8fd81f3 100644 --- a/jenkins/jobs/macros.yaml +++ b/jenkins/jobs/macros.yaml @@ -776,6 +776,14 @@ builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {envlist}" +- builder: + name: tox-with-tmpdir-passed-in + builders: + - shell: | + #!/bin/bash + export TMPDIR={new_tmpdir_loc} + exit /usr/local/jenkins/slave_scripts/run-tox.sh {envlist}" + - builder: name: assert-no-extra-files builders: @@ -1069,6 +1077,20 @@ #Make 'elasticsearch' binary callable from within functional tests sudo ln -s /usr/share/elasticsearch/bin/elasticsearch /usr/local/bin/elasticsearch +- builder: + name: mount-xfs-tmp-space + builders: + - shell: | + #!/bin/bash -xe + # create a large-ish file that we'll mount as a loopback + truncate -s 1GB {xfs_file} + # format the new file as xfs + mkfs.xfs {xfs_file} + # loopback mount the file + mkdir -p {xfs_mount_point} + sudo mount -o loop,noatime,nodiratime {xfs_file} {xfs_mount_point} + sudo chmod 777 {xfs_mount_point} + # ====================================================================== - wrapper: diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index 78690430a9..76be4ccec5 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -14215,18 +14215,16 @@ service: object-storage - install-guide-jobs: service: object-storage - - 'gate-{name}-tox-{envlist}-{node}': + - 'gate-{name}-tox-xfs-tmp-{envlist}-{node}': envlist: + - py27 + - py34 - bandit - func - func-in-process-fast-post - node: - - ubuntu-trusty - - ubuntu-xenial - - 'gate-{name}-tox-{envlist}-{node}': - envlist: - func-in-process-encryption node: + - ubuntu-trusty - ubuntu-xenial - '{pipeline}-swift-dsvm-functional{special}-{node}{job-suffix}': pipeline: gate diff --git a/jenkins/jobs/python-jobs.yaml b/jenkins/jobs/python-jobs.yaml index a399f93bda..bfd98f965e 100644 --- a/jenkins/jobs/python-jobs.yaml +++ b/jenkins/jobs/python-jobs.yaml @@ -506,6 +506,34 @@ - test-results - console-log +- job-template: + name: 'gate-{name}-tox-xfs-tmp-{envlist}-{node}' + description: 'Same job as gate-{name}-tox-{envlist}, but with TMPDIR set to an xfs mount point.' + node: '{node}' + + wrappers: + - build-timeout: + timeout: 40 + - timestamps + + builders: + - print-template-name: + template-name: "{template-name}" + - zuul-git-prep-upper-constraints + - install-distro-packages + - mount-xfs-tmp-space: + xfs_mount_point: "$HOME/xfstmp" + xfs_file: "$HOME/1G_xfs_file" + - revoke-sudo + - tox-with-tmpdir-passed-in: + new_tmpdir_loc: "$HOME/xfstmp" + envlist: '{envlist}' + - assert-no-extra-files + + publishers: + - test-results + - console-log + - job-template: name: 'gate-{name}-tox-db-{envlist}-{node}' description: 'Same job as gate-{name}-tox-db-{envlist} but on node {node}' diff --git a/jenkins/scripts/run-tox.sh b/jenkins/scripts/run-tox.sh index a1d8350d2b..b9a9e02984 100755 --- a/jenkins/scripts/run-tox.sh +++ b/jenkins/scripts/run-tox.sh @@ -113,7 +113,9 @@ export PYTHON=$bin_path/python export NOSE_WITH_XUNIT=1 export NOSE_WITH_HTML_OUTPUT=1 export NOSE_HTML_OUT_FILE='nose_results.html' -export TMPDIR=$(/bin/mktemp -d) +if [[ -z "$TMPDIR" ]]; then + export TMPDIR=$(/bin/mktemp -d) +fi export UPPER_CONSTRAINTS_FILE=$(pwd)/upper-constraints.txt trap "rm -rf $TMPDIR" EXIT diff --git a/zuul/layout.yaml b/zuul/layout.yaml index 0ea519b57d..94a233f3fa 100755 --- a/zuul/layout.yaml +++ b/zuul/layout.yaml @@ -3181,7 +3181,7 @@ jobs: - name: gate-keystone-python34-db branch: ^(?!stable/liberty).*$ - - name: ^gate-swift-tox-bandit-ubuntu-(trusty|xenial)$ + - name: ^gate-swift-tox-xfs-tmp-bandit-ubuntu-(trusty|xenial)$ voting: false - name: gate-swift-python34 @@ -16832,11 +16832,15 @@ projects: - name: api-ref-jobs - name: install-guide-jobs check: - - gate-swift-tox-func-ubuntu-trusty - - gate-swift-tox-func-ubuntu-xenial - - gate-swift-tox-func-in-process-fast-post-ubuntu-trusty - - gate-swift-tox-func-in-process-fast-post-ubuntu-xenial - - gate-swift-tox-func-in-process-encryption-ubuntu-xenial + - gate-swift-tox-xfs-tmp-py27-ubuntu-trusty + - gate-swift-tox-xfs-tmp-py34-ubuntu-trusty + - gate-swift-tox-xfs-tmp-py27-ubuntu-xenial + - gate-swift-tox-xfs-tmp-py34-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-ubuntu-trusty + - gate-swift-tox-xfs-tmp-func-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-trusty + - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-in-process-encryption-ubuntu-xenial - gate-tempest-dsvm-full-ubuntu-trusty - gate-tempest-dsvm-full-ubuntu-xenial - gate-tempest-dsvm-neutron-identity-v3-only-full-ubuntu-xenial-nv @@ -16844,15 +16848,19 @@ projects: - gate-grenade-dsvm-ubuntu-xenial - gate-swift-dsvm-functional-ubuntu-trusty - gate-swift-dsvm-functional-ubuntu-xenial - - gate-swift-tox-bandit-ubuntu-trusty - - gate-swift-tox-bandit-ubuntu-xenial + - gate-swift-tox-xfs-tmp-bandit-ubuntu-trusty + - gate-swift-tox-xfs-tmp-bandit-ubuntu-xenial - gate-swift-dsvm-functional-identity-v3-only-ubuntu-xenial-nv gate: - - gate-swift-tox-func-ubuntu-trusty - - gate-swift-tox-func-ubuntu-xenial - - gate-swift-tox-func-in-process-fast-post-ubuntu-trusty - - gate-swift-tox-func-in-process-fast-post-ubuntu-xenial - - gate-swift-tox-func-in-process-encryption-ubuntu-xenial + - gate-swift-tox-xfs-tmp-py27-ubuntu-trusty + - gate-swift-tox-xfs-tmp-py34-ubuntu-trusty + - gate-swift-tox-xfs-tmp-py27-ubuntu-xenial + - gate-swift-tox-xfs-tmp-py34-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-ubuntu-trusty + - gate-swift-tox-xfs-tmp-func-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-trusty + - gate-swift-tox-xfs-tmp-func-in-process-fast-post-ubuntu-xenial + - gate-swift-tox-xfs-tmp-func-in-process-encryption-ubuntu-xenial - gate-tempest-dsvm-full-ubuntu-trusty - gate-tempest-dsvm-full-ubuntu-xenial - gate-grenade-dsvm-ubuntu-trusty