- builder: name: contact builders: - shell: | #!/bin/sh echo "For problems with this job, contact {name} <{email}> freenode: {irc}" - builder: name: print-template-name builders: - shell: 'echo JJB template: {template-name}' - builder: name: gerrit-git-prep builders: - shell: | #!/bin/bash count=0 until /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org git://git.openstack.org ; do echo "git failed, retrying" count=$(($count + 1)) if [ $count -eq 3 ]; then echo "Maximum of 3 git retries reached" exit 1 fi sleep 5 done - builder: name: zuul-git-prep builders: - shell: | #!/bin/bash -xe CLONEMAP=`mktemp` function cleanup { rm -f $CLONEMAP } trap cleanup EXIT cat > $CLONEMAP << EOF clonemap: - name: $ZUUL_PROJECT dest: . EOF /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ git://git.openstack.org $ZUUL_PROJECT - builder: name: zuul-git-prep-upper-constraints builders: - shell: | #!/bin/bash -xe export BRANCH=${ZUUL_BRANCH:-$ZUUL_REFNAME} CLONEMAP=`mktemp` REQS_DIR=`mktemp -d` function cleanup { rm -rf $CLONEMAP $REQS_DIR } trap cleanup EXIT cat > $CLONEMAP << EOF clonemap: - name: $ZUUL_PROJECT dest: . - name: openstack/requirements dest: $REQS_DIR EOF /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ --branch=$BRANCH \ git://git.openstack.org $ZUUL_PROJECT openstack/requirements cp $REQS_DIR/upper-constraints.txt ./ - builder: name: zuul-git-branch-prep-upper-constraints builders: - shell: | #!/bin/bash -xe export BRANCH={branch} CLONEMAP=`mktemp` REQS_DIR=`mktemp -d` function cleanup {{ rm -rf $CLONEMAP $REQS_DIR }} trap cleanup EXIT cat > $CLONEMAP << EOF clonemap: - name: $ZUUL_PROJECT dest: . - name: openstack/requirements dest: $REQS_DIR EOF /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ --branch=$BRANCH \ git://git.openstack.org $ZUUL_PROJECT openstack/requirements cp $REQS_DIR/upper-constraints.txt ./ - builder: name: zuul-git-branch-prep builders: - shell: | #!/bin/bash -xe CLONEMAP=`mktemp` function cleanup {{ rm -f $CLONEMAP }} trap cleanup EXIT cat > $CLONEMAP << EOF clonemap: - name: $ZUUL_PROJECT dest: . EOF # If this is called in a periodic job, these will not be set export ZUUL_BRANCH=${{ZUUL_BRANCH:-master}} export ZUUL_REF=${{ZUUL_REF:-None}} export BRANCH_OVERRIDE={branch-override} if [ "$BRANCH_OVERRIDE" != "default" ] ; then BRANCH_ARG="--branch $BRANCH_OVERRIDE" fi /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \ $BRANCH_ARG git://git.openstack.org $ZUUL_PROJECT - builder: name: branch-git-prep builders: - shell: | #!/bin/bash export BRANCH={branch} /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org git://git.openstack.org - builder: name: zuul-clone builders: - shell: | #!/bin/bash /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \ git://git.openstack.org {project} - builder: name: revoke-sudo builders: - shell: | #!/bin/bash -x sudo rm -f /etc/sudoers.d/jenkins-sudo # Prove that general sudo access is actually revoked ! sudo -n true - builder: name: coverage builders: - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh {env}" - builder: name: docs-env builders: - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh {env}" - builder: name: docs builders: - docs-env: env: venv - builder: name: maven-test builders: - shell: "mvn test -B" - builder: name: maven-package builders: - shell: "mvn package -B" - builder: name: gerrit-package builders: - shell: "/usr/local/jenkins/slave_scripts/package-gerrit.sh" - builder: name: bashate builders: - tox: envlist: bashate - builder: name: pep8 builders: - shell: "/usr/local/jenkins/slave_scripts/run-pep8.sh {env}" - builder: name: pylint builders: - shell: "/usr/local/jenkins/slave_scripts/run-pylint.sh {env}" - builder: name: npm-install builders: - shell: | sudo apt-get update sudo apt-get install -y nodejs nodejs-legacy npm - builder: name: npm-run builders: - shell: | export DISPLAY=:99 npm install # Execute shrinkwrap, and output the dependencies actually in use. npm shrinkwrap cat ./npm-shrinkwrap.json # Try running as a standard lifecycle script, otherwise try custom. npm_lifecycle_phases="publish install version test stop start restart pack" if [[ $npm_lifecycle_phases =~ (^| ){command}($| ) ]]; then npm {command} --silent else npm run {command} --silent fi - builder: name: xvfb-start builders: - shell: | DIMENSIONS='1280x1024x24' /usr/bin/Xvfb :99 -screen 0 ${DIMENSIONS} -ac +extension GLX +render -noreset 2>&1 > /dev/null & - builder: name: xvfb-install builders: - shell: | sudo apt-get update sudo apt-get install -y xvfb - builder: name: firefox-install builders: - shell: | sudo apt-get update sudo apt-get install -y firefox - builder: name: chrome-install builders: - shell: | sudo apt-get update sudo apt-get install -y chromium-browser - builder: name: diskimage-builder-install builders: - shell: | sudo apt-get update sudo apt-get install -y qemu kpartx debootstrap sudo -H pip install dib-utils sudo -H pip install diskimage-builder - builder: name: puppet-prepare-node builders: - shell: | if [ -f /usr/bin/yum ]; then sudo yum -y remove facter puppet rdo-release sudo yum -y install libxml2-devel libxslt-devel ruby-devel sudo yum -y groupinstall "Development Tools" elif [ -f /usr/bin/apt-get ]; then sudo apt-get remove -y --purge facter puppet puppet-common sudo apt-get update sudo apt-get install -y libxml2-dev libxslt-dev zlib1g-dev fi - builder: name: puppet-lint builders: - shell: | if [ -f Modulefile -o -f metadata.json ]; then if [ -f Modulefile ]; then MODULE=$(awk '/^name/ {print $NF}' Modulefile |tr -d \"\') elif [ -f metadata.json ]; then MODULE=$(python -c 'import json;print json.load(open("metadata.json"))["name"]') fi if [ -z "$MODULE" ]; then echo "Module name not defined in Modulefile or metadata.json" else mkdir -p "$MODULE" rsync -a --exclude="$MODULE" --exclude ".*" . "$MODULE" cd "$MODULE" fi fi if [ -f Gemfile ]; then mkdir .bundled_gems export GEM_HOME=`pwd`/.bundled_gems gem install bundler --no-rdoc --no-ri --verbose $GEM_HOME/bin/bundle install --without system_tests $GEM_HOME/bin/bundle exec rake lint 2>&1 else rake lint 2>&1 fi - builder: name: puppet-syntax builders: - shell: | find . -iname *.pp | xargs puppet parser validate --modulepath=`pwd`/modules for f in `find . -iname *.erb` ; do erb -x -T '-' $f | ruby -c done # note that this job uses an in-repo script for execution # in order to cater for differing requirements per branch - builder: name: ansible-check-lint builders: - shell: | #!/bin/bash -xe sudo scripts/gate-check-lint.sh # note that this job uses an in-repo script for execution # in order to cater for differing requirements per branch - builder: name: ansible-check-commit builders: - shell: | #!/bin/bash -xe sudo scripts/gate-check-commit.sh - builder: name: ansible-prep builders: - shell: | #!/bin/bash -xe sudo -H pip install -U ansible - builder: name: ansible-syntax builders: - shell: | #!/bin/bash -xe cd tests ansible-playbook --syntax-check -i inventory test.yml -e rolename=$(basename $(dirname $(pwd))) - builder: name: ansible-lint-prep builders: - shell: | #!/bin/bash -xe sudo -H pip install ansible-lint - builder: name: ansible-lint builders: - shell: | #!/bin/bash -xe ansible-lint **/*.yml - builder: name: ruby-191-prep builders: - shell: | #!/bin/bash -x # Set 1.9.1 ruby and gem as default sudo update-alternatives --install /usr/bin/ruby ruby /usr/bin/ruby1.9.1 20 sudo update-alternatives --set ruby /usr/bin/ruby1.9.1 sudo update-alternatives --install /usr/bin/gem gem /usr/bin/gem1.9.1 10 sudo update-alternatives --set gem /usr/bin/gem1.9.1 sudo gem install bundler --no-rdoc --no-ri --verbose sudo gem install rake --no-rdoc --no-ri --verbose - builder: name: chef-bundler-prep builders: - shell: | #!/bin/bash -x mkdir -p .bundle grep -E .*berkshelf.*3\.\d*\.\d*.* Gemfile if [ $? -eq 0 ]; then # For Berkshelf 3.x use the system gecode 3.x libraries sudo apt-get update sudo apt-get install -y libgecode-dev libxml2-dev libxml2 libxslt-dev build-essential USE_SYSTEM_GECODE=1 bundle install --path=.bundle --jobs 1 --retry 3 --verbose else bundle install --path=.bundle --jobs 1 --retry 3 --verbose fi - builder: name: chef-bootstrap builders: - shell: | #!/bin/bash -x ./bootstrap.sh - builder: name: chef-rake-test builders: - shell: | #!/bin/bash -x chef exec rake - builder: name: chef-rake-integration builders: - shell: | #!/bin/bash -x chef exec rake integration - builder: name: chef-berkshelf-prep builders: - shell: | #!/bin/bash -x grep -E .*berkshelf.*3\.\d*\.\d*.* Gemfile if [ $? -eq 0 ]; then bundle exec berks vendor .cookbooks else mkdir -p .cookbooks bundle exec berks install --path=.cookbooks fi - builder: name: chef-cookbook-rubocop builders: - shell: | #!/bin/bash -x if grep rubocop: Strainerfile; then bundle exec strainer test --cookbooks-path=.cookbooks --only=rubocop else bundle exec rubocop fi - builder: name: chef-cookbook-foodcritic builders: - shell: | #!/bin/bash -x if grep foodcritic: Strainerfile; then bundle exec strainer test --cookbooks-path=.cookbooks --only=foodcritic else bundle exec foodcritic -f any -t ~FC003 -t ~FC023 . fi - builder: name: chef-cookbook-chefspec builders: - shell: | #!/bin/bash -x if grep chefspec: Strainerfile; then bundle exec strainer test --cookbooks-path=.cookbooks --only=chefspec else bundle exec rspec --format documentation fi - builder: name: run-tests builders: - shell: "./run-tests.sh" - builder: name: selenium builders: - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh" - builder: name: js-build builders: - shell: "/usr/local/jenkins/slave_scripts/run-jsbuild.sh {command}" - builder: name: python27 builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py27" - builder: name: python33 builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py33" - builder: name: python34 builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py34" - builder: name: pypy builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh pypy" - builder: name: tox builders: - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {envlist}" - builder: name: assert-no-extra-files builders: - shell: | #!/bin/bash OUT=`git ls-files --other --exclude-standard --directory` if [ -z "$OUT" ]; then echo "No extra files created during test." exit 0 else echo "The following un-ignored files were created during the test:" echo "$OUT" exit 0 # TODO: change to 1 to fail tests. fi - builder: name: tarball builders: - shell: | #!/bin/bash -xe /usr/local/jenkins/slave_scripts/run-tarball.sh {env} /usr/local/jenkins/slave_scripts/run-wheel.sh {env} - builder: name: devstack-checkout builders: - shell: | #!/bin/bash -xe cat > clonemap.yaml << EOF clonemap: - name: openstack-infra/devstack-gate dest: devstack-gate EOF /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \ git://git.openstack.org \ openstack-infra/devstack-gate - builder: name: link-logs builders: - shell: | #!/bin/sh echo "Detailed logs: http://logs.openstack.org/$LOG_PATH/" - builder: name: net-info builders: - shell: | #!/bin/sh export PATH=$PATH:/sbin echo "Host & kernel" uname -a echo "Network interface addresses..." ip address show echo "Network routing tables..." ip route show ip -6 route show echo "Network neighbors..." ip neighbor show - builder: name: install-distro-packages builders: - shell: /usr/local/jenkins/slave_scripts/install-distro-packages.sh - builder: name: install-distro-packages-fallback-test builders: - shell: | #!/bin/bash -xe if [ "$ZUUL_PROJECT" == "openstack-infra/bindep" ] ; then # if testing a change to bindep, use its entrypoint virtualenv bindep-env bindep-env/bin/pip install -U . export BINDEP=bindep-env/bin/bindep elif [ "$ZUUL_PROJECT" == "openstack-infra/project-config" ] ; then # if testing a change to project-config, use its fallback list export PACKAGES=jenkins/data/bindep-fallback.txt fi /usr/local/jenkins/slave_scripts/install-distro-packages.sh - builder: name: mysql-prep builders: - shell: | #!/bin/bash -xe DB_ROOT_PW=insecure_slave DB_USER=openstack_citest DB_PW=openstack_citest sudo -H mysqladmin -u root password $DB_ROOT_PW sudo -H mysql -u root -p$DB_ROOT_PW -h 127.0.0.1 -e " GRANT ALL PRIVILEGES ON *.* TO '$DB_USER'@'%' identified by '$DB_PW';" - builder: name: pgsql-prep builders: - shell: | #!/bin/bash -xe DB_ROOT_PW=insecure_slave DB_USER=openstack_citest DB_PW=openstack_citest root_roles=$(sudo -H -u postgres psql -t -c " SELECT 'HERE' from pg_roles where rolname='$DB_USER'") if [[ ${root_roles} == *HERE ]];then sudo -H -u postgres psql -c " ALTER ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" else sudo -H -u postgres psql -c " CREATE ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'" fi cat << EOF > $HOME/.pgpass *:*:*:$DB_USER:$DB_PW EOF chmod 0600 $HOME/.pgpass - builder: name: mysql-db builders: - shell: | #!/bin/bash -xe DB_USER=openstack_citest DB_PW=openstack_citest mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e " SET default_storage_engine=MYISAM; DROP DATABASE IF EXISTS {db_name}; CREATE DATABASE {db_name} CHARACTER SET utf8;" - builder: name: pgsql-db builders: - shell: | #!/bin/bash -xe DB_USER=openstack_citest DB_PW=openstack_citest psql -h 127.0.0.1 -U $DB_USER -d template1 -c " DROP DATABASE IF EXISTS {db_name}" createdb -h 127.0.0.1 -U $DB_USER -l C -T template0 -E utf8 {db_name} - builder: name: static-publish-prep # We want to publish to the docroot but the scp publisher cannot remove # paths, so use a shell builder to move the content in the right place and # publish from there. builders: - shell: | #!/bin/bash -xe rm -rf ,content mv {source} ,content rm -rf .[^.]* [^,]* # Disable errexit here because there might not be any dotfiles. set +e mv ,content/.[^.]* ./ set -e mv ,content/* ./ rm -r ,content - builder: name: install-buck builders: - shell: | #!/bin/bash -xe if [[ ! -d "buck" ]] then git clone https://gerrit.googlesource.com/buck cd buck ant fi - builder: name: zuul-swift-upload builders: - shell: "/usr/zuul-swift-logs-env/bin/python /usr/local/jenkins/slave_scripts/zuul_swift_upload.py --name {zuul_log_instruction_set} {upload_source}" - builder: name: zuul-swift-upload-logs builders: - zuul-swift-upload: zuul_log_instruction_set: logs upload_source: '{upload_source}' - builder: name: zuul-swift-upload-console-log builders: - shell: "/usr/local/jenkins/slave_scripts/grab_console_log.sh" - zuul-swift-upload-logs: upload_source: '/tmp/console.html' - builder: name: zuul-swift-upload-logs-with-console builders: - shell: "/usr/local/jenkins/slave_scripts/grab_console_log.sh" - zuul-swift-upload-logs: upload_source: '{upload_source} /tmp/console.html' - builder: name: print-node-uuid builders: - shell: | #!/bin/sh [ -f /etc/nodepool/uuid ] && echo "Node UUID: $(cat /etc/nodepool/uuid)" - builder: name: bundle-install builders: - shell: | mkdir .bundled_gems export GEM_HOME=`pwd`/.bundled_gems bundle install - builder: name: exec-rake builders: - shell: | mkdir -p .bundled_gems export GEM_HOME=`pwd`/.bundled_gems bundle exec rake - builder: name: allow-local-ssh-root builders: - shell: | #!/bin/bash echo "" | sudo tee -a /etc/ssh/sshd_config echo "Match address 127.0.0.1" | sudo tee -a /etc/ssh/sshd_config echo " PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config echo "" | sudo tee -a /etc/ssh/sshd_config echo "Match address ::1" | sudo tee -a /etc/ssh/sshd_config echo " PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config mkdir -p .ssh ssh-keygen -f ~/.ssh/id_rsa -b 2048 -P "" sudo mkdir -p /root/.ssh cat ~/.ssh/id_rsa.pub | sudo tee -a /root/.ssh/authorized_keys if [ -f /usr/bin/yum ]; then sudo systemctl restart sshd elif [ -f /usr/bin/apt-get ]; then sudo service ssh restart fi # ====================================================================== - wrapper: name: build-timeout wrappers: - timeout: timeout: '{timeout}' timeout-var: 'BUILD_TIMEOUT' fail: true # ====================================================================== - publisher: name: tarball publishers: - archive: artifacts: 'dist/*.tar.gz' - scp: site: '{site}' files: - target: 'tarballs/{project}/' source: 'dist/*.tar.gz' - publisher: name: wheel publishers: - archive: artifacts: 'dist/*.whl' - scp: site: '{site}' files: - target: 'tarballs/{project}/' source: 'dist/*.whl' - publisher: name: war publishers: - archive: artifacts: '{warfile}' - scp: site: '{site}' files: - target: '{target}' source: '{warfile}' - publisher: name: console-log publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' copy-console: true copy-after-failure: true - publisher: name: devstack-logs publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' source: 'logs/**' keep-hierarchy: true copy-after-failure: true - publisher: name: coverage-log publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' source: 'cover/**' keep-hierarchy: true copy-after-failure: true - publisher: name: coverage-log-ruby publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' source: 'coverage/**' keep-hierarchy: true copy-after-failure: true - publisher: name: test-results publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' source: '**/*nose_results.html' keep-hierarchy: false copy-after-failure: true - target: 'logs/$LOG_PATH' source: '**/*testr_results.html.gz' keep-hierarchy: false copy-after-failure: true - target: 'logs/$LOG_PATH' source: '.testrepository/tmp*' keep-hierarchy: false copy-after-failure: true - target: 'logs/$LOG_PATH' source: '**/*testrepository.subunit.gz' keep-hierarchy: false copy-after-failure: true - target: 'logs/$LOG_PATH/tox' source: '.tox/*/log/*' keep-hierarchy: false copy-after-failure: true - publisher: name: upload-sphinx-draft publishers: - scp: site: 'static.openstack.org' files: - target: 'docs-draft/$LOG_PATH' source: 'doc/build/html/**' keep-hierarchy: true copy-after-failure: true - publisher: name: upload-api-guide-draft publishers: - scp: site: 'static.openstack.org' files: - target: 'docs-draft/$LOG_PATH' source: 'api-guide/build/html/**' keep-hierarchy: true copy-after-failure: true - publisher: name: upload-docs-draft publishers: - scp: site: 'static.openstack.org' files: - target: 'docs-draft/$LOG_PATH/' source: 'publish-docs/**' keep-hierarchy: true copy-after-failure: true - publisher: name: upload-releasenotes-draft publishers: - scp: site: 'static.openstack.org' files: - target: 'docs-draft/$LOG_PATH' source: 'releasenotes/build/html/**' keep-hierarchy: true copy-after-failure: true - publisher: name: upload-to-docs-site publishers: - ftp: site: docs.openstack.org source: 'publish-docs/**' target: '/' remove-prefix: 'publish-docs/' - publisher: name: upload-doc-build-results publishers: - scp: site: 'static.openstack.org' files: - target: 'logs/$LOG_PATH' source: 'build-*.log.gz' keep-hierarchy: false copy-after-failure: true - publisher: name: zuul-swift-upload publishers: - postbuildscript: builders: - zuul-swift-upload: zuul_log_instruction_set: '{zuul_log_instruction_set}' upload_source: '{upload_source}' # These flags really mean only if on * when # checked. When both set to false the post script # should run on every job regardless of status. onfailure: False onsuccess: False - publisher: name: zuul-swift-upload-logs publishers: - postbuildscript: builders: - zuul-swift-upload-logs: upload_source: '{upload_source}' # These flags really mean only if on * when # checked. When both set to false the post script # should run on every job regardless of status. onfailure: False onsuccess: False - publisher: name: zuul-swift-upload-console-log publishers: - postbuildscript: builders: - zuul-swift-upload-console-log # These flags really mean only if on * when # checked. When both set to false the post script # should run on every job regardless of status. onfailure: False onsuccess: False - publisher: name: zuul-swift-upload-logs-with-console publishers: - postbuildscript: builders: - zuul-swift-upload-logs-with-console: upload_source: '{upload_source}' # These flags really mean only if on * when # checked. When both set to false the post script # should run on every job regardless of status. onfailure: False onsuccess: False - publisher: name: zuul-swift-devstack-logs-with-console publishers: - postbuildscript: builders: - shell: | #!/bin/bash -x cp devstack-gate/help/tempest-logs.html logs/index_footer.html mkdir zuul_swift_devstack_logs_extra cp devstack-gate/help/tempest-overview.html zuul_swift_devstack_logs_extra/index_footer.html - zuul-swift-upload-logs-with-console: upload_source: 'logs zuul_swift_devstack_logs_extra/index_footer.html' - publisher: name: zuul-swift-test-results-with-console publishers: - zuul-swift-upload-logs-with-console: upload_source: '"**/*nose_results.html" "**/*testr_results.html.gz" ".testrepository/tmp*" "**/*testrepository.subunit.gz" ".tox/*/log/*"' - publisher: name: proposal-slave-cleanup publishers: - postbuildscript: builders: - shell: "/usr/local/jenkins/slave_scripts/slave-cleanup.sh" - publisher: name: copy-puppet-logs publishers: - postbuildscript: builders: - shell: | #!/bin/bash -x /usr/local/jenkins/slave_scripts/copy_puppet_logs.sh # These flags really mean only if on * when # checked. When both set to false the post script # should run on every job regardless of status. onfailure: False onsuccess: False