project-config/jenkins/jobs/macros.yaml

- builder:
    name: contact
    builders:
      - shell: |
          #!/bin/sh
          echo "For problems with this job, contact {name} <{email}> freenode: {irc}"          

- builder:
    name: gerrit-git-prep
    builders:
      - shell: |
          #!/bin/bash
          count=0
          until /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org git://git.openstack.org ; do
              echo "git failed, retrying"
              count=$(($count + 1))
              if [ $count -eq 3 ]; then
                  echo "Maximum of 3 git retries reached"
                  exit 1
              fi
              sleep 5
          done          

- builder:
    name: zuul-git-prep
    builders:
      - shell: |
          #!/bin/bash -xe
          cat > clonemap.yaml << EOF
          clonemap:
            - name: $ZUUL_PROJECT
              dest: .
          EOF
          # If this is called in a periodic job, these will not be set
          export ZUUL_BRANCH=${{ZUUL_BRANCH:-master}}
          export ZUUL_REF=${{ZUUL_REF:-None}}
          export BRANCH_OVERRIDE={branch-override}
          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
              export ZUUL_BRANCH=$BRANCH_OVERRIDE
          fi
          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
              git://git.openstack.org \
              $ZUUL_PROJECT          

- builder:
    name: branch-git-prep
    builders:
      - shell: |
          #!/bin/bash
          export BRANCH={branch}
          /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org git://git.openstack.org          

- builder:
    name: revoke-sudo
    builders:
      - shell: |
          #!/bin/bash -x
          sudo rm -f /etc/sudoers.d/jenkins-sudo
          # Prove that general sudo access is actually revoked
          ! sudo -n true          

- builder:
    name: coverage
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-cover.sh"

- builder:
    name: docs
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-docs.sh"

- builder:
    name: maven-test
    builders:
      - shell: "mvn test -B"

- builder:
    name: maven-package
    builders:
      - shell: "mvn package -B"

- builder:
    name: gerrit-package
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/package-gerrit.sh"

- builder:
    name: gerrit-preclean
    builders:
      - shell: |
          #!/bin/bash -xe
          rm -fr ~/.m2
          rm -fr ~/.java
          ./tools/version.sh --release          

- builder:
    name: gerrit-postrun
    builders:
      - shell: "./tools/version.sh --reset"

- builder:
    name: bashate
    builders:
      - tox:
          envlist: bashate

- builder:
    name: pep8
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-pep8.sh"

- builder:
    name: pylint
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-pylint.sh"

- builder:
    name: npm-install
    builders:
      - shell: |
          sudo apt-get update
          sudo apt-get install -y nodejs nodejs-legacy npm          

- builder:
    name: npm-run
    builders:
      - shell: |
          export DISPLAY=:99
          npm install
          npm run {command}          

- builder:
    name: xvfb-start
    builders:
      - shell: |
          DIMENSIONS='1280x1024x24'
          /usr/bin/Xvfb :99 -screen 0 ${DIMENSIONS} -ac +extension GLX +render -noreset 2>&1 > /dev/null &          

- builder:
    name: xvfb-install
    builders:
      - shell: |
          sudo apt-get update
          sudo apt-get install -y xvfb          

- builder:
    name: firefox-install
    builders:
      - shell: |
          sudo apt-get update
          sudo apt-get install -y firefox          

- builder:
    name: chrome-install
    builders:
      - shell: |
          sudo apt-get update
          sudo apt-get install -y chromium-browser          

- builder:
    name: diskimage-builder-install
    builders:
      - shell: |
          sudo apt-get update
          sudo apt-get install -y qemu kpartx debootstrap
          sudo -H pip install dib-utils
          sudo -H pip install diskimage-builder          

- builder:
    name: uninstall-puppet
    builders:
      - shell: |
          if [ -f /usr/bin/yum ]; then
              sudo yum -y remove facter puppet
          elif [ -f /usr/bin/apt-get ]; then
              sudo apt-get remove -y --purge facter puppet puppet-common
          fi          

- builder:
    name: puppet-lint
    builders:
      - shell: |
          if [ -f Modulefile -o -f metadata.json ]; then
            if [ -f Modulefile ]; then
              MODULE=$(awk '/^name/ {print $NF}' Modulefile |tr -d \"\')
            elif [ -f metadata.json ]; then
              MODULE=$(python -c 'import json;print json.load(open("metadata.json"))["name"]')
            fi
            if [ -z "$MODULE" ]; then
              echo "Module name not defined in Modulefile or metadata.json"
            else
              mkdir -p "$MODULE"
              rsync -a --exclude="$MODULE" --exclude ".*" . "$MODULE"
              cd "$MODULE"
            fi
          fi
          if [ -f Gemfile ]; then
            mkdir .bundled_gems
            export GEM_HOME=`pwd`/.bundled_gems
            bundle install --without system_tests
            bundle exec rake lint 2>&1
          else
            rake lint 2>&1
          fi          

- builder:
    name: puppet-syntax
    builders:
      - shell: |
          find . -iname *.pp | xargs puppet parser validate --modulepath=`pwd`/modules
          for f in `find . -iname *.erb` ; do
            erb -x -T '-' $f | ruby -c
          done          

# note that this job uses an in-repo script for execution
# in order to cater for differing requirements per branch
- builder:
    name: ansible-check-lint
    builders:
      - shell: |
          #!/bin/bash -xe
          sudo scripts/gate-check-lint.sh          

# note that this job uses an in-repo script for execution
# in order to cater for differing requirements per branch
- builder:
    name: ansible-check-commit
    builders:
      - shell: |
          #!/bin/bash -xe
          sudo scripts/gate-check-commit.sh          

- builder:
    name: ansible-prep
    builders:
      - shell: |
          #!/bin/bash -xe
          sudo -H pip install -U ansible          

- builder:
    name: ansible-syntax
    builders:
      - shell: |
          #!/bin/bash -xe
          cd tests
          ansible-playbook --syntax-check -i inventory test.yml -e rolename=$(basename $(dirname $(pwd)))          

- builder:
    name: ansible-lint-prep
    builders:
      - shell: |
          #!/bin/bash -xe
          sudo -H pip install ansible-lint          

- builder:
    name: ansible-lint
    builders:
      - shell: |
          #!/bin/bash -xe
          ansible-lint **/*.yml          

- builder:
    name: ruby-191-prep
    builders:
      - shell: |
          #!/bin/bash -x
          # Set 1.9.1 ruby and gem as default
          sudo update-alternatives --install /usr/bin/ruby ruby /usr/bin/ruby1.9.1 20
          sudo update-alternatives --set ruby /usr/bin/ruby1.9.1
          sudo update-alternatives --install /usr/bin/gem gem /usr/bin/gem1.9.1 10
          sudo update-alternatives --set gem /usr/bin/gem1.9.1
          sudo gem install bundler --no-rdoc --no-ri --verbose
          sudo gem install rake    --no-rdoc --no-ri --verbose          

- builder:
    name: chef-bundler-prep
    builders:
      - shell: |
          #!/bin/bash -x
          mkdir -p .bundle
          grep -E .*berkshelf.*3\.\d*\.\d*.* Gemfile
          if [ $? -eq 0 ]; then
            # For Berkshelf 3.x use the system gecode 3.x libraries
            sudo apt-get update
            sudo apt-get install -y libgecode-dev libxml2-dev libxml2 libxslt-dev build-essential
            USE_SYSTEM_GECODE=1 bundle install --path=.bundle --jobs 1 --retry 3 --verbose
          else
            bundle install --path=.bundle --jobs 1 --retry 3 --verbose
          fi          

- builder:
    name: chef-bootstrap
    builders:
      - shell: |
          #!/bin/bash -x
          ./bootstrap.sh          

- builder:
    name: chef-rake-test
    builders:
      - shell: |
          #!/bin/bash -x
          chef exec rake          

- builder:
    name: chef-rake-integration
    builders:
      - shell: |
          #!/bin/bash -x
          chef exec rake integration          

- builder:
    name: chef-berkshelf-prep
    builders:
      - shell: |
          #!/bin/bash -x
          grep -E .*berkshelf.*3\.\d*\.\d*.* Gemfile
          if [ $? -eq 0 ]; then
            bundle exec berks vendor .cookbooks
          else
            mkdir -p .cookbooks
            bundle exec berks install --path=.cookbooks
          fi          

- builder:
    name: chef-cookbook-rubocop
    builders:
      - shell: |
          #!/bin/bash -x
          if grep rubocop: Strainerfile; then
              bundle exec strainer test --cookbooks-path=.cookbooks --only=rubocop
          else
              bundle exec rubocop
          fi          

- builder:
    name: chef-cookbook-foodcritic
    builders:
      - shell: |
          #!/bin/bash -x
          if grep foodcritic: Strainerfile; then
              bundle exec strainer test --cookbooks-path=.cookbooks --only=foodcritic
          else
              bundle exec foodcritic -f any -t ~FC003 -t ~FC023 .
          fi          

- builder:
    name: chef-cookbook-chefspec
    builders:
      - shell: |
          #!/bin/bash -x
          if grep chefspec: Strainerfile; then
              bundle exec strainer test --cookbooks-path=.cookbooks --only=chefspec
          else
              bundle exec rspec --format documentation
          fi          

- builder:
    name: run-tests
    builders:
      - shell: "./run-tests.sh"

- builder:
    name: selenium
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-selenium.sh"

- builder:
    name: js-build
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-jsbuild.sh {command}"

- builder:
    name: python26
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py26"

- builder:
    name: python27
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py27"

- builder:
    name: python33
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh py33"

- builder:
    name: python34
    builders:
      - shell:  "/usr/local/jenkins/slave_scripts/run-tox.sh py34"

- builder:
    name: pypy
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh pypy"

- builder:
    name: tox
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/run-tox.sh {envlist}"

- builder:
    name: assert-no-extra-files
    builders:
      - shell: |
          #!/bin/bash
          OUT=`git ls-files --other --exclude-standard --directory`
          if [ -z "$OUT" ]; then
              echo "No extra files created during test."
              exit 0
          else
              echo "The following un-ignored files were created during the test:"
              echo "$OUT"
              exit 0  # TODO: change to 1 to fail tests.
          fi          

- builder:
    name: tarball
    builders:
      - shell: |
          #!/bin/bash -xe
          /usr/local/jenkins/slave_scripts/run-tarball.sh
          /usr/local/jenkins/slave_scripts/run-wheel.sh          

- builder:
    name: devstack-checkout
    builders:
      - shell: |
          #!/bin/bash -xe
          cat > clonemap.yaml << EOF
          clonemap:
            - name: openstack-infra/devstack-gate
              dest: devstack-gate
          EOF
          # If this is called in a periodic job, these will not be set
          export ZUUL_BRANCH=${ZUUL_BRANCH:-master}
          export ZUUL_REF=${ZUUL_REF:-None}
          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
              git://git.openstack.org \
              openstack-infra/devstack-gate          

- builder:
    name: experimental-devstack-checkout
    builders:
      - shell: |
          #!/bin/bash -xe
          if [[ ! -e devstack-gate ]]; then
              git clone git://git.openstack.org/openstack-infra/devstack-gate
          fi
          cd devstack-gate
          /usr/local/jenkins/slave_scripts/gerrit-git-prep.sh https://review.openstack.org git://git.openstack.org
          cd ..          

- builder:
    name: link-logs
    builders:
      - shell: |
          #!/bin/sh
          echo "Detailed logs: http://logs.openstack.org/$LOG_PATH/"          


- builder:
    name: net-info
    builders:
      - shell: |
          #!/bin/sh
          export PATH=$PATH:/sbin
          echo "Host & kernel"
          uname -a
          echo "Network interface addresses..."
          ip address show
          echo "Network routing tables..."
          ip route show
          ip -6 route show
          echo "Network neighbors..."
          ip neighbor show          

- builder:
    name: base-packages-trusty
    builders:
      - shell: |
          #!/bin/bash -xe
          sudo apt-get update
          sudo PATH=/usr/sbin:/sbin:$PATH DEBIAN_FRONTEND=noninteractive \
              apt-get --option "Dpkg::Options::=--force-confold" --assume-yes \
              install \
                  build-essential \
                  python-dev \
                  mysql-client \
                  mysql-server \
                  libmysqlclient-dev \
                  postgresql-client \
                  postgresql \
                  libpq-dev \
                  libffi-dev \
                  libxslt-dev \          

- builder:
    name: mysql-prep
    builders:
      - shell: |
          #!/bin/bash -xe
          DB_ROOT_PW=insecure_slave
          DB_USER=openstack_citest
          DB_PW=openstack_citest
          sudo -H mysqladmin -u root password $DB_ROOT_PW
          sudo -H mysql -u root -p$DB_ROOT_PW -h 127.0.0.1 -e "
              GRANT ALL PRIVILEGES ON *.*
                  TO '$DB_USER'@'%' identified by '$DB_PW';"          

- builder:
    name: pgsql-prep
    builders:
      - shell: |
          #!/bin/bash -xe
          DB_ROOT_PW=insecure_slave
          DB_USER=openstack_citest
          DB_PW=openstack_citest
          root_roles=$(sudo -H -u postgres psql -t -c "
              SELECT 'HERE' from pg_roles where rolname='$DB_USER'")
          if [[ ${root_roles} == *HERE ]];then
              sudo -H -u postgres psql -c "
                  ALTER ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'"
          else
              sudo -H -u postgres psql -c "
                  CREATE ROLE $DB_USER WITH SUPERUSER LOGIN PASSWORD '$DB_PW'"
          fi
          cat << EOF > $HOME/.pgpass
          *:*:*:$DB_USER:$DB_PW
          EOF
          chmod 0600 $HOME/.pgpass          

- builder:
    name: mysql-db
    builders:
      - shell: |
          #!/bin/bash -xe
          DB_USER=openstack_citest
          DB_PW=openstack_citest
          mysql -u $DB_USER -p$DB_PW -h 127.0.0.1 -e "
              SET default_storage_engine=MYISAM;
              DROP DATABASE IF EXISTS {db_name};
              CREATE DATABASE {db_name} CHARACTER SET utf8;"          

- builder:
    name: pgsql-db
    builders:
      - shell: |
          #!/bin/bash -xe
          DB_USER=openstack_citest
          DB_PW=openstack_citest
          psql -h 127.0.0.1 -U $DB_USER -d template1 -c "
              DROP DATABASE IF EXISTS {db_name}"
          createdb -h 127.0.0.1 -U $DB_USER -l C -T template0 -E utf8 {db_name}          

- builder:
    name: static-publish-prep

    # We want to publish to the docroot but the scp publisher cannot remove
    # paths, so use a shell builder to move the content in the right place and
    # publish from there.
    builders:
      - shell: |
          #!/bin/bash -xe
          rm -rf ,content
          mv {source} ,content
          rm -rf .[^.]* [^,]*
          # Disable errexit here because there might not be any dotfiles.
          set +e
          mv ,content/.[^.]* ./
          set -e
          mv ,content/* ./
          rm -r ,content          

- builder:
    name: install-buck
    builders:
      - shell: |
          #!/bin/bash -xe
          if [[ ! -d "buck" ]]
          then
            git clone https://gerrit.googlesource.com/buck
            cd buck
            ant
          fi          


- builder:
    name: zuul-swift-upload
    builders:
      - shell: "/usr/zuul-swift-logs-env/bin/python /usr/local/jenkins/slave_scripts/zuul_swift_upload.py --name {zuul_log_instruction_set} {upload_source}"


- builder:
    name: zuul-swift-upload-logs
    builders:
      - zuul-swift-upload:
         zuul_log_instruction_set: logs
         upload_source: '{upload_source}'


- builder:
    name: zuul-swift-upload-console-log
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/grab_console_log.sh"
      - zuul-swift-upload-logs:
         upload_source: '/tmp/console.html'


- builder:
    name: zuul-swift-upload-logs-with-console
    builders:
      - shell: "/usr/local/jenkins/slave_scripts/grab_console_log.sh"
      - zuul-swift-upload-logs:
         upload_source: '{upload_source} /tmp/console.html'


- builder:
    name: print-node-uuid
    builders:
      - shell: |
          #!/bin/sh
          [ -f /etc/nodepool/uuid ] && echo "Node UUID: $(cat /etc/nodepool/uuid)"          

- builder:
    name: bundle-install
    builders:
      - shell: |
          mkdir .bundled_gems
          export GEM_HOME=`pwd`/.bundled_gems
          bundle install          

- builder:
    name: exec-rake
    builders:
      - shell: |
          mkdir -p .bundled_gems
          export GEM_HOME=`pwd`/.bundled_gems
          bundle exec rake          


- builder:
    name: allow_local_ssh_root
    builders:
      - shell: |
          #!/bin/bash
          echo "" | sudo tee -a /etc/ssh/sshd_config
          echo "Match address 127.0.0.1" | sudo tee -a /etc/ssh/sshd_config
          echo "    PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config
          echo "" | sudo tee -a /etc/ssh/sshd_config
          echo "Match address ::1" | sudo tee -a /etc/ssh/sshd_config
          echo "    PermitRootLogin without-password" | sudo tee -a /etc/ssh/sshd_config
          mkdir -p .ssh
          ssh-keygen -f ~/.ssh/id_rsa -b 2048 -P ""
          sudo mkdir -p /root/.ssh
          cat ~/.ssh/id_rsa.pub | sudo tee -a /root/.ssh/authorized_keys
          if [ -f /usr/bin/yum ]; then
              sudo systemctl restart sshd
          elif [ -f /usr/bin/apt-get ]; then
              sudo service ssh restart
          fi          

# ======================================================================

- wrapper:
    name: build-timeout
    wrappers:
      - timeout:
          timeout: '{timeout}'
          timeout-var: 'BUILD_TIMEOUT'
          fail: true

# ======================================================================

- publisher:
    name: tarball
    publishers:
      - archive:
          artifacts: 'dist/*.tar.gz'
      - scp:
          site: '{site}'
          files:
            - target: 'tarballs/{project}/'
              source: 'dist/*.tar.gz'

- publisher:
    name: wheel
    publishers:
      - archive:
          artifacts: 'dist/*.whl'
      - scp:
          site: '{site}'
          files:
            - target: 'tarballs/{project}/'
              source: 'dist/*.whl'

- publisher:
    name: war
    publishers:
      - archive:
          artifacts: '{warfile}'
      - scp:
          site: '{site}'
          files:
            - target: '{target}'
              source: '{warfile}'

- publisher:
    name: console-log
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              copy-console: true
              copy-after-failure: true


- publisher:
    name: devstack-logs
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              source: 'logs/**'
              keep-hierarchy: true
              copy-after-failure: true


- publisher:
    name: coverage-log
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              source: 'cover/**'
              keep-hierarchy: true
              copy-after-failure: true

- publisher:
    name: coverage-log-ruby
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              source: 'coverage/**'
              keep-hierarchy: true
              copy-after-failure: true

- publisher:
    name: test-results
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              source: '**/*nose_results.html'
              keep-hierarchy: false
              copy-after-failure: true
            - target: 'logs/$LOG_PATH'
              source: '**/*testr_results.html.gz'
              keep-hierarchy: false
              copy-after-failure: true
            - target: 'logs/$LOG_PATH'
              source: '.testrepository/tmp*'
              keep-hierarchy: false
              copy-after-failure: true
            - target: 'logs/$LOG_PATH'
              source: '**/*testrepository.subunit.gz'
              keep-hierarchy: false
              copy-after-failure: true
            - target: 'logs/$LOG_PATH/tox'
              source: '.tox/*/log/*'
              keep-hierarchy: false
              copy-after-failure: true

- publisher:
    name: upload-sphinx-draft
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'docs-draft/$LOG_PATH'
              source: 'doc/build/html/**'
              keep-hierarchy: true
              copy-after-failure: true

- publisher:
    name: upload-docs-draft
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'docs-draft/$LOG_PATH/'
              source: 'publish-docs/**'
              keep-hierarchy: true
              copy-after-failure: true


- publisher:
    name: upload-to-docs-site
    publishers:
      - ftp:
          site: docs.openstack.org
          source: 'publish-docs/**'
          target: '/'
          remove-prefix: 'publish-docs/'

- publisher:
    name: upload-doc-build-results
    publishers:
      - scp:
          site: 'static.openstack.org'
          files:
            - target: 'logs/$LOG_PATH'
              source: 'build-*.log.gz'
              keep-hierarchy: false
              copy-after-failure: true

- publisher:
    name: zuul-swift-upload
    publishers:
      - postbuildscript:
          builders:
            - zuul-swift-upload:
                zuul_log_instruction_set: '{zuul_log_instruction_set}'
                upload_source: '{upload_source}'
          # These flags really mean only if on * when
          # checked. When both set to false the post script
          # should run on every job regardless of status.
          onfailure: False
          onsuccess: False

- publisher:
    name: zuul-swift-upload-logs
    publishers:
      - postbuildscript:
          builders:
            - zuul-swift-upload-logs:
                upload_source: '{upload_source}'
          # These flags really mean only if on * when
          # checked. When both set to false the post script
          # should run on every job regardless of status.
          onfailure: False
          onsuccess: False

- publisher:
    name: zuul-swift-upload-console-log
    publishers:
      - postbuildscript:
          builders:
            - zuul-swift-upload-console-log
          # These flags really mean only if on * when
          # checked. When both set to false the post script
          # should run on every job regardless of status.
          onfailure: False
          onsuccess: False

- publisher:
    name: zuul-swift-upload-logs-with-console
    publishers:
      - postbuildscript:
          builders:
            - zuul-swift-upload-logs-with-console:
                upload_source: '{upload_source}'
          # These flags really mean only if on * when
          # checked. When both set to false the post script
          # should run on every job regardless of status.
          onfailure: False
          onsuccess: False

- publisher:
    name: zuul-swift-devstack-logs-with-console
    publishers:
      - postbuildscript:
          builders:
            - shell: |
                #!/bin/bash -x
                cp devstack-gate/help/tempest-logs.html logs/index_footer.html
                mkdir zuul_swift_devstack_logs_extra
                cp devstack-gate/help/tempest-overview.html zuul_swift_devstack_logs_extra/index_footer.html                
      - zuul-swift-upload-logs-with-console:
          upload_source: 'logs zuul_swift_devstack_logs_extra/index_footer.html'

- publisher:
    name: zuul-swift-test-results-with-console
    publishers:
      - zuul-swift-upload-logs-with-console:
          upload_source: '"**/*nose_results.html" "**/*testr_results.html.gz" ".testrepository/tmp*" "**/*testrepository.subunit.gz" ".tox/*/log/*"'