Add a new job on fedora-28 images to test python3

This patch is creating a new job running on fedora-28 images which
runs scenario-py3.sh to validate python3 builds on fedora.

Job is non-voting.

Additionally this patch is fixing puppetfile to get newer versions,
aligned with what we have in p-o-i.

Change-Id: I404fc6d38dd5177a3aaef51b0450ad2850595e32

.zuul.yaml

@@ -2,7 +2,6 @@
     name: packstack-base
     parent: base
     timeout: 7200
-    pre-run: playbooks/packstack-pre.yaml
     post-run: playbooks/upload-logs.yaml
     irrelevant-files:
       - ^docs/.*$
@@ -36,15 +35,31 @@
 - job:
     name: packstack-allinone
     parent: packstack-base
+    pre-run: playbooks/packstack-pre.yaml
     nodeset:
       nodes:
         - name: allinone
           label: centos-7
     run: playbooks/packstack-integration-tempest.yaml
 
+- job:
+    name: packstack-fedora
+    parent: packstack-base
+    vars:
+      delorean: "https://trunk.rdoproject.org/fedora/consistent/delorean.repo"
+      delorean_deps: "https://trunk.rdoproject.org/fedora/delorean-deps.repo"
+      selinux_enforcing: "false"
+    pre-run: playbooks/packstack-fedora-pre.yaml
+    nodeset:
+      nodes:
+        - name: fedora
+          label: fedora-28
+    run: playbooks/packstack-integration-tempest.yaml
+
 - job:
     name: packstack-multinode
     parent: packstack-base
+    pre-run: playbooks/packstack-pre.yaml
     nodeset:
       nodes:
         - name: controller
@@ -71,6 +86,12 @@
     vars:
       scenario: scenario003
 
+- job:
+    name: packstack-fedora-integration-scenario-py3-tempest
+    parent: packstack-fedora
+    vars:
+      scenario: scenario-py3
+
 # We should have nested post-run jobs, according to
 # https://docs.openstack.org/infra/zuul/user/config.html#job
 - job:
@@ -87,6 +108,8 @@
         - packstack-integration-scenario002-tempest
         - packstack-integration-scenario003-tempest
         - packstack-multinode-scenario002-tempest
+        - packstack-fedora-integration-scenario-py3-tempest:
+            voting: false
         - openstack-tox-py36:
             voting: false
     gate:

Puppetfile

@@ -98,7 +98,7 @@ mod 'vswitch',
 
 mod 'apache',
   :git => 'https://github.com/puppetlabs/puppetlabs-apache',
-  :ref => '1.11.0'
+  :ref => 'b4cbc858a61ba2eea4a1178c261a982a59db963d'
 
 mod 'certmonger',
   :git => 'https://github.com/rcritten/puppet-certmonger',
@@ -106,23 +106,23 @@ mod 'certmonger',
 
 mod 'concat',
   :git => 'https://github.com/puppetlabs/puppetlabs-concat',
-  :ref => '3.0.0'
+  :ref => '4.2.1'
 
 mod 'firewall',
   :git => 'https://github.com/puppetlabs/puppetlabs-firewall',
-  :ref => '1.8.2'
+  :ref => '1.12.0'
 
 mod 'inifile',
   :git => 'https://github.com/puppetlabs/puppetlabs-inifile',
-  :ref => '1.6.0'
+  :ref => '2.2.0'
 
 mod 'memcached',
   :git => 'https://github.com/saz/puppet-memcached',
-  :ref => 'v3.0.1'
+  :ref => 'v3.3.0'
 
 mod 'mysql',
   :git => 'https://github.com/puppetlabs/puppetlabs-mysql',
-  :ref => '3.10.0'
+  :ref => '6.0.0'
 
 mod 'nssdb',
   :git => 'https://github.com/rcritten/puppet-nssdb',
@@ -142,7 +142,7 @@ mod 'remote',
 
 mod 'rsync',
   :git => 'https://github.com/puppetlabs/puppetlabs-rsync',
-  :ref => '0.4.0'
+  :ref => '1.1.0'
 
 mod 'ssh',
   :git => 'https://github.com/saz/puppet-ssh',
@@ -150,17 +150,17 @@ mod 'ssh',
 
 mod 'stdlib',
   :git => 'https://github.com/puppetlabs/puppetlabs-stdlib',
-  :ref => '4.18.0'
+  :ref => '5.0.0'
 
 mod 'sysctl',
   :git => 'https://github.com/duritong/puppet-sysctl',
-  :ref => 'v0.0.11'
+  :ref => 'v0.0.12'
 
 mod 'vcsrepo',
   :git => 'https://github.com/puppetlabs/puppetlabs-vcsrepo',
-  :ref => '1.5.0'
+  :ref => '2.3.0'
 
 mod 'xinetd',
   :git => 'https://github.com/puppetlabs/puppetlabs-xinetd',
-  :ref => '2.0.0'
+  :ref => '3.1.0'
 

playbooks/packstack-fedora-pre.yaml

@@ -0,0 +1,28 @@
+- hosts: all
+  name: packstack-fedora-pre
+  tasks:
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          cp -pr /home/zuul/src/git.openstack.org/openstack/packstack {{ ansible_user_dir }}/workspace
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          dnf -y install wget
+          rm -rf /etc/yum.repos.d/fedora*
+          wget -O /etc/yum.repos.d/fedora-stable.repo http://trunk.rdoproject.org/fedora/delorean-deps.repo
+          dnf clean all
+          sudo sed -i '/^exclude.*/d' /etc/dnf/dnf.conf
+          dnf -y install libxml2-devel libxslt-devel ruby-devel zlib-devel
+          dnf -y install gcc gettext diffstat doxygen patch patchutils subversion systemtap git python-setuptools wget redhat-lsb-core libselinux-python yum-plugin-priorities dnf-yum
+          pip uninstall requests -y || true
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      become: true
+      environment: '{{ zuul }}'

roles/packstack-integration-tempest/tasks/main.yaml

@@ -4,6 +4,9 @@
         set -e
         set -x
         export SCENARIO='{{ scenario }}'
+        export DELOREAN='{{ delorean | default('') }}'
+        export DELOREAN_DEPS='{{ delorean_deps | default('') }}'
+        export SELINUX_ENFORCING='{{ selinux_enforcing | default('true') }}'
         ./run_tests.sh
       executable: /bin/bash
       chdir: '{{ ansible_user_dir }}/workspace/packstack'

run_tests.sh

@@ -32,6 +32,9 @@ COPY_LOGS=${COPY_LOGS:-true}
 CONTROLLER_NODE=${CONTROLLER_NODE:-}
 COMPUTE_NODE=${COMPUTE_NODE:-}
 
+# Use dnf as package manager if it exist
+type -p dnf && export PKG_MGR=dnf || export PKG_MGR=yum
+
 # Install external Puppet modules with r10k
 # Uses the following variables:
 #
@@ -150,11 +153,11 @@ fi
 if [ "${MANAGE_REPOS}" = true ]; then
     $SUDO curl -L ${DELOREAN} -o /etc/yum.repos.d/delorean.repo
     $SUDO curl -L ${DELOREAN_DEPS} -o /etc/yum.repos.d/delorean-deps.repo
-    $SUDO yum update -y
+    $SUDO $PKG_MGR update -y
 fi
 
 # Install dependencies
-$SUDO yum -y install puppet \
+$SUDO $PKG_MGR -y install puppet \
                      yum-plugin-priorities \
                      iproute \
                      dstat \
@@ -170,7 +173,13 @@ $SUDO yum -y install puppet \
                      policycoreutils \
                      rubygems \
                      wget \
-                     "@Development Tools"
+                     gettext \
+                     diffstat \
+                     doxygen \
+                     patch \
+                     patchutils \
+                     subversion \
+                     systemtap
 
 # Don't assume pip is installed
 which pip || $SUDO easy_install pip
@@ -253,7 +262,12 @@ fi
 # Setup packstack
 if [ "${INSTALL_FROM_SOURCE}" = true ]; then
   $SUDO pip install .
-  export GEM_BIN_DIR=/tmp/packstackgems/bin/
+  # In Fedora when running with sudo gems are installed at /usr/local/bin/ even when GEM_HOME/GEM_BIN_DIR are set
+  if [ "${PKG_MGR}" = "dnf" ]; then
+      export GEM_BIN_DIR=/usr/local/bin/
+  else
+      export GEM_BIN_DIR=/tmp/packstackgems/bin/
+  fi
   export PUPPETFILE_DIR=/usr/share/openstack-puppet/modules
   export GEM_HOME=/tmp/packstackgems
   $SUDO gem install r10k -v 2.6.4 --no-ri --no-rdoc
@@ -261,7 +275,7 @@ if [ "${INSTALL_FROM_SOURCE}" = true ]; then
   $SUDO rm -rf "${PUPPETFILE_DIR:?}/"*
   install_modules
 else
-  $SUDO yum -y install openstack-packstack
+  $SUDO $PKG_MGR -y install openstack-packstack
 fi
 
 # Make sure there are no other puppet modules in the system (happens in gate)

tests/scenario-py3.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+CONTROLLER_NODE=${CONTROLLER_NODE:-}
+COMPUTE_NODE=${COMPUTE_NODE:-}
+
+if [ $(id -u) != 0 ]; then
+    SUDO='sudo'
+fi
+
+echo -e "Generating packstack config for:
+- keystone
+- glance (file backend)
+- nova
+- neutron (ovs+vxlan)
+- cinder (lvm+iscsi)
+- tempest (regex: 'smoke')"
+echo "tempest will run if packstack's installation completes successfully."
+echo
+
+if [ -z $COMPUTE_NODE ]; then
+  NODE_FLAGS="--allinone"
+else
+  NODE_FLAGS="--allinone --os-controller-host=$CONTROLLER_NODE --os-network-hosts=$CONTROLLER_NODE --os-compute-hosts=$COMPUTE_NODE"
+fi
+
+$SUDO packstack ${ADDITIONAL_ARGS} \
+          ${NODE_FLAGS} \
+          --cinder-volume-name="aVolume" \
+          --debug \
+          --service-workers=2 \
+          --default-password="packstack" \
+          --os-aodh-install=n \
+          --os-ceilometer-install=n \
+          --os-swift-install=n \
+          --os-manila-install=n \
+          --os-horizon-ssl=n \
+          --amqp-enable-ssl=n \
+          --glance-backend=file \
+          --provision-uec-kernel-url="/tmp/cirros/cirros-$CIRROS_VERSION-$CIRROS_ARCH-vmlinuz" \
+          --provision-uec-ramdisk-url="/tmp/cirros/cirros-$CIRROS_VERSION-$CIRROS_ARCH-initrd" \
+          --provision-uec-disk-url="/tmp/cirros/cirros-$CIRROS_VERSION-$CIRROS_ARCH-disk.img" \
+          --provision-demo=y \
+          --provision-tempest=y \
+          --run-tempest=y \
+          --run-tempest-tests="smoke dashboard" || export FAILURE=true

tools/copy-logs.sh

@@ -27,6 +27,9 @@ PROJECTS_URL="${GIT_URL}/openstack/governance/plain/reference/projects.yaml"
 if [ $(id -u) != 0 ]; then
     SUDO='sudo'
 fi
+
+type -p dnf && export PKG_MGR=dnf || export PKG_MGR=yum
+
 $SUDO mkdir -p "${DIAG_LOGDIR}"
 $SUDO mkdir -p "${CONF_LOGDIR}"
 
@@ -48,14 +51,14 @@ function get_diag_commands {
         'netstat -ntlp'
         'pstree -p'
         'sysctl -a'
-        'yum repolist -v'
+        "$PKG_MGR repolist -v"
         'rpm -qa'
         'journalctl --no-pager'
         'ulimit -n'
     )
 
     echo "Installing required RPM packages..."
-    $SUDO yum -y install coreutils curl file lsof net-tools psmisc
+    $SUDO $PKG_MGR -y install coreutils curl file lsof net-tools psmisc
 
     echo "Running diagnostic commands..."
     for ((i = 0; i < ${#commands[@]}; i++)); do