Add py35 gate for ironic-inspector

This patch adds py35 gate for ironic-inspector and
also adds support for inspection in python3 environment
by rolling out configuration of swift in inspector.

Change-Id: I83429a1ba79208245f6c6e1f8b4eb8a16f014868

devstack/plugin.sh

@@ -1,3 +1,8 @@
+#!/usr/bin/env bash
+
+# This package should be tested under python 3, when the job enables Python 3
+enable_python3_package ironic-inspector
+
 IRONIC_INSPECTOR_DEBUG=${IRONIC_INSPECTOR_DEBUG:-True}
 IRONIC_INSPECTOR_DIR=$DEST/ironic-inspector
 IRONIC_INSPECTOR_DATA_DIR=$DATA_DIR/ironic-inspector
@@ -180,7 +185,10 @@ function configure_inspector {
     inspector_iniset firewall dnsmasq_interface $IRONIC_INSPECTOR_INTERFACE
     inspector_iniset database connection `database_connection_url ironic_inspector`
 
-    is_service_enabled swift && configure_inspector_swift
+    # FIXME(ankit) Remove this when swift supports python3
+    if [[ "$USE_PYTHON3" == "False" ]] && is_service_enabled swift; then
+        configure_inspector_swift
+    fi
 
     iniset "$IRONIC_CONF_FILE" inspector enabled True
     iniset "$IRONIC_CONF_FILE" inspector service_url $IRONIC_INSPECTOR_URI

ironic_inspector/pxe_filter/iptables.py

@@ -154,7 +154,8 @@ class IptablesFilter(pxe_filter.BaseFilter):
         try:
             subprocess.check_output(cmd, **kwargs)
         except subprocess.CalledProcessError as exc:
-            output = exc.output.replace('\n', '. ')
+            decoded_output = exc.output.decode("utf-8")
+            output = decoded_output.replace('\n', '. ')
             if ignore:
                 LOG.debug('Ignoring failed iptables %(args)s: %(output)s',
                           {'args': args, 'output': output})

ironic_inspector/test/inspector_tempest_plugin/tests/test_basic.py

@@ -21,7 +21,6 @@ from ironic_inspector.test.inspector_tempest_plugin.tests import manager
 class InspectorBasicTest(manager.InspectorScenarioTest):
 
     def verify_node_introspection_data(self, node):
-        self.assertEqual('yes', node['extra']['rule_success'])
         data = self.introspection_data(node['uuid'])
         self.assertEqual(data['cpu_arch'],
                          self.flavor['properties']['cpu_arch'])
@@ -58,7 +57,7 @@ class InspectorBasicTest(manager.InspectorScenarioTest):
             interval=self.wait_provisioning_state_interval)
 
     @decorators.idempotent_id('03bf7990-bee0-4dd7-bf74-b97ad7b52a4b')
-    @utils.services('compute', 'image', 'network', 'object_storage')
+    @utils.services('compute', 'image', 'network')
     def test_baremetal_introspection(self):
         """This smoke test case follows this set of operations:
 
@@ -93,7 +92,9 @@ class InspectorBasicTest(manager.InspectorScenarioTest):
 
         for node_id in self.node_ids:
             node = self.node_show(node_id)
-            self.verify_node_introspection_data(node)
+            self.assertEqual('yes', node['extra']['rule_success'])
+            if CONF.service_available.swift:
+                self.verify_node_introspection_data(node)
             self.verify_node_flavor(node)
 
         for node_id in self.node_ids:

ironic_inspector/test/inspector_tempest_plugin/tests/test_discovery.py

@@ -143,7 +143,8 @@ class InspectorDiscoveryTest(manager.InspectorScenarioTest):
 
         inspected_node = self.node_show(self.node_info['name'])
         self.verify_node_flavor(inspected_node)
-        self.verify_node_introspection_data(inspected_node)
+        if CONF.service_available.swift:
+            self.verify_node_introspection_data(inspected_node)
         self.verify_node_driver_info(self.node_info, inspected_node)
         self.assertEqual(ProvisionStates.ENROLL,
                          inspected_node['provision_state'])

playbooks/legacy/ironic-inspector-tempest-dsvm-python3/post.yaml

@@ -0,0 +1,15 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs

playbooks/legacy/ironic-inspector-tempest-dsvm-python3/run.yaml

@@ -0,0 +1,143 @@
+- hosts: all
+  name: ironic-inspector-tempest-dsvm-python3
+  tasks:
+
+    - name: Ensure workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_USE_PYTHON3=True
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat << 'EOF' >>"/tmp/dg-local.conf"
+          [[local|localrc]]
+          disable_service s-account
+          disable_service s-container
+          disable_service s-object
+          disable_service s-proxy
+
+          EOF
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+            export DEVSTACK_GATE_TEMPEST_REGEX="Inspector"
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-vars-early
+            # use tempest plugin
+          if [[ "$ZUUL_BRANCH" != "master" ]] ; then
+            # NOTE(jroll) if this is not a patch against master, then
+            # fetch master to install the plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' git+git://git.openstack.org/openstack/ironic'"
+          else
+            # on master, use the local change, so we can pick up any changes to the plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic'"
+          fi
+          export TEMPEST_CONCURRENCY=1
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          cat << 'EOF' >> ironic-extra-vars
+           # use tempest plugin
+          if [[ "$ZUUL_BRANCH" != "master" ]] ; then
+            # NOTE(jroll) if this is not a patch against master, then
+            # fetch master to install the plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' git+git://git.openstack.org/openstack/ironic-inspector'"
+          else
+            # on master, use the local change, so we can pick up any changes to the plugin
+            export DEVSTACK_LOCAL_CONFIG+=$'\n'"TEMPEST_PLUGINS+=' /opt/stack/new/ironic-inspector'"
+          fi
+
+          export IRONIC_INSPECTOR_AUTO_DISCOVERY=1
+          if [ "$IRONIC_INSPECTOR_AUTO_DISCOVERY" == "1" ]; then
+               # discovery test requires sudo for iptables and virsh
+               export DEVSTACK_GATE_REMOVE_STACK_SUDO=0
+               # enable enroll hook
+               export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_NODE_NOT_FOUND_HOOK=enroll"
+               # we are deleting node from ironic for simulate node discovery,
+               # so inspector has to sync cache asap
+               export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_INSPECTOR_CLEAN_UP_PERIOD=5"
+          fi
+
+          EOF
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+          export DEVSTACK_GATE_TEMPEST=1
+          export DEVSTACK_GATE_IRONIC=1
+          export DEVSTACK_GATE_IRONIC_INSPECTOR=1
+          export DEVSTACK_GATE_NEUTRON=1
+          export DEVSTACK_GATE_VIRT_DRIVER=ironic
+          export DEVSTACK_GATE_CONFIGDRIVE=1
+          export BRANCH_OVERRIDE=default
+          if [ "$BRANCH_OVERRIDE" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          export PROJECTS="openstack/ironic openstack/ironic-inspector $PROJECTS"
+          export DEVSTACK_LOCAL_CONFIG="enable_plugin ironic git://git.openstack.org/openstack/ironic"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ironic-inspector https://git.openstack.org/openstack/ironic-inspector"
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_AUTOMATED_CLEAN_ENABLED=False"
+
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_BUILD_DEPLOY_RAMDISK=False"
+          # IPA requires at least 1 GiB of RAM
+          export DEVSTACK_LOCAL_CONFIG+=$'\n'"IRONIC_VM_SPECS_RAM=1024"$'\n'"IRONIC_VM_COUNT=1"
+
+          # Ensure the ironic-vars-EARLY file exists
+          touch ironic-vars-early
+          # Pull in the EARLY variables injected by the optional builders
+          source ironic-vars-early
+
+          # Ensure the ironic-EXTRA-vars file exists
+          touch ironic-extra-vars
+          # Pull in the EXTRA variables injected by the optional builders
+          source ironic-extra-vars
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'

zuul.d/legacy-ironic-inspector-jobs.yaml

@@ -59,3 +59,10 @@
     run: playbooks/legacy/ironic-inspector-tempest-dsvm-discovery/run.yaml
     post-run: playbooks/legacy/ironic-inspector-tempest-dsvm-discovery/post.yaml
     timeout: 10800
+
+- job:
+    name: ironic-inspector-tempest-dsvm-python3
+    parent: ironic-inspector-dsvm-base
+    run: playbooks/legacy/ironic-inspector-tempest-dsvm-python3/run
+    post-run: playbooks/legacy/ironic-inspector-tempest-dsvm-python3/post
+    timeout: 10800

zuul.d/project.yaml

@@ -7,6 +7,8 @@
         - ironic-inspector-tox-func
         - ironic-inspector-tox-func3
         - ironic-tempest-dsvm-ironic-inspector
+        - ironic-inspector-tempest-dsvm-python3:
+            voting: false
     gate:
       jobs:
         - ironic-inspector-grenade-dsvm