Add devstack Zuul job

Change-Id: I7121000356dfc992fc4ebe4eb5fa91b4a68faaa4

playbooks/devstack.yaml

@@ -12,8 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-#TODO(fressi): fill this playbook
-
 ---
 
-
+# Changes that run through tobiko-devstack are likely to have an impact on
+# the devstack part of the job, so we keep devstack in the main play to
+# avoid zuul retrying on legitimate failures.
+- hosts: all
+  roles:
+    - orchestrate-devstack

playbooks/post-tobiko.yaml

@@ -12,8 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-#TODO(fressi): fill this playbook
-
 ---
 
-
+- hosts: tempest
+  become: true
+  roles:
+    - role: fetch-subunit-output
+      zuul_work_dir: '{{ devstack_base_dir }}/tempest'
+    - role: process-stackviz

roles/process-stackviz/README.rst

@@ -0,0 +1,22 @@
+Generate stackviz report.
+
+Generate stackviz report using subunit and dstat data, using
+the stackviz archive embedded in test images.
+
+**Role Variables**
+
+.. zuul:rolevar:: devstack_base_dir
+   :default: /opt/stack
+
+   The devstack base directory.
+
+.. zuul:rolevar:: stage_dir
+   :default: "{{ ansible_user_dir }}"
+
+   The stage directory where the input data can be found and
+   the output will be produced.
+
+.. zuul:rolevar:: zuul_work_dir
+   :default: {{ devstack_base_dir }}/tempest
+
+   Directory to work in. It has to be a fully qualified path.

roles/process-stackviz/defaults/main.yaml

@@ -0,0 +1,3 @@
+devstack_base_dir: /opt/stack
+stage_dir: "{{ ansible_user_dir }}"
+zuul_work_dir: "{{ devstack_base_dir }}/tempest"

roles/process-stackviz/tasks/main.yaml

@@ -0,0 +1,65 @@
+- name: Check if stackviz archive exists
+  stat:
+    path: "/opt/cache/files/stackviz-latest.tar.gz"
+  register: stackviz_archive
+
+- debug:
+    msg: "Stackviz archive could not be found in /opt/cache/files/stackviz-latest.tar.gz"
+  when: not stackviz_archive.stat.exists
+
+- name: Check if subunit data exists
+  stat:
+    path: "{{ zuul_work_dir }}/testrepository.subunit"
+  register: subunit_input
+
+- debug:
+    msg: "Subunit file could not be found at {{ zuul_work_dir }}/testrepository.subunit"
+  when: not subunit_input.stat.exists
+
+- name: Install stackviz
+  pip:
+    name: "file://{{ stackviz_archive.stat.path }}"
+    virtualenv: /tmp/stackviz
+    extra_args: -U
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Deploy stackviz static html+js
+  command: cp -pR /tmp/stackviz/share/stackviz-html {{ stage_dir }}/stackviz
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Check if dstat data exists
+  stat:
+    path: "{{ devstack_base_dir }}/logs/dstat-csv.log"
+  register: dstat_input
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+
+- name: Run stackviz with dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --dstat "{{ devstack_base_dir }}/logs/dstat-csv.log" \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - dstat_input.stat.exists
+  failed_when: False
+
+- name: Run stackviz without dstat
+  shell: |
+    cat {{ subunit_input.stat.path }} | \
+      /tmp/stackviz/bin/stackviz-export \
+        --env --stdin \
+        {{ stage_dir }}/stackviz/data
+  when:
+    - stackviz_archive.stat.exists
+    - subunit_input.stat.exists
+    - not dstat_input.stat.exists
+  failed_when: False

zuul.d/devstack.yaml

@@ -0,0 +1,57 @@
+- job:
+    name: tobiko-devstack
+    parent: devstack
+    abstract: true
+    nodeset: openstack-single-node
+    description: |
+      Base Tobiko devstack job.
+
+      This job provides the base for both the single and multi-node
+      test setup. To run a multi-node test inherit from tobiko-devstack and
+      set the nodeset to a multi-node one.
+    required-projects:
+      - git.openstack.org/openstack/tobiko
+    timeout: 7200
+    roles:
+      - zuul: git.openstack.org/openstack-dev/devstack
+    vars:
+      devstack_services:
+        tempest: false
+      devstack_local_conf:
+        test-config:
+          $TEMPEST_CONFIG:
+            compute:
+              min_compute_nodes: "{{ groups['compute'] | default(['controller']) | length }}"
+      test_results_stage_name: test_results
+      zuul_copy_output:
+        '{{ devstack_base_dir }}/tempest/etc/tempest.conf': logs
+        '{{ devstack_base_dir }}/tempest/etc/accounts.yaml': logs
+        '{{ devstack_base_dir }}/tempest/tempest.log': logs
+        '{{ stage_dir }}/{{ test_results_stage_name }}.subunit': logs
+        '{{ stage_dir }}/{{ test_results_stage_name }}.html': logs
+        '{{ stage_dir }}/stackviz': logs
+      extensions_to_txt:
+        conf: true
+        log: true
+        yaml: true
+        yml: true
+    run: playbooks/devstack.yaml
+    # post-run: playbooks/post-tobiko.yaml
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^api-ref/.*$
+      - ^doc/.*$
+      - ^etc/.*$
+      - ^releasenotes/.*$
+
+
+- job:
+    name: tobiko-devstack-neutron
+    parent: tobiko-devstack
+    description: |
+      Integration test that runs all tests against DevStack provided cloud
+    vars:
+      tox_envlist: neutron
+      # tempest_test_regex: tempest
+      devstack_localrc:
+        ENABLE_FILE_INJECTION: true

zuul.d/jobs.yaml

@@ -1,34 +0,0 @@
-- job:
-    name: tobiko-tempest
-    parent: devstack-tempest
-    timeout: 7800
-    required-projects:
-      - openstack/tobiko
-    irrelevant-files:
-      - ^.*\.rst$
-      - ^api-ref/.*$
-      - ^doc/.*$
-      - ^etc/.*$
-      - ^releasenotes/.*$
-    vars:
-      devstack_localrc:
-        TEMPEST_PLUGINS: "'{{ ansible_user_dir }}/src/git.openstack.org/openstack/tobiko'"
-      devstack_services:
-        c-bak: false
-        ceilometer-acentral: false
-        ceilometer-acompute: false
-        ceilometer-alarm-evaluator: false
-        ceilometer-alarm-notifier: false
-        ceilometer-anotification: false
-        ceilometer-api: false
-        ceilometer-collector: false
-        c-sch: false
-        c-api: false
-        c-vol: false
-        cinder: false
-        swift: false
-        s-account: false
-        s-container: false
-        s-object: false
-        s-proxy: false
-        tempest: true

zuul.d/project.yaml

@@ -10,6 +10,7 @@
         - openstack-tox-py35
         - openstack-tox-py36
         - openstack-tox-py27
+        - tobiko-devstack-neutron
     gate:
       jobs:
         - openstack-tox-pep8
@@ -17,3 +18,4 @@
         - openstack-tox-py35
         - openstack-tox-py36
         - openstack-tox-py27
+        - tobiko-devstack-neutron