Create upstream CI faults jobs

Reorganize jobs:

  - new faults jobs that execute scenario+faults+scenario tests
  - rename some job names

Change-Id: Ic3493edae66df736ff29e3c3e82774e2649cef9d

tox.ini

@@ -133,15 +133,18 @@ setenv =
     OS_TEST_PATH={toxinidir}/tobiko/tests/scenario/neutron
 
 
-[testenv:os-faults]
+[testenv:faults]
 
 envdir = {toxworkdir}/scenario
 deps = {[testenv:scenario]deps}
 passenv = {[testenv:scenario]passenv}
 setenv =
     {[testenv:scenario]setenv}
-    OS_TEST_PATH={toxinidir}/tobiko/tests/os_faults
-commands = stestr run --serial {posargs}
+    TOBIKO_PREVENT_CREATE=true
+commands =
+    stestr run --serial \
+        --test-path '{toxinidir}/tobiko/tests/faults' {posargs}
+    stestr run --combine {posargs}
 
 
 [testenv:venv]

zuul.d/project.yaml

@@ -18,17 +18,15 @@
 
         - tobiko-devstack-functional
         - tobiko-devstack-scenario
-        - tobiko-devstack-scenario-centos-7
-        - tobiko-devstack-scenario-centos-7-queens
-        - tobiko-devstack-scenario-centos-7-rocky
-        - tobiko-devstack-scenario-centos-7-stein
-        - tobiko-devstack-scenario-centos-7-train
-        - tobiko-devstack-scenario-ubuntu-bionic
+        - tobiko-devstack-faults
+        - tobiko-devstack-faults-centos-7
+        - tobiko-devstack-faults-centos-7-queens
+        - tobiko-devstack-faults-ubuntu-bionic
 
         - tobiko-tripleo-functional
         - tobiko-tripleo-scenario
-        - tobiko-tripleo-upgrade-scenario
-        - tobiko-tripleo-faults-scenario
+        - tobiko-tripleo-faults
+        - tobiko-tripleo-upgrade
 
     gate:
       jobs:

zuul.d/tobiko-devstack.yaml

@@ -144,6 +144,7 @@
       - ^infrared/
       - ^releasenotes/
       - ^report/
+      - ^tobiko/tests/faults/
       - ^tobiko/tests/scenario/
       - ^tobiko/tests/unit/
 
@@ -162,6 +163,7 @@
       - ^infrared/
       - ^releasenotes/
       - ^report/
+      - ^tobiko/tests/faults/
       - ^tobiko/tests/functional/
       - ^tobiko/tests/unit/
 
@@ -203,8 +205,26 @@
 
 
 - job:
-    name: tobiko-devstack-scenario-centos-7
+    name: tobiko-devstack-faults
+    voting: false
     parent: tobiko-devstack-scenario-multinode
+    description: |
+      Base Tobiko devstack job to execute scenario+faults+scenario test cases.
+    vars:
+      tox_envlist: scenario,faults
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/
+      - ^infrared/
+      - ^releasenotes/
+      - ^report/
+      - ^tobiko/tests/functional/
+      - ^tobiko/tests/unit/
+
+
+- job:
+    name: tobiko-devstack-faults-centos-7
+    parent: tobiko-devstack-faults
     nodeset: openstack-three-node-centos-7
     voting: false
     vars:
@@ -214,52 +234,17 @@
 
 
 - job:
-    name: tobiko-devstack-scenario-centos-7-queens
-    parent: tobiko-devstack-scenario-centos-7
+    name: tobiko-devstack-faults-centos-7-queens
+    parent: tobiko-devstack-faults-centos-7
     override-checkout: stable/queens
     voting: false
 
 
 - job:
-    name: tobiko-devstack-scenario-centos-7-rocky
-    parent: tobiko-devstack-scenario-centos-7
-    override-checkout: stable/rocky
-    voting: false
-
-
-- job:
-    name: tobiko-devstack-scenario-centos-7-stein
-    parent: tobiko-devstack-scenario-centos-7
-    override-checkout: stable/stein
-    voting: false
-
-
-- job:
-    name: tobiko-devstack-scenario-centos-7-train
-    parent: tobiko-devstack-scenario-centos-7
-    override-checkout: stable/train
-    voting: false
-
-
-- job:
-    name: tobiko-devstack-scenario-fedora-latest
-    parent: tobiko-devstack-scenario-multinode
-    nodeset: openstack-three-node-fedora-latest
-    voting: false
-    vars:
-      devstack_localrc:
-        # Add dns-domain-ports,dns-integration extensions again when bug
-        # https://bugzilla.redhat.com/show_bug.cgi?id=1695647
-        # will be fixed for Fedora
-        NETWORK_API_EXTENSIONS: address-scope,agent,allowed-address-pairs,auto-allocated-topology,availability_zone,binding,default-subnetpools,dhcp_agent_scheduler,dvr,empty-string-filtering,ext-gw-mode,external-net,extra_dhcp_opt,extraroute,filter-validation,fip-port-details,flavors,ip-substring-filtering,l3-flavors,l3-ha,l3_agent_scheduler,logging,metering,multi-provider,net-mtu,net-mtu-writable,network-ip-availability,network_availability_zone,pagination,port-security,project-id,provider,qos,qos-bw-minimum-ingress,qos-fip,quotas,quota_details,rbac-policies,router,router_availability_zone,security-group,port-mac-address-regenerate,port-security-groups-filtering,segment,service-type,sorting,standard-attr-description,standard-attr-revisions,standard-attr-segment,standard-attr-timestamp,standard-attr-tag,subnet_allocation,trunk,trunk-details,uplink-status-propagation
-      devstack_services:
-        q-dns: false
-
-
-- job:
-    name: tobiko-devstack-scenario-ubuntu-bionic
-    parent: tobiko-devstack-scenario-multinode
+    name: tobiko-devstack-faults-ubuntu-bionic
+    parent: tobiko-devstack-faults
     nodeset: openstack-three-node-bionic
+    voting: false
     vars:
       devstack_services:
         tls-proxy: true

zuul.d/tobiko-tripleo.yaml

@@ -23,6 +23,7 @@
         - multinode-standalone.yml
         - tobiko.yml
 
+
 - job:
     name: tobiko-tripleo-functional
     description: Functional testing Tobiko TripleO job
@@ -36,13 +37,13 @@
     name: tobiko-tripleo-scenario
     description: Scenario testing Tobiko TripleO job
     parent: tobiko-tripleo
-    voting: false
+    voting: true
     vars:
       tox_envlist: scenario
 
 
 - job:
-    name: tobiko-tripleo-upgrade-scenario
+    name: tobiko-tripleo-upgrade
     description: Scenario testing Tobiko TripleO upgrade job
     parent: tobiko-tripleo-scenario
     voting: false
@@ -56,9 +57,9 @@
 
 
 - job:
-    name: tobiko-tripleo-faults-scenario
+    name: tobiko-tripleo-faults
     description: OS faults and scenario testing Tobiko TripleO job
     parent: tobiko-tripleo
     voting: false
     vars:
-      tox_envlist: os-faults,scenario
+      tox_envlist: scenario,faults