Use the openstack-python-jobs-neutron templates

For the neutron stadium projects, they all need neutron. Use the
template that defines variants adding this.

Also, added the script I used to do this for reference for folks.

Change-Id: I8b622159074bb34324b28a58431e399f0c29508e
Depends-On: I2d79907e67895e6edb832cddce1ae2e3342ddaff

tools/fix-neutron-stadium.py

@@ -0,0 +1,189 @@
+#!/usr/bin/env python
+
+# Copyright 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import collections
+import subprocess
+
+import ruamel.yaml
+import yaml
+
+
+# from :
+# http://stackoverflow.com/questions/8640959/how-can-i-control-what-scalar-form-pyyaml-uses-for-my-data  flake8: noqa
+def should_use_block(value):
+    for c in u"\u000a\u000d\u001c\u001d\u001e\u0085\u2028\u2029":
+        if c in value:
+            return True
+    return False
+
+
+def my_represent_scalar(self, tag, value, style=None):
+    if style is None:
+        if should_use_block(value):
+             style='|'
+        else:
+            style = self.default_style
+
+    node = yaml.representer.ScalarNode(tag, value, style=style)
+    if self.alias_key is not None:
+        self.represented_objects[self.alias_key] = node
+    return node
+
+def project_representer(dumper, data):
+    return dumper.represent_mapping('tag:yaml.org,2002:map',
+                                    data.items())
+
+
+def construct_yaml_map(self, node):
+    data = collections.OrderedDict()
+    yield data
+    value = self.construct_mapping(node)
+
+    if isinstance(node, yaml.MappingNode):
+        self.flatten_mapping(node)
+    else:
+        raise yaml.constructor.ConstructorError(
+            None, None,
+            'expected a mapping node, but found %s' % node.id,
+            node.start_mark)
+
+    mapping = collections.OrderedDict()
+    for key_node, value_node in node.value:
+        key = self.construct_object(key_node, deep=False)
+        try:
+            hash(key)
+        except TypeError as exc:
+            raise yaml.constructor.ConstructorError(
+                'while constructing a mapping', node.start_mark,
+                'found unacceptable key (%s)' % exc, key_node.start_mark)
+        value = self.construct_object(value_node, deep=False)
+        mapping[key] = value
+    data.update(mapping)
+
+
+class IndentedEmitter(yaml.emitter.Emitter):
+    def expect_block_sequence(self):
+        self.increase_indent(flow=False, indentless=False)
+        self.state = self.expect_first_block_sequence_item
+
+
+class IndentedDumper(IndentedEmitter, yaml.serializer.Serializer,
+                     yaml.representer.Representer, yaml.resolver.Resolver):
+    def __init__(self, stream,
+                 default_style=None, default_flow_style=None,
+                 canonical=None, indent=None, width=None,
+                 allow_unicode=None, line_break=None,
+                 encoding=None, explicit_start=None, explicit_end=None,
+                 version=None, tags=None):
+        IndentedEmitter.__init__(
+            self, stream, canonical=canonical,
+            indent=indent, width=width,
+            allow_unicode=allow_unicode,
+            line_break=line_break)
+        yaml.serializer.Serializer.__init__(
+            self, encoding=encoding,
+            explicit_start=explicit_start,
+            explicit_end=explicit_end,
+            version=version, tags=tags)
+        yaml.representer.Representer.__init__(
+            self, default_style=default_style,
+            default_flow_style=default_flow_style)
+        yaml.resolver.Resolver.__init__(self)
+
+
+def ordered_load(stream, *args, **kwargs):
+    yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                         construct_yaml_map)
+
+    return yaml.load(stream=stream, *args, **kwargs)
+
+
+def ordered_dump(data, stream=None, *args, **kwargs):
+    dumper = IndentedDumper
+    # We need to do this because of how template expasion into a project
+    # works. Without it, we end up with YAML references to the expanded jobs.
+    dumper.ignore_aliases = lambda self, data: True
+    yaml.add_representer(collections.OrderedDict, project_representer,
+                         Dumper=IndentedDumper)
+
+    output = yaml.dump(
+        data, default_flow_style=False,
+        Dumper=dumper, width=80, *args, **kwargs).replace(
+            '\n-', '\n\n-')
+    if stream:
+        stream.write(output)
+    else:
+        return output
+
+
+def get_single_key(var):
+    if isinstance(var, str):
+        return var
+    elif isinstance(var, list):
+        return var[0]
+    return list(var.keys())[0]
+
+
+def has_single_key(var):
+    if isinstance(var, list):
+        return len(var) == 1
+    if isinstance(var, str):
+        return True
+    dict_keys = list(var.keys())
+    if len(dict_keys) != 1:
+        return False
+    if var[get_single_key(var)]:
+        return False
+    return True
+
+
+
+def main():
+    subprocess.run(['git', 'checkout', '--', 'zuul.d/projects.yaml'])
+    yaml = ruamel.yaml.YAML()
+    yaml.indent(mapping=2, sequence=4, offset=2)
+    projects = yaml.load(open('zuul.d/projects.yaml', 'r'))
+
+    for project in projects:
+        if project['project']['name'].split('/')[1].startswith('networking-'):
+            if 'templates' not in project['project']:
+                continue
+            templates = project['project']['templates']
+            for template in ('openstack-python-jobs',
+                             'openstack-python35-jobs'):
+                if template in templates:
+                    new_name = template + '-neutron'
+                    templates[templates.index(template)] = new_name
+
+    yaml.dump(projects, open('zuul.d/projects.yaml', 'w'))
+
+    # Strip the extra 2 spaces that ruamel.yaml appends because we told it
+    # to indent an extra 2 spaces. Because the top level entry is a list it
+    # applies that indentation at the top. It doesn't indent the comment lines
+    # extra though, so don't do them.
+    with open('zuul.d/projects.yaml', 'r') as main_in:
+        main_content = main_in.readlines()
+    with open('zuul.d/projects.yaml', 'w') as main_out:
+        for line in main_content:
+            if '#' in line:
+                main_out.write(line)
+            else:
+                if line.startswith('  - project'):
+                    main_out.write('\n')
+                main_out.write(line[2:])
+
+if __name__ == '__main__':
+    main()

zuul.d/projects.yaml

@@ -16015,21 +16015,21 @@
     name: openstack/networking-6wind
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
 
 - project:
     name: openstack/networking-ale-omniswitch
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-arista
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16043,8 +16043,8 @@
     name: openstack/networking-bagpipe
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - publish-openstack-sphinx-docs
       - release-notes-jobs
       - check-requirements
@@ -16086,8 +16086,8 @@
     name: openstack/networking-baremetal
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - publish-openstack-sphinx-docs
       - check-requirements
       - release-notes-jobs
@@ -16097,8 +16097,8 @@
     name: openstack/networking-bgpvpn
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - publish-openstack-sphinx-docs
       - release-notes-jobs
       - periodic-newton
@@ -16135,7 +16135,7 @@
     name: openstack/networking-bigswitch
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - openstack-python-jobs-trusty
       - publish-to-pypi
 
@@ -16143,14 +16143,14 @@
     name: openstack/networking-brocade
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
 
 - project:
     name: openstack/networking-calico
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
     check:
@@ -16176,8 +16176,8 @@
     name: openstack/networking-cisco
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - docs-on-readthedocs
       - publish-to-pypi
     check:
@@ -16199,8 +16199,8 @@
     name: openstack/networking-cumulus
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16208,8 +16208,8 @@
     name: openstack/networking-dpm
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - docs-on-readthedocs
       - publish-to-pypi
@@ -16218,13 +16218,13 @@
     name: openstack/networking-edge-vpn
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
 
 - project:
     name: openstack/networking-extreme
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
     check:
       jobs:
         - legacy-requirements:
@@ -16244,8 +16244,8 @@
     name: openstack/networking-fortinet
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16253,8 +16253,8 @@
     name: openstack/networking-fujitsu
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16262,8 +16262,8 @@
     name: openstack/networking-generic-switch
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
     check:
@@ -16300,7 +16300,7 @@
     name: openstack/networking-gluon
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
     check:
       jobs:
         - legacy-requirements:
@@ -16320,8 +16320,8 @@
     name: openstack/networking-h3c
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16329,8 +16329,8 @@
     name: openstack/networking-hpe
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16338,8 +16338,8 @@
     name: openstack/networking-huawei
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16348,8 +16348,8 @@
     templates:
       - merge-check
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - release-notes-jobs
       - publish-to-pypi
 
@@ -16357,14 +16357,14 @@
     name: openstack/networking-icc
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-infoblox
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
@@ -16372,8 +16372,8 @@
     templates:
       - merge-check
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - tripleo-scenario-jobs-scenario004
       - publish-to-pypi
     check:
@@ -16389,7 +16389,7 @@
     name: openstack/networking-lagopus
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - check-requirements
       - publish-to-pypi
 
@@ -16397,7 +16397,7 @@
     name: openstack/networking-lenovo
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
@@ -16405,8 +16405,8 @@
     templates:
       - merge-check
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - periodic-newton
       - periodic-ocata
       - periodic-pike
@@ -16627,21 +16627,21 @@
     name: openstack/networking-mlnx
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
 
 - project:
     name: openstack/networking-nec
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
 
 - project:
     name: openstack/networking-odl
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - periodic-newton
       - periodic-ocata
       - periodic-pike
@@ -16789,7 +16789,7 @@
     templates:
       - check-requirements
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
     check:
       jobs:
@@ -16804,19 +16804,19 @@
     templates:
       - check-requirements
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-ovn
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-openstack-sphinx-docs
       - release-notes-jobs
       - translation-jobs
       - check-requirements
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - periodic-jobs-with-neutron-lib-master
       - periodic-jobs-with-ovsdbapp-master
       - tripleo-scenario-jobs-scenario007-nv
@@ -16949,7 +16949,7 @@
     name: openstack/networking-ovs-dpdk
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
     experimental:
       jobs:
         - legacy-tempest-dsvm-networking-ovs-dpdk:
@@ -16962,14 +16962,14 @@
     name: openstack/networking-peregrine
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-plumgrid
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
@@ -16977,8 +16977,8 @@
     templates:
       - merge-check
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - docs-on-readthedocs
       - publish-to-pypi
 
@@ -16986,10 +16986,10 @@
     name: openstack/networking-sfc
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-openstack-sphinx-docs
       - check-requirements
-      - openstack-python35-jobs
+      - openstack-python35-jobs-neutron
       - periodic-jobs-with-neutron-lib-master
       - release-notes-jobs
       - publish-to-pypi
@@ -17023,7 +17023,7 @@
     name: openstack/networking-vpp
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
     check:
       jobs:
         - legacy-networking-vpp-smoke:
@@ -17034,21 +17034,21 @@
     templates:
       - merge-check
       - check-requirements
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-zte
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
       - publish-to-pypi
 
 - project:
     name: openstack/networking-zvm
     templates:
       - merge-check
-      - openstack-python-jobs
+      - openstack-python-jobs-neutron
 
 - project:
     name: openstack/neutron