Introduce the "deliverables" concept

Our current projects.yaml file lists project teams and git
repositories associated to those. However, things we publish
as a single "thing" may be represented by multiple code
repositories. For example, a "neutron" release is actually made
of openstack/neutron and openstack/neutron-*aas. A "sahara"
release is actually made of openstack/sahara, openstack/sahara-extra
and openstack/sahara-image-elements. Those are all tagged at the
same time with the same version number, and published together
as a single "deliverable".

This change proposes to encode this layer in the projects.yaml file.
It also proposes to apply tags to the deliverable level rather than
at git repository level. Tags are meant to apply to things that are
consumed by our users, not to the technical pieces that help us
build those things. Note that for most projects have single-repo
deliverables and won't really look different. See Sahara and Neutron
in the proposed file to see examples of multi-repo deliverables.

Since this introduces a breaking change in the file format, we also
take the opportunity to simplify the YAML format: since tags no
longer have attributes they can be listed as an array rather than
as a dictionary, sparing us the useless "name:" part.

Change-Id: I2aa729d1b4278743a5e99b41178dc2d11b3e1348

doc/source/_exts/tags.py

@@ -52,13 +52,13 @@ class TaggedProjectsDirective(rst.Directive):
                 source_name,
             )
         else:
-            for team_name, repo in sorted(project_data):
-                if repo is None:
+            for team_name, deliverable in sorted(project_data):
+                if deliverable is None:
                     line = '- :ref:`project-%s`' % projects.slugify(team_name)
                 else:
-                    line = '- :ref:`project-%s` -- :repo:`%s`' % (
+                    line = '- %s (:ref:`project-%s`)' % (
+                        deliverable,
                         projects.slugify(team_name),
-                        repo,
                     )
                 result.append(line, source_name)
 
@@ -73,14 +73,12 @@ class TaggedProjectsDirective(rst.Directive):
 def _build_projects_by_tag():
     for proj_name, info in projects.get_project_data().items():
         for tag in info.get('tags', []):
-            tn = tag['name']
-            l = _projects_by_tag.setdefault(tn, [])
+            l = _projects_by_tag.setdefault(tag, [])
             l.append((proj_name, None))
-        for repo in info.get('projects', []):
-            for tag in repo.get('tags', []):
-                tn = tag['name']
-                l = _projects_by_tag.setdefault(tn, [])
-                l.append((proj_name, repo['repo']))
+        for name, deliverable in info.get('deliverables', {}).items():
+            for tag in deliverable.get('tags', []):
+                l = _projects_by_tag.setdefault(tag, [])
+                l.append((proj_name, name))
 
 
 def setup(app):

doc/source/_exts/teams.py

@@ -48,29 +48,33 @@ def _team_to_rst(name, info):
         yield ''
         yield mission
         yield ''
-    tags = [
-        ':ref:`tag-%s`' % t['name']
-        for t in info.get('tags', [])
-    ]
+    tags = info.get('tags', [])
     if tags:
         yield 'Team based tags'
         yield '----------------'
         yield ''
         for tag in tags:
-            yield '- %s' % tag
+            yield '- :ref:`tag-%s`' % tag
         yield ''
-    yield 'Repositories and Tags'
+    yield 'Deliverables and Tags'
     yield '---------------------'
     yield ''
-    project_repos = info.get('projects', [])
-    if project_repos:
-        for project in project_repos:
-            yield '- :repo:`%s`' % project['repo']
-            tags = project.get('tags', [])
+    deliverables = info.get('deliverables', [])
+    if deliverables:
+        for name, deliverable in deliverables.items():
+            title = '- %s' %  name
+            repos = deliverable.get('repos', [])
+            if repos:
+                repolist = " ("
+                for repo in repos:
+                    repolist += ':repo:`%s`, ' % repo
+                title += repolist[0:-2] + ")"
+            yield title
+            tags = deliverable.get('tags', [])
             if tags is not []:
                 yield ''
             for tag in tags:
-                yield '  - :ref:`tag-%s`' % tag['name']
+                yield '  - :ref:`tag-%s`' % tag
     else:
         yield 'None'
     yield ''

tools/validate_tags.py

@@ -50,18 +50,18 @@ def main():
         # Check team based tags
         for validator in team_validators:
             validate(team, teams[team], validator)
-        # Check repo based tags
-        for repo in teams[team]['projects']:
-            repo_name = repo['repo']
-            if not repo_exists(repo_name):
-                continue
-            for validator in repo_validators:
-                validate(repo_name, repo, validator)
+        # Check deliverable based tags
+        for name, deliverable in teams[team]['deliverables'].items():
+            for repo in deliverable['repos']:
+                if not repo_exists(repo):
+                    continue
+                for validator in repo_validators:
+                    validate(repo, deliverable, validator)
 
 
 def validate(name, data, validator):
     tag_name = validator.get_tag_name()
-    contains_tag = any([tag_name == tag['name'] for tag in
+    contains_tag = any([tag_name == tag for tag in
                         data.get('tags', [])])
     if validator.validate(name):
         # should contain tag