ignore unofficial projects in latest.yaml

Update the template generator to ignore unofficial projects when
rendering for the current series. Ignoring, rather than throwing an
error, allows us to clean up the data without the build breaking.

To make the validation work for glance-store, a new field is needed to
set the 'deliverable-name', since it is different from the project
name used to build URLs.

Change-Id: I0876c93b49d57818d0b689fd03a0171a96e27547
Signed-off-by: Doug Hellmann <doug@doughellmann.com>

doc/doc-contrib-guide/source/doc-tools/template-generator.rst

@@ -194,6 +194,11 @@ define the name, service, and type properties.
 
 The ``name`` should be the base name of a git repository.
 
+The ``deliverable-name`` should be the name of the deliverable as
+defined in ``openstack/governance/reference/projects.yaml``. This
+value only needs to be set if the deliverable name does not match the
+project name (such as ``glance_store`` and ``glance-store``).
+
 The ``service`` string should be taken from the governance repository
 definition of the project.
 

tools/www-generator.py

@@ -185,7 +185,8 @@ _URLS = [
 def load_project_data(source_directory,
                       check_all_links=False,
                       skip_links=False,
-                      series_to_load=None):
+                      series_to_load=None,
+                      governed_deliverables=[]):
     "Return a dict with project data grouped by series."
     logger = logging.getLogger()
     series_to_load = series_to_load or []
@@ -215,13 +216,22 @@ def load_project_data(source_directory,
 
         logger.info('loading %s project data from %s', series, filename)
         with open(filename, 'r') as f:
-            data = yaml.safe_load(f.read())
-        for error in validator.iter_errors(data):
+            raw_data = yaml.safe_load(f.read())
+        for error in validator.iter_errors(raw_data):
             logger.error(str(error))
             fail = True
 
         links_to_check = []
-        for project in data:
+        data = []
+        for project in raw_data:
+            deliverable_name = project.get('deliverable-name', project['name'])
+            if (series == 'latest' and
+                    deliverable_name not in governed_deliverables):
+                logger.warning(
+                    ('%s is no longer part of an official project, '
+                     'ignoring in %s'),
+                    deliverable_name, filename)
+                continue
             # If the project has a service-type set, ensure it matches
             # the value in the service-type-authority data.base.
             st = project.get('service_type')
@@ -340,8 +350,10 @@ def _get_official_repos():
     seen_repos = set()
     regular_repos = []
     infra_repos = []
+    deliverables = []
     for t_name, team in data.items():
         for d_name, d_data in team.get('deliverables', {}).items():
+            deliverables.append(d_name)
             if t_name == 'Infrastructure':
                 add = infra_repos.append
             else:
@@ -359,7 +371,7 @@ def _get_official_repos():
                                           'base': repo.rsplit('/')[-1]})
                 elif repo not in _IGNORED_REPOS:
                     add({'name': repo, 'base': repo.rsplit('/')[-1]})
-    return (regular_repos, infra_repos)
+    return (regular_repos, infra_repos, deliverables)
 
 
 def render_template(environment, project_data, regular_repos, infra_repos,
@@ -445,13 +457,14 @@ def main():
     args = parse_command_line_arguments()
     logger = initialize_logging(args.debug, args.verbose)
 
+    regular_repos, infra_repos, deliverables = _get_official_repos()
     project_data = load_project_data(
-        args.source_directory,
-        args.check_all_links,
-        args.skip_links,
-        args.series,
+        source_directory=args.source_directory,
+        check_all_links=args.check_all_links,
+        skip_links=args.skip_links,
+        series_to_load=args.series,
+        governed_deliverables=deliverables,
     )
-    regular_repos, infra_repos = _get_official_repos()
 
     # Set up jinja to discover the templates.
     try:

www/project-data/latest.yaml

@@ -39,6 +39,7 @@
   type: service-client
   description: glance client
 - name: glance_store
+  deliverable-name: glance-store
   service: glance backend abstraction library
   type: library
   has_user_guide: true

www/project-data/schema.yaml

@@ -10,6 +10,8 @@ items:
   properties:
     name:
       type: string
+    deliverable-name:
+      type: string
     service:
       type: string
     service_type: