Support of a new format of projects.yaml

Recently the program definition file was changed in governance repo.
The new format is project oriented, incubation status is removed and
integration status is specified as tag.

Closes bug 1417385

Change-Id: I06d8c27d7afac2660f09175e17173b8e089485e1

etc/default_data.json

@@ -11694,7 +11694,7 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["official-integrated", "official-incubated", "official-other", "governance", "python-openstackclient", "api-wg"]
+            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
         },
         {
             "id": "integrated",
@@ -11702,12 +11702,6 @@
             "child": true,
             "modules": ["official-integrated"]
         },
-        {
-            "id": "incubated",
-            "title": "incubated",
-            "child": true,
-            "modules": ["official-incubated"]
-        },
         {
             "id": "other",
             "title": "other",

etc/stackalytics.conf

@@ -38,8 +38,8 @@
 # Forcibly read default data and update records
 # force_update = False
 
-# The address of file with list of programs
+# The address of file with the official projects list
-# program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml
+# project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
 
 # The address of DriverLog data
 # driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json

etc/test_default_data.json

@@ -183,7 +183,7 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["openstack", "openstack-infra", "openstack-dev"]
+            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
         },
         {
             "id": "integrated",
@@ -191,12 +191,6 @@
             "child": true,
             "modules": ["official-integrated"]
         },
-        {
-            "id": "incubated",
-            "title": "incubated",
-            "child": true,
-            "modules": ["official-incubated"]
-        },
         {
             "id": "other",
             "title": "other",

etc/test_programs.yaml

@@ -1,9 +0,0 @@
-Image Service:
-  codename: Glance
-  ptl: Mark Washenberger (markwash)
-  url: https://wiki.openstack.org/wiki/Glance
-  projects:
-    - repo: openstack/glance
-      integrated-since: havana
-      incubated-since: grizzly
-    - repo: openstack/python-glanceclient

etc/test_projects.yaml

@@ -0,0 +1,16 @@
+Sahara:
+  ptl: Sergey Lukjanov (SergeyLukjanov)
+  mission: >
+    To provide a scalable data processing stack and associated management
+    interfaces.
+  url: https://wiki.openstack.org/wiki/Sahara
+  projects:
+    - repo: openstack/sahara
+      tags:
+        - name: integrated-release
+          since: juno
+    - repo: openstack/python-saharaclient
+    - repo: openstack/sahara-dashboard
+    - repo: openstack/sahara-extra
+    - repo: openstack/sahara-image-elements
+    - repo: openstack/sahara-specs

stackalytics/dashboard/templates/_macros/module_details.html

@@ -28,8 +28,8 @@
         <h2>${name}</h2>
         <div style="font-style: italic; margin-bottom: 0.5em;">
             {%if tag == 'program' %}
-                The official OpenStack program as defined in
+                The official OpenStack project as defined in
-                <a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/programs.yaml" target="_blank">programs.yaml</a>
+                <a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/projects.yaml" target="_blank">projects.yaml</a>
             {%/if%}
             {%if tag == 'group' %}
                 Custom module group as defined in

stackalytics/dashboard/templates/layout.html

@@ -54,12 +54,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
             </div>
 
             <div class="drop">
-                <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
+                <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
                 <input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
             </div>
 
             <div class="drop">
-                <label for="module_selector" title="Module represents a repo (black), official program (violet) or pre-configured group of modules (cyan)">Module</label>
+                <label for="module_selector" title="Module represents a repo (black), official project (violet) or pre-configured group of modules (cyan)">Module</label>
                 <input type="hidden" id="module_selector" style="width: 140px" data-placeholder="Any module"/>
             </div>
 

stackalytics/dashboard/templates/widget.html

@@ -121,7 +121,7 @@
         </div>
 
         <div class="drop" style="margin-right: 15px;">
-            <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
+            <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
             <input type="hidden" id="project_type_selector" style="width: 95px" data-placeholder="Select project type"/>
         </div>
 

stackalytics/processor/config.py

@@ -45,10 +45,10 @@ OPTS = [
                help='SSH username for gerrit review system access'),
     cfg.BoolOpt('force-update', default=False,
                 help='Forcibly read default data and update records'),
-    cfg.StrOpt('program-list-uri',
+    cfg.StrOpt('project-list-uri',
                default=('https://git.openstack.org/cgit/'
-                        'openstack/governance/plain/reference/programs.yaml'),
+                        'openstack/governance/plain/reference/projects.yaml'),
-               help='The address of file with list of programs'),
+               help='The address of file with the official projects list'),
     cfg.StrOpt('driverlog-data-uri',
                default='https://git.openstack.org/cgit/'
                        'stackforge/driverlog/plain/etc/default_data.json',

stackalytics/processor/main.py

@@ -240,30 +240,24 @@ def apply_corrections(uri, runtime_storage_inst):
     runtime_storage_inst.apply_corrections(valid_corrections)
 
 
-def _read_official_programs_yaml(program_list_uri, release_names):
+def _read_official_projects_yaml(project_list_uri, release_names):
-    LOG.debug('Process list of programs from uri: %s', program_list_uri)
+    LOG.debug('Process list of projects from uri: %s', project_list_uri)
-    content = yaml.safe_load(utils.read_uri(program_list_uri))
+    content = yaml.safe_load(utils.read_uri(project_list_uri))
     module_groups = collections.defaultdict(
         lambda: {'modules': [], 'releases': collections.defaultdict(list)})
 
     official_integrated = module_groups['official-integrated']
     official_integrated['tag'] = 'project_type'
     official_integrated['module_group_name'] = 'official-integrated'
-    official_incubated = module_groups['official-incubated']
-    official_incubated['tag'] = 'project_type'
-    official_incubated['module_group_name'] = 'official-incubated'
     official_other = module_groups['official-other']
     official_other['tag'] = 'project_type'
     official_other['module_group_name'] = 'official-other'
 
     for name, info in six.iteritems(content):
-        # for one program
+        # take one official project
-        group_id = name.lower()
-        if 'codename' in info:
-            name = '%s (%s)' % (info['codename'], name)
-            group_id = '%s-group' % info['codename'].lower()
 
-        module_groups[group_id]['module_group_name'] = name
+        group_id = '%s-group' % name.lower()
+        module_groups[group_id]['module_group_name'] = '%s Official' % name
         module_groups[group_id]['tag'] = 'program'
 
         for module in info['projects']:
@@ -271,17 +265,22 @@ def _read_official_programs_yaml(program_list_uri, release_names):
 
             module_groups[group_id]['modules'].append(module_name)
 
-            if ('integrated-since' in module) or ('incubated-since' in module):
+            type_matched = False
-                project_type = 'official-other'
+            if 'tags' in module:
-                for release_name in release_names:
+                for tag in module.get('tags'):
-                    if release_name == module.get('incubated-since'):
+                    tag_name = tag.get('name')
-                        project_type = 'official-incubated'
-                    elif release_name == module.get('integrated-since'):
-                        project_type = 'official-integrated'
 
-                    module_groups[project_type]['releases'][
+                    if tag_name == 'integrated-release':
-                        release_name].append(module_name)
+                        type_matched = True  # project type is matched here
-            else:
+                        project_type = 'official-other'
+                        for release_name in release_names:
+                            if release_name == tag.get('since'):
+                                project_type = 'official-integrated'
+
+                            module_groups[project_type]['releases'][
+                                release_name].append(module_name)
+
+            if not type_matched:
                 module_groups['official-other']['modules'].append(module_name)
 
     # set ids for module groups
@@ -291,13 +290,13 @@ def _read_official_programs_yaml(program_list_uri, release_names):
     return module_groups
 
 
-def process_program_list(runtime_storage_inst, program_list_uri):
+def process_project_list(runtime_storage_inst, project_list_uri):
     module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
     release_names = [r['release_name'].lower()
                      for r in runtime_storage_inst.get_by_key('releases')[1:]]
 
-    official_module_groups = _read_official_programs_yaml(
+    official_module_groups = _read_official_projects_yaml(
-        program_list_uri, release_names)
+        project_list_uri, release_names)
     LOG.debug('Update module groups with official: %s', official_module_groups)
     module_groups.update(official_module_groups)
 
@@ -335,7 +334,7 @@ def main():
                                    default_data,
                                    cfg.CONF.driverlog_data_uri)
 
-    process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
+    process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
 
     update_pids(runtime_storage_inst)