Merge "Support of a new format of projects.yaml"

etc/default_data.json

@@ -11618,7 +11618,7 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["official-integrated", "official-incubated", "official-other", "governance", "python-openstackclient", "api-wg"]
+            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
         },
         {
             "id": "integrated",
@@ -11626,12 +11626,6 @@
             "child": true,
             "modules": ["official-integrated"]
         },
-        {
-            "id": "incubated",
-            "title": "incubated",
-            "child": true,
-            "modules": ["official-incubated"]
-        },
         {
             "id": "other",
             "title": "other",

etc/stackalytics.conf

@@ -38,8 +38,8 @@
 # Forcibly read default data and update records
 # force_update = False
 
-# The address of file with list of programs
+# The address of file with the official projects list
-# program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml
+# project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml
 
 # The address of DriverLog data
 # driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json

etc/test_default_data.json

@@ -183,7 +183,7 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["openstack", "openstack-infra", "openstack-dev"]
+            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
         },
         {
             "id": "integrated",
@@ -191,12 +191,6 @@
             "child": true,
             "modules": ["official-integrated"]
         },
-        {
-            "id": "incubated",
-            "title": "incubated",
-            "child": true,
-            "modules": ["official-incubated"]
-        },
         {
             "id": "other",
             "title": "other",

etc/test_programs.yaml

@@ -1,9 +0,0 @@
-Image Service:
-  codename: Glance
-  ptl: Mark Washenberger (markwash)
-  url: https://wiki.openstack.org/wiki/Glance
-  projects:
-    - repo: openstack/glance
-      integrated-since: havana
-      incubated-since: grizzly
-    - repo: openstack/python-glanceclient

etc/test_projects.yaml

@@ -0,0 +1,16 @@
+Sahara:
+  ptl: Sergey Lukjanov (SergeyLukjanov)
+  mission: >
+    To provide a scalable data processing stack and associated management
+    interfaces.
+  url: https://wiki.openstack.org/wiki/Sahara
+  projects:
+    - repo: openstack/sahara
+      tags:
+        - name: integrated-release
+          since: juno
+    - repo: openstack/python-saharaclient
+    - repo: openstack/sahara-dashboard
+    - repo: openstack/sahara-extra
+    - repo: openstack/sahara-image-elements
+    - repo: openstack/sahara-specs

stackalytics/dashboard/templates/_macros/module_details.html

@@ -28,8 +28,8 @@
         <h2>${name}</h2>
         <div style="font-style: italic; margin-bottom: 0.5em;">
             {%if tag == 'program' %}
-                The official OpenStack program as defined in
+                The official OpenStack project as defined in
-                <a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/programs.yaml" target="_blank">programs.yaml</a>
+                <a href="https://git.openstack.org/cgit/openstack/governance/tree/reference/projects.yaml" target="_blank">projects.yaml</a>
             {%/if%}
             {%if tag == 'group' %}
                 Custom module group as defined in

stackalytics/dashboard/templates/layout.html

@@ -54,12 +54,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
             </div>
 
             <div class="drop">
-                <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
+                <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
                 <input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
             </div>
 
             <div class="drop">
-                <label for="module_selector" title="Module represents a repo (black), official program (violet) or pre-configured group of modules (cyan)">Module</label>
+                <label for="module_selector" title="Module represents a repo (black), official project (violet) or pre-configured group of modules (cyan)">Module</label>
                 <input type="hidden" id="module_selector" style="width: 140px" data-placeholder="Any module"/>
             </div>
 

stackalytics/dashboard/templates/widget.html

@@ -121,7 +121,7 @@
         </div>
 
         <div class="drop" style="margin-right: 15px;">
-            <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated, incubated, other) or belonging to same organization (stackforge, infra)">Project Type</label>
+            <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
             <input type="hidden" id="project_type_selector" style="width: 95px" data-placeholder="Select project type"/>
         </div>
 

stackalytics/processor/config.py

@@ -45,10 +45,10 @@ OPTS = [
                help='SSH username for gerrit review system access'),
     cfg.BoolOpt('force-update', default=False,
                 help='Forcibly read default data and update records'),
-    cfg.StrOpt('program-list-uri',
+    cfg.StrOpt('project-list-uri',
                default=('https://git.openstack.org/cgit/'
-                        'openstack/governance/plain/reference/programs.yaml'),
+                        'openstack/governance/plain/reference/projects.yaml'),
-               help='The address of file with list of programs'),
+               help='The address of file with the official projects list'),
     cfg.StrOpt('driverlog-data-uri',
                default='https://git.openstack.org/cgit/'
                        'stackforge/driverlog/plain/etc/default_data.json',

stackalytics/processor/main.py

@@ -240,30 +240,24 @@ def apply_corrections(uri, runtime_storage_inst):
     runtime_storage_inst.apply_corrections(valid_corrections)
 
 
-def _read_official_programs_yaml(program_list_uri, release_names):
+def _read_official_projects_yaml(project_list_uri, release_names):
-    LOG.debug('Process list of programs from uri: %s', program_list_uri)
+    LOG.debug('Process list of projects from uri: %s', project_list_uri)
-    content = yaml.safe_load(utils.read_uri(program_list_uri))
+    content = yaml.safe_load(utils.read_uri(project_list_uri))
     module_groups = collections.defaultdict(
         lambda: {'modules': [], 'releases': collections.defaultdict(list)})
 
     official_integrated = module_groups['official-integrated']
     official_integrated['tag'] = 'project_type'
     official_integrated['module_group_name'] = 'official-integrated'
-    official_incubated = module_groups['official-incubated']
-    official_incubated['tag'] = 'project_type'
-    official_incubated['module_group_name'] = 'official-incubated'
     official_other = module_groups['official-other']
     official_other['tag'] = 'project_type'
     official_other['module_group_name'] = 'official-other'
 
     for name, info in six.iteritems(content):
-        # for one program
+        # take one official project
-        group_id = name.lower()
-        if 'codename' in info:
-            name = '%s (%s)' % (info['codename'], name)
-            group_id = '%s-group' % info['codename'].lower()
 
-        module_groups[group_id]['module_group_name'] = name
+        group_id = '%s-group' % name.lower()
+        module_groups[group_id]['module_group_name'] = '%s Official' % name
         module_groups[group_id]['tag'] = 'program'
 
         for module in info['projects']:
@@ -271,17 +265,22 @@ def _read_official_programs_yaml(program_list_uri, release_names):
 
             module_groups[group_id]['modules'].append(module_name)
 
-            if ('integrated-since' in module) or ('incubated-since' in module):
+            type_matched = False
+            if 'tags' in module:
+                for tag in module.get('tags'):
+                    tag_name = tag.get('name')
+
+                    if tag_name == 'integrated-release':
+                        type_matched = True  # project type is matched here
                         project_type = 'official-other'
                         for release_name in release_names:
-                    if release_name == module.get('incubated-since'):
+                            if release_name == tag.get('since'):
-                        project_type = 'official-incubated'
-                    elif release_name == module.get('integrated-since'):
                                 project_type = 'official-integrated'
 
                             module_groups[project_type]['releases'][
                                 release_name].append(module_name)
-            else:
+
+            if not type_matched:
                 module_groups['official-other']['modules'].append(module_name)
 
     # set ids for module groups
@@ -291,13 +290,13 @@ def _read_official_programs_yaml(program_list_uri, release_names):
     return module_groups
 
 
-def process_program_list(runtime_storage_inst, program_list_uri):
+def process_project_list(runtime_storage_inst, project_list_uri):
     module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
     release_names = [r['release_name'].lower()
                      for r in runtime_storage_inst.get_by_key('releases')[1:]]
 
-    official_module_groups = _read_official_programs_yaml(
+    official_module_groups = _read_official_projects_yaml(
-        program_list_uri, release_names)
+        project_list_uri, release_names)
     LOG.debug('Update module groups with official: %s', official_module_groups)
     module_groups.update(official_module_groups)
 
@@ -335,7 +334,7 @@ def main():
                                    default_data,
                                    cfg.CONF.driverlog_data_uri)
 
-    process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri)
+    process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri)
 
     update_pids(runtime_storage_inst)