Support new format of official projects.yaml

This patch makes support of latest schema of project.yaml
(introduced by https://review.openstack.org/#/c/202583/)

Change-Id: I3307300711638cd33de9420b8efd7635417926ab

etc/default_data.json

@@ -14141,19 +14141,13 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
+            "modules": ["tc-approved-release", "governance", "api-wg"]
         },
         {
-            "id": "integrated",
+            "id": "tc-approved-release",
-            "title": "integrated",
+            "title": "tc-approved",
             "child": true,
-            "modules": ["official-integrated"]
+            "modules": ["tc-approved-release"]
-        },
-        {
-            "id": "other",
-            "title": "other",
-            "child": true,
-            "modules": ["official-other"]
         },
         {
             "id": "stackforge",

etc/test_default_data.json

@@ -168,6 +168,10 @@
         {
             "release_name": "Kilo",
             "end_date": "2015-Apr-30"
+        },
+        {
+            "release_name": "Liberty",
+            "end_date": "2015-Oct-15"
         }
     ],
 
@@ -183,19 +187,13 @@
         {
             "id": "openstack",
             "title": "OpenStack",
-            "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
+            "modules": ["tc-approved-release", "governance", "python-openstackclient", "api-wg"]
         },
         {
-            "id": "integrated",
+            "id": "tc-approved-release",
-            "title": "integrated",
+            "title": "tc-approved",
             "child": true,
-            "modules": ["official-integrated"]
+            "modules": ["tc-approved-release"]
-        },
-        {
-            "id": "other",
-            "title": "other",
-            "child": true,
-            "modules": ["official-other"]
         },
         {
             "id": "stackforge",

etc/test_projects.yaml

@@ -1,16 +1,38 @@
 Sahara:
   ptl: Sergey Lukjanov (SergeyLukjanov)
+  irc-channel: openstack-sahara
+  service: Data processing service
   mission: >
     To provide a scalable data processing stack and associated management
     interfaces.
   url: https://wiki.openstack.org/wiki/Sahara
-  projects:
+  deliverables:
-    - repo: openstack/sahara
+    python-saharaclient:
+      repos:
+        - openstack/python-saharaclient
       tags:
-        - name: integrated-release
+        - release:cycle-with-intermediary
-          since: juno
+        - release:has-stable-branches
-    - repo: openstack/python-saharaclient
+        - type:library
-    - repo: openstack/sahara-dashboard
+        - release:managed
-    - repo: openstack/sahara-extra
+        - vulnerability:managed
-    - repo: openstack/sahara-image-elements
+    sahara:
-    - repo: openstack/sahara-specs
+      repos:
+        - openstack/sahara
+        - openstack/sahara-extra
+        - openstack/sahara-image-elements
+      tags:
+        - tc-approved-release
+        - release:managed
+        - release:cycle-with-milestones
+        - release:has-stable-branches
+        - type:service
+        - vulnerability:managed
+    sahara-dashboard:
+      repos:
+        - openstack/sahara-dashboard
+      tags:
+        - type:library
+    sahara-specs:
+      repos:
+        - openstack/sahara-specs

stackalytics/processor/governance.py

@@ -0,0 +1,62 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import collections
+
+from oslo_log import log as logging
+import six
+import yaml
+
+from stackalytics.processor import utils
+
+
+LOG = logging.getLogger(__name__)
+
+TAGS = ['tc-approved-release']  # list of supported tags
+
+
+def read_projects_yaml(project_list_uri):
+    LOG.debug('Process list of projects from uri: %s', project_list_uri)
+    content = yaml.safe_load(utils.read_uri(project_list_uri))
+    module_groups = collections.defaultdict(lambda: {'modules': []})
+
+    for tag in TAGS:
+        m = module_groups[tag]  # object created by defaultdict
+        m['tag'] = 'project_type'
+        m['module_group_name'] = tag
+
+    for name, project in six.iteritems(content):
+        group_id = '%s-group' % name.lower()
+        module_groups[group_id]['module_group_name'] = '%s Official' % name
+        module_groups[group_id]['tag'] = 'program'
+
+        for d_name, deliverable in six.iteritems(project['deliverables']):
+            for repo in deliverable['repos']:
+                repo_split = repo.split('/')
+                if len(repo_split) < 2:
+                    continue  # valid repo must be in form of 'org/module'
+                module_name = repo_split[1]
+
+                module_groups[group_id]['modules'].append(module_name)
+
+                tags = deliverable.get('tags', [])
+                for tag in tags:
+                    if tag in TAGS:
+                        module_groups[tag]['modules'].append(module_name)
+
+    # set ids for module groups
+    for group_id, group in six.iteritems(module_groups):
+        group['id'] = group_id
+        group['modules'].sort()
+
+    return module_groups

stackalytics/processor/main.py

@@ -13,19 +13,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-import collections
-
 from oslo_config import cfg
 from oslo_log import log as logging
 import psutil
-import six
 from six.moves.urllib import parse
-import yaml
 
 from stackalytics.processor import bps
 from stackalytics.processor import config
 from stackalytics.processor import default_data_processor
 from stackalytics.processor import driverlog
+from stackalytics.processor import governance
 from stackalytics.processor import lp
 from stackalytics.processor import mls
 from stackalytics.processor import mps
@@ -35,7 +32,6 @@ from stackalytics.processor import runtime_storage
 from stackalytics.processor import utils
 from stackalytics.processor import vcs
 
-
 LOG = logging.getLogger(__name__)
 
 
@@ -242,66 +238,10 @@ def apply_corrections(uri, runtime_storage_inst):
     runtime_storage_inst.apply_corrections(valid_corrections)
 
 
-def _read_official_projects_yaml(project_list_uri, release_names):
-    LOG.debug('Process list of projects from uri: %s', project_list_uri)
-    content = yaml.safe_load(utils.read_uri(project_list_uri))
-    module_groups = collections.defaultdict(
-        lambda: {'modules': [], 'releases': collections.defaultdict(list)})
-
-    official_integrated = module_groups['official-integrated']
-    official_integrated['tag'] = 'project_type'
-    official_integrated['module_group_name'] = 'official-integrated'
-    official_other = module_groups['official-other']
-    official_other['tag'] = 'project_type'
-    official_other['module_group_name'] = 'official-other'
-
-    for name, info in six.iteritems(content):
-        # take one official project
-
-        group_id = '%s-group' % name.lower()
-        module_groups[group_id]['module_group_name'] = '%s Official' % name
-        module_groups[group_id]['tag'] = 'program'
-
-        for module in info['projects']:
-            repo_split = module['repo'].split('/')
-            if len(repo_split) < 2:
-                continue  # valid repo must be in form of 'org/module'
-            module_name = repo_split[1]
-
-            module_groups[group_id]['modules'].append(module_name)
-
-            type_matched = False
-            if 'tags' in module:
-                for tag in module.get('tags'):
-                    tag_name = tag.get('name')
-
-                    if tag_name == 'integrated-release':
-                        type_matched = True  # project type is matched here
-                        project_type = 'official-other'
-                        for release_name in release_names:
-                            if release_name == tag.get('since'):
-                                project_type = 'official-integrated'
-
-                            module_groups[project_type]['releases'][
-                                release_name].append(module_name)
-
-            if not type_matched:
-                module_groups['official-other']['modules'].append(module_name)
-
-    # set ids for module groups
-    for group_id, group in six.iteritems(module_groups):
-        group['id'] = group_id
-
-    return module_groups
-
-
 def process_project_list(runtime_storage_inst, project_list_uri):
     module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
-    release_names = [r['release_name'].lower()
-                     for r in runtime_storage_inst.get_by_key('releases')[1:]]
 
-    official_module_groups = _read_official_projects_yaml(
+    official_module_groups = governance.read_projects_yaml(project_list_uri)
-        project_list_uri, release_names)
     LOG.debug('Update module groups with official: %s', official_module_groups)
     module_groups.update(official_module_groups)
 

stackalytics/processor/rcs.py

@@ -126,6 +126,7 @@ class Gerrit(Rcs):
                       start_id=0, last_id=0, is_open=False,
                       grab_comments=False):
         sort_key = start_id
+        last_id = last_id or 0
 
         while True:
             cmd = self._get_cmd(project_organization, module, branch, sort_key,

tests/unit/test_governance.py

@@ -0,0 +1,87 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import mock
+import testtools
+
+from stackalytics.processor import governance
+
+
+SAMPLE = """
+Sahara:
+  ptl: Sergey Lukjanov (SergeyLukjanov)
+  irc-channel: openstack-sahara
+  service: Data processing service
+  mission: >
+    To provide a scalable data processing stack and associated management
+    interfaces.
+  url: https://wiki.openstack.org/wiki/Sahara
+  deliverables:
+    python-saharaclient:
+      repos:
+        - openstack/python-saharaclient
+      tags:
+        - release:cycle-with-intermediary
+        - release:has-stable-branches
+        - type:library
+        - release:managed
+        - vulnerability:managed
+    sahara:
+      repos:
+        - openstack/sahara
+        - openstack/sahara-extra
+        - openstack/sahara-image-elements
+      tags:
+        - tc-approved-release
+        - release:managed
+        - release:cycle-with-milestones
+        - release:has-stable-branches
+        - type:service
+        - vulnerability:managed
+    sahara-dashboard:
+      repos:
+        - openstack/sahara-dashboard
+      tags:
+        - type:library
+    sahara-specs:
+      repos:
+        - openstack/sahara-specs
+"""
+
+
+class TestGovernance(testtools.TestCase):
+
+    @mock.patch('stackalytics.processor.utils.read_uri')
+    def test_read_official_projects_yaml(self, read_uri):
+        read_uri.return_value = SAMPLE
+
+        expected = {
+            'sahara-group': {
+                'id': 'sahara-group',
+                'module_group_name': 'Sahara Official',
+                'modules': ['python-saharaclient', 'sahara',
+                            'sahara-dashboard', 'sahara-extra',
+                            'sahara-image-elements', 'sahara-specs'],
+                'tag': 'program'
+            },
+            'tc-approved-release': {
+                'id': 'tc-approved-release',
+                'module_group_name': 'tc-approved-release',
+                'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
+                'tag': 'project_type'
+            }
+        }
+
+        actual = governance.read_projects_yaml('uri')
+
+        self.assertEqual(expected, actual)