Fill in openstack-others project type

All projects located in 'openstack' organization, but not listed
in the official projects list are grouped into 'openstack-others'
project type.

Change-Id: Ib7caee39294a24e0030de6c8a5116c9a31147a27

etc/default_data.json

@@ -15638,7 +15638,7 @@
         },
         {
             "release_name": "Mitaka",
-            "end_date": "2015-Apr-07",
+            "end_date": "2016-Apr-07",
             "refs": {
                 "governance": {
                     "type": "big_tent",
@@ -15668,7 +15668,7 @@
         },
         {
             "id": "tc-approved-release",
-            "title": "tc-approved",
+            "title": "tc-approved-release",
             "child": true,
             "modules": ["tc-approved-release"]
         },
@@ -15684,6 +15684,11 @@
             "child": true,
             "modules": ["type:library"]
         },
+        {
+            "id": "openstack-others",
+            "title": "OpenStack Others",
+            "modules": ["openstack-others", "unknown"]
+        },
         {
             "id": "stackforge",
             "title": "Stackforge",

etc/test_default_data.json

@@ -247,7 +247,7 @@
         },
         {
             "release_name": "Mitaka",
-            "end_date": "2015-Apr-07",
+            "end_date": "2016-Apr-07",
             "refs": {
                 "governance": {
                     "type": "big_tent",
@@ -273,7 +273,7 @@
         },
         {
             "id": "tc-approved-release",
-            "title": "tc-approved",
+            "title": "tc-approved-release",
             "child": true,
             "modules": ["tc-approved-release"]
         },
@@ -289,6 +289,11 @@
             "child": true,
             "modules": ["type:library"]
         },
+        {
+            "id": "openstack-others",
+            "title": "OpenStack Others",
+            "modules": ["openstack-others", "unknown"]
+        },
         {
             "id": "stackforge",
             "title": "Stackforge",

stackalytics/dashboard/templates/layout.html

@@ -57,7 +57,8 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
             </div>
 
             <div class="drop">
-                <label for="project_type_selector" title="Project type groups modules of same kind: all official (listed in governance's projects.yaml), having the same tag and complementary">Project Type</label>
+                <label for="project_type_selector" title="Project type groups modules of the same kind. 'OpenStack' are projects defined in the official governance projects.yaml.
+                    'OpenStack Others' are projects not included into any program. 'Complementary' are projects related to OpenStack ecosystem">Project Type</label>
                 <input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
             </div>
 

stackalytics/processor/governance.py

@@ -30,6 +30,7 @@ def _make_module_group(module_groups, name):
     m = module_groups[name]  # object created by defaultdict
     m['tag'] = 'project_type'
     m['module_group_name'] = name
+    m['releases'] = collections.defaultdict(list)
     return m
 
 
@@ -37,15 +38,6 @@ def read_legacy_programs_yaml(module_groups, release_name, content):
     all_official = module_groups['openstack-official']
 
     for name, info in six.iteritems(content):
-        # for one program
-        # group_id = name.lower()
-        # if 'codename' in info:
-        #     name = '%s (%s)' % (info['codename'], name)
-        #     group_id = '%s-group' % info['codename'].lower()
-        #
-        # module_groups[group_id]['module_group_name'] = name
-        # module_groups[group_id]['tag'] = 'program'
-
         for module in info['projects']:
             mn = module['repo'].split('/')[1]  # module_name
 
@@ -57,10 +49,6 @@ def read_early_big_tent_projects_yaml(module_groups, release_name, content):
     all_official = module_groups['openstack-official']
 
     for name, info in six.iteritems(content):
-        # group_id = '%s-group' % name.lower()
-        # module_groups[group_id]['module_group_name'] = '%s Official' % name
-        # module_groups[group_id]['tag'] = 'program'
-
         for module in info['projects']:
             repo_split = module['repo'].split('/')
             if len(repo_split) < 2:
@@ -94,7 +82,7 @@ def read_big_tent_projects_yaml(module_groups, release_name, content):
                 tags = deliverable.get('tags', [])
                 for tag in tags:
                     if tag in TAGS:
-                        module_groups[tag]['modules'].append(mn)
+                        module_groups[tag]['releases'][release_name].append(mn)
 
 
 def _make_default_module_groups():
@@ -103,8 +91,9 @@ def _make_default_module_groups():
 
     # openstack official
     _make_module_group(module_groups, 'openstack-official')
-    module_groups['openstack-official']['releases'] = (
+
-        collections.defaultdict(list))
+    # openstack others
+    _make_module_group(module_groups, 'openstack-others')
 
     # tags
     for tag in TAGS:

stackalytics/processor/main.py

@@ -16,7 +16,7 @@
 from oslo_config import cfg
 from oslo_log import log as logging
 import psutil
-from six.moves.urllib import parse
+import six
 
 from stackalytics.processor import bps
 from stackalytics.processor import config
@@ -129,7 +129,8 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
     for branch in branches:
         LOG.debug('Processing commits in repo: %s, branch: %s', uri, branch)
 
-        vcs_key = 'vcs:' + str(parse.quote_plus(uri) + ':' + branch)
+        vcs_key = 'vcs:' + str(six.moves.urllib.parse.quote_plus(uri) +
+                               ':' + branch)
         last_id = runtime_storage_inst.get_by_key(vcs_key)
 
         commit_iterator = vcs_inst.log(branch, last_id)
@@ -144,7 +145,8 @@ def _process_repo(repo, runtime_storage_inst, record_processor_inst,
 
         LOG.debug('Processing reviews for repo: %s, branch: %s', uri, branch)
 
-        rcs_key = 'rcs:' + str(parse.quote_plus(uri) + ':' + branch)
+        rcs_key = 'rcs:' + str(six.moves.urllib.parse.quote_plus(uri) +
+                               ':' + branch)
         last_id = runtime_storage_inst.get_by_key(rcs_key)
 
         review_iterator = rcs_inst.log(repo, branch, last_id,
@@ -247,12 +249,21 @@ def process_project_list(runtime_storage_inst, project_list_uri):
     LOG.debug('Update module groups with official: %s', official_module_groups)
     module_groups.update(official_module_groups)
 
+    others = module_groups['openstack-others']
+    off_rm = module_groups['openstack-official']['releases']
+    official = dict((r, set(m)) for r, m in six.iteritems(off_rm))
+
     # register modules as module groups
     repos = runtime_storage_inst.get_by_key('repos') or []
     for repo in repos:
         module = repo['module']
         module_groups[module] = utils.make_module_group(module, tag='module')
 
+        # check if repo is official or not
+        for r, off_m in six.iteritems(official):
+            if module not in off_m:
+                others['releases'][r].append(module)
+
     # register module 'unknown' - used for emails not mapped to any module
     module_groups['unknown'] = utils.make_module_group('unknown', tag='module')
 

stackalytics/tests/unit/test_governance.py

@@ -77,19 +77,30 @@ class TestGovernance(testtools.TestCase):
             'tc-approved-release': {
                 'id': 'tc-approved-release',
                 'module_group_name': 'tc-approved-release',
-                'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
+                'modules': [],
+                'releases': {
+                    'liberty': ['sahara', 'sahara-extra',
+                                'sahara-image-elements'],
+                },
                 'tag': 'project_type'
             },
             'type:library': {
                 'id': 'type:library',
                 'module_group_name': 'type:library',
-                'modules': ['python-saharaclient', 'sahara-dashboard'],
+                'modules': [],
+                'releases': {
+                    'liberty': ['python-saharaclient', 'sahara-dashboard'],
+                },
                 'tag': 'project_type'
             },
             'type:service': {
                 'id': 'type:service',
                 'module_group_name': 'type:service',
-                'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
+                'modules': [],
+                'releases': {
+                    'liberty': ['sahara', 'sahara-extra',
+                                'sahara-image-elements'],
+                },
                 'tag': 'project_type'
             },
             'openstack-official': {
@@ -103,6 +114,13 @@ class TestGovernance(testtools.TestCase):
                 },
                 'tag': 'project_type'
             },
+            'openstack-others': {
+                'id': 'openstack-others',
+                'module_group_name': 'openstack-others',
+                'modules': [],
+                'releases': {},
+                'tag': 'project_type'
+            }
         }
 
         releases = [{