Merge "Respect history of official projects list"

2015-10-01 18:38:44 +00:00
parent 842e32b0fe bfee58eb06
commit dac6da4bdd
7 changed files with 278 additions and 43 deletions
--- a/etc/default_data.json
+++ b/etc/default_data.json
@@ -15518,51 +15518,123 @@
        },
        {
            "release_name": "Austin",
-            "end_date": "2010-Oct-21"
+            "end_date": "2010-Oct-21",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Bexar",
-            "end_date": "2011-Feb-03"
+            "end_date": "2011-Feb-03",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Cactus",
-            "end_date": "2011-Apr-15"
+            "end_date": "2011-Apr-15",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Diablo",
-            "end_date": "2011-Sep-22"
+            "end_date": "2011-Sep-22",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Essex",
-            "end_date": "2012-Apr-05"
+            "end_date": "2012-Apr-05",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Folsom",
-            "end_date": "2012-Oct-04"
+            "end_date": "2012-Oct-04",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Grizzly",
-            "end_date": "2013-Apr-04"
+            "end_date": "2013-Apr-04",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Havana",
-            "end_date": "2013-Oct-17"
+            "end_date": "2013-Oct-17",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Icehouse",
-            "end_date": "2014-Apr-17"
+            "end_date": "2014-Apr-17",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Juno",
-            "end_date": "2014-Oct-16"
+            "end_date": "2014-Oct-16",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Kilo",
-            "end_date": "2015-Apr-30"
+            "end_date": "2015-Apr-30",
+            "refs": {
+                "governance": {
+                    "type": "early_big_tent",
+                    "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d"
+                }
+            }
        },
        {
            "release_name": "Liberty",
-            "end_date": "2015-Oct-15"
+            "end_date": "2015-Oct-15",
+            "refs": {
+                "governance": {
+                    "type": "big_tent",
+                    "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml"
+                }
+            }
        }
    ],
    "mail_lists": [
--- a/etc/default_data.schema.json
+++ b/etc/default_data.schema.json
@@ -64,6 +64,9 @@
                    },
                    "end_date": {
                        "$ref": "#/definitions/date_format"
+                    },
+                    "refs": {
+                        "type": "object"
                    }
                },
                "required": ["release_name", "end_date"],
--- a/etc/test_default_data.json
+++ b/etc/test_default_data.json
@@ -127,51 +127,123 @@
        },
        {
            "release_name": "Austin",
-            "end_date": "2010-Oct-21"
+            "end_date": "2010-Oct-21",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Bexar",
-            "end_date": "2011-Feb-03"
+            "end_date": "2011-Feb-03",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Cactus",
-            "end_date": "2011-Apr-15"
+            "end_date": "2011-Apr-15",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Diablo",
-            "end_date": "2011-Sep-22"
+            "end_date": "2011-Sep-22",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Essex",
-            "end_date": "2012-Apr-05"
+            "end_date": "2012-Apr-05",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Folsom",
-            "end_date": "2012-Oct-04"
+            "end_date": "2012-Oct-04",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Grizzly",
-            "end_date": "2013-Apr-04"
+            "end_date": "2013-Apr-04",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Havana",
-            "end_date": "2013-Oct-17"
+            "end_date": "2013-Oct-17",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Icehouse",
-            "end_date": "2014-Apr-17"
+            "end_date": "2014-Apr-17",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Juno",
-            "end_date": "2014-Oct-16"
+            "end_date": "2014-Oct-16",
+            "refs": {
+                "governance": {
+                    "type": "legacy",
+                    "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af"
+                }
+            }
        },
        {
            "release_name": "Kilo",
-            "end_date": "2015-Apr-30"
+            "end_date": "2015-Apr-30",
+            "refs": {
+                "governance": {
+                    "type": "early_big_tent",
+                    "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d"
+                }
+            }
        },
        {
            "release_name": "Liberty",
-            "end_date": "2015-Oct-15"
+            "end_date": "2015-Oct-15",
+            "refs": {
+                "governance": {
+                    "type": "big_tent",
+                    "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml"
+                }
+            }
        }
    ],

--- a/stackalytics/dashboard/templates/layout.html
+++ b/stackalytics/dashboard/templates/layout.html
@@ -57,7 +57,7 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
            </div>

            <div class="drop">
-                <label for="project_type_selector" title="Project type groups modules of same kind: official (integrated-release, other) or belonging to same organization (openstack, stackforge)">Project Type</label>
+                <label for="project_type_selector" title="Project type groups modules of same kind: all official (listed in governance's projects.yaml), having the same tag and complementary">Project Type</label>
                <input type="hidden" id="project_type_selector" style="width: 140px" data-placeholder="Select project type"/>
            </div>

--- a/stackalytics/processor/governance.py
+++ b/stackalytics/processor/governance.py
@@ -33,19 +33,51 @@ def _make_module_group(module_groups, name):
    return m


-def read_projects_yaml(project_list_uri):
-    LOG.debug('Process list of projects from uri: %s', project_list_uri)
-    content = yaml.safe_load(utils.read_uri(project_list_uri))
-    module_groups = collections.defaultdict(lambda: {'modules': []})
+def read_legacy_programs_yaml(module_groups, release_name, content):
+    all_official = module_groups['openstack-official']

-    all_official = _make_module_group(module_groups, 'openstack-official')
+    for name, info in six.iteritems(content):
+        # for one program
+        # group_id = name.lower()
+        # if 'codename' in info:
+        #     name = '%s (%s)' % (info['codename'], name)
+        #     group_id = '%s-group' % info['codename'].lower()
+        #
+        # module_groups[group_id]['module_group_name'] = name
+        # module_groups[group_id]['tag'] = 'program'

-    for tag in TAGS:
-        _make_module_group(module_groups, tag)
+        for module in info['projects']:
+            mn = module['repo'].split('/')[1]  # module_name
+
+            # module_groups[group_id]['releases'][release_name].append(mn)
+            all_official['releases'][release_name].append(mn)
+
+
+def read_early_big_tent_projects_yaml(module_groups, release_name, content):
+    all_official = module_groups['openstack-official']
+
+    for name, info in six.iteritems(content):
+        # group_id = '%s-group' % name.lower()
+        # module_groups[group_id]['module_group_name'] = '%s Official' % name
+        # module_groups[group_id]['tag'] = 'program'
+
+        for module in info['projects']:
+            repo_split = module['repo'].split('/')
+            if len(repo_split) < 2:
+                continue  # valid repo must be in form of 'org/module'
+            mn = repo_split[1]
+
+            # module_groups[group_id]['releases'][release_name].append(mn)
+            all_official['releases'][release_name].append(mn)
+
+
+def read_big_tent_projects_yaml(module_groups, release_name, content):
+    all_official = module_groups['openstack-official']

    for name, project in six.iteritems(content):
        group_id = '%s-group' % name.lower()
-        module_groups[group_id]['module_group_name'] = '%s Official' % name
+        module_groups[group_id]['module_group_name'] = (
+            '%s Official' % name.title())
        module_groups[group_id]['tag'] = 'program'

        for d_name, deliverable in six.iteritems(project['deliverables']):
@@ -53,20 +85,66 @@ def read_projects_yaml(project_list_uri):
                repo_split = repo.split('/')
                if len(repo_split) < 2:
                    continue  # valid repo must be in form of 'org/module'
-                module_name = repo_split[1]

-                module_groups[group_id]['modules'].append(module_name)
+                mn = repo_split[1]  # module_name

-                all_official['modules'].append(module_name)
+                module_groups[group_id]['modules'].append(mn)
+                all_official['releases'][release_name].append(mn)

                tags = deliverable.get('tags', [])
                for tag in tags:
                    if tag in TAGS:
-                        module_groups[tag]['modules'].append(module_name)
+                        module_groups[tag]['modules'].append(mn)
+
+
+def _make_default_module_groups():
+    # create default module groups
+    module_groups = collections.defaultdict(lambda: {'modules': []})
+
+    # openstack official
+    _make_module_group(module_groups, 'openstack-official')
+    module_groups['openstack-official']['releases'] = (
+        collections.defaultdict(list))
+
+    # tags
+    for tag in TAGS:
+        _make_module_group(module_groups, tag)
+
+    return module_groups
+
+
+GOVERNANCE_PROCESSORS = {
+    'legacy': read_legacy_programs_yaml,
+    'early_big_tent': read_early_big_tent_projects_yaml,
+    'big_tent': read_big_tent_projects_yaml,
+}
+
+
+def process_official_list(releases):
+    module_groups = _make_default_module_groups()
+    releases_with_refs = (r for r in releases if r.get('refs'))
+
+    for release in releases_with_refs:
+        ref_governance = release['refs'].get('governance')
+        if not ref_governance:
+            continue
+
+        gov_type = ref_governance['type']
+        gov_source = ref_governance['source']
+        release_name = release['release_name'].lower()
+
+        LOG.debug('Process governance content from uri: %s', gov_source)
+        content = yaml.safe_load(utils.read_uri(gov_source))
+
+        GOVERNANCE_PROCESSORS[gov_type](module_groups, release_name, content)

    # set ids for module groups
    for group_id, group in six.iteritems(module_groups):
        group['id'] = group_id
        group['modules'].sort()

+        if 'releases' in group:
+            for gr in six.itervalues(group['releases']):
+                gr.sort()
+
    return module_groups
--- a/stackalytics/processor/main.py
+++ b/stackalytics/processor/main.py
@@ -240,8 +240,10 @@ def apply_corrections(uri, runtime_storage_inst):

 def process_project_list(runtime_storage_inst, project_list_uri):
    module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
+    releases = runtime_storage_inst.get_by_key('releases') or {}
+
+    official_module_groups = governance.process_official_list(releases)

-    official_module_groups = governance.read_projects_yaml(project_list_uri)
    LOG.debug('Update module groups with official: %s', official_module_groups)
    module_groups.update(official_module_groups)

--- a/stackalytics/tests/unit/test_governance.py
+++ b/stackalytics/tests/unit/test_governance.py
@@ -62,7 +62,7 @@ Sahara:
 class TestGovernance(testtools.TestCase):

    @mock.patch('stackalytics.processor.utils.read_uri')
-    def test_read_official_projects_yaml(self, read_uri):
+    def test_process_official_list(self, read_uri):
        read_uri.return_value = SAMPLE

        expected = {
@@ -95,13 +95,21 @@ class TestGovernance(testtools.TestCase):
            'openstack-official': {
                'id': 'openstack-official',
                'module_group_name': 'openstack-official',
-                'modules': ['python-saharaclient', 'sahara',
-                            'sahara-dashboard', 'sahara-extra',
-                            'sahara-image-elements', 'sahara-specs'],
+                'modules': [],
+                'releases': {
+                    'liberty': ['python-saharaclient', 'sahara',
+                                'sahara-dashboard', 'sahara-extra',
+                                'sahara-image-elements', 'sahara-specs'],
+                },
                'tag': 'project_type'
            },
        }

-        actual = governance.read_projects_yaml('uri')
+        releases = [{
+            'release_name': 'Liberty',
+            'refs': {'governance': {'type': 'big_tent', 'source': 'uri'}}
+        }]
+
+        actual = governance.process_official_list(releases)

        self.assertEqual(expected, actual)