From bfee58eb06b8e5d968375fe8cf34e04986538d2a Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Thu, 1 Oct 2015 18:32:10 +0300 Subject: [PATCH] Respect history of official projects list List of official projects constantly changes, if the project became official it appears in the past stats too. To prevent this Stackalytics now checks list of official projects once per release using tags specified in the default_data.json. For initial approach we use election tags for openstack/governance project. Change-Id: I140025775f049fa2be85f42380ab620f84abf267 Partially-Bug: #1497403 --- etc/default_data.json | 96 ++++++++++++++--- etc/default_data.schema.json | 3 + etc/test_default_data.json | 96 ++++++++++++++--- stackalytics/dashboard/templates/layout.html | 2 +- stackalytics/processor/governance.py | 102 ++++++++++++++++--- stackalytics/processor/main.py | 4 +- stackalytics/tests/unit/test_governance.py | 18 +++- 7 files changed, 278 insertions(+), 43 deletions(-) diff --git a/etc/default_data.json b/etc/default_data.json index 0533b8fae..30f70a501 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -15460,51 +15460,123 @@ }, { "release_name": "Austin", - "end_date": "2010-Oct-21" + "end_date": "2010-Oct-21", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Bexar", - "end_date": "2011-Feb-03" + "end_date": "2011-Feb-03", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Cactus", - "end_date": "2011-Apr-15" + "end_date": "2011-Apr-15", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Diablo", - "end_date": "2011-Sep-22" + "end_date": "2011-Sep-22", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Essex", - "end_date": "2012-Apr-05" + "end_date": "2012-Apr-05", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Folsom", - "end_date": "2012-Oct-04" + "end_date": "2012-Oct-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Grizzly", - "end_date": "2013-Apr-04" + "end_date": "2013-Apr-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Havana", - "end_date": "2013-Oct-17" + "end_date": "2013-Oct-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Icehouse", - "end_date": "2014-Apr-17" + "end_date": "2014-Apr-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Juno", - "end_date": "2014-Oct-16" + "end_date": "2014-Oct-16", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Kilo", - "end_date": "2015-Apr-30" + "end_date": "2015-Apr-30", + "refs": { + "governance": { + "type": "early_big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d" + } + } }, { "release_name": "Liberty", - "end_date": "2015-Oct-15" + "end_date": "2015-Oct-15", + "refs": { + "governance": { + "type": "big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" + } + } } ], "mail_lists": [ diff --git a/etc/default_data.schema.json b/etc/default_data.schema.json index 844fcb76e..6807970ec 100644 --- a/etc/default_data.schema.json +++ b/etc/default_data.schema.json @@ -64,6 +64,9 @@ }, "end_date": { "$ref": "#/definitions/date_format" + }, + "refs": { + "type": "object" } }, "required": ["release_name", "end_date"], diff --git a/etc/test_default_data.json b/etc/test_default_data.json index 2e2857c5e..18198e4a0 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -127,51 +127,123 @@ }, { "release_name": "Austin", - "end_date": "2010-Oct-21" + "end_date": "2010-Oct-21", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Bexar", - "end_date": "2011-Feb-03" + "end_date": "2011-Feb-03", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Cactus", - "end_date": "2011-Apr-15" + "end_date": "2011-Apr-15", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Diablo", - "end_date": "2011-Sep-22" + "end_date": "2011-Sep-22", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Essex", - "end_date": "2012-Apr-05" + "end_date": "2012-Apr-05", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Folsom", - "end_date": "2012-Oct-04" + "end_date": "2012-Oct-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Grizzly", - "end_date": "2013-Apr-04" + "end_date": "2013-Apr-04", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Havana", - "end_date": "2013-Oct-17" + "end_date": "2013-Oct-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Icehouse", - "end_date": "2014-Apr-17" + "end_date": "2014-Apr-17", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Juno", - "end_date": "2014-Oct-16" + "end_date": "2014-Oct-16", + "refs": { + "governance": { + "type": "legacy", + "source": "http://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml?id=5923e6e2eacf961cf2d97b21b0b38cc29461b7af" + } + } }, { "release_name": "Kilo", - "end_date": "2015-Apr-30" + "end_date": "2015-Apr-30", + "refs": { + "governance": { + "type": "early_big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml?id=c13377dd51e78977d673b64a9a47915b7e4bfe1d" + } + } }, { "release_name": "Liberty", - "end_date": "2015-Oct-15" + "end_date": "2015-Oct-15", + "refs": { + "governance": { + "type": "big_tent", + "source": "https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml" + } + } } ], diff --git a/stackalytics/dashboard/templates/layout.html b/stackalytics/dashboard/templates/layout.html index f3dd3bf26..c258140d2 100644 --- a/stackalytics/dashboard/templates/layout.html +++ b/stackalytics/dashboard/templates/layout.html @@ -57,7 +57,7 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
- +
diff --git a/stackalytics/processor/governance.py b/stackalytics/processor/governance.py index b3f700229..3c35c4d51 100644 --- a/stackalytics/processor/governance.py +++ b/stackalytics/processor/governance.py @@ -33,19 +33,51 @@ def _make_module_group(module_groups, name): return m -def read_projects_yaml(project_list_uri): - LOG.debug('Process list of projects from uri: %s', project_list_uri) - content = yaml.safe_load(utils.read_uri(project_list_uri)) - module_groups = collections.defaultdict(lambda: {'modules': []}) +def read_legacy_programs_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] - all_official = _make_module_group(module_groups, 'openstack-official') + for name, info in six.iteritems(content): + # for one program + # group_id = name.lower() + # if 'codename' in info: + # name = '%s (%s)' % (info['codename'], name) + # group_id = '%s-group' % info['codename'].lower() + # + # module_groups[group_id]['module_group_name'] = name + # module_groups[group_id]['tag'] = 'program' - for tag in TAGS: - _make_module_group(module_groups, tag) + for module in info['projects']: + mn = module['repo'].split('/')[1] # module_name + + # module_groups[group_id]['releases'][release_name].append(mn) + all_official['releases'][release_name].append(mn) + + +def read_early_big_tent_projects_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] + + for name, info in six.iteritems(content): + # group_id = '%s-group' % name.lower() + # module_groups[group_id]['module_group_name'] = '%s Official' % name + # module_groups[group_id]['tag'] = 'program' + + for module in info['projects']: + repo_split = module['repo'].split('/') + if len(repo_split) < 2: + continue # valid repo must be in form of 'org/module' + mn = repo_split[1] + + # module_groups[group_id]['releases'][release_name].append(mn) + all_official['releases'][release_name].append(mn) + + +def read_big_tent_projects_yaml(module_groups, release_name, content): + all_official = module_groups['openstack-official'] for name, project in six.iteritems(content): group_id = '%s-group' % name.lower() - module_groups[group_id]['module_group_name'] = '%s Official' % name + module_groups[group_id]['module_group_name'] = ( + '%s Official' % name.title()) module_groups[group_id]['tag'] = 'program' for d_name, deliverable in six.iteritems(project['deliverables']): @@ -53,20 +85,66 @@ def read_projects_yaml(project_list_uri): repo_split = repo.split('/') if len(repo_split) < 2: continue # valid repo must be in form of 'org/module' - module_name = repo_split[1] - module_groups[group_id]['modules'].append(module_name) + mn = repo_split[1] # module_name - all_official['modules'].append(module_name) + module_groups[group_id]['modules'].append(mn) + all_official['releases'][release_name].append(mn) tags = deliverable.get('tags', []) for tag in tags: if tag in TAGS: - module_groups[tag]['modules'].append(module_name) + module_groups[tag]['modules'].append(mn) + + +def _make_default_module_groups(): + # create default module groups + module_groups = collections.defaultdict(lambda: {'modules': []}) + + # openstack official + _make_module_group(module_groups, 'openstack-official') + module_groups['openstack-official']['releases'] = ( + collections.defaultdict(list)) + + # tags + for tag in TAGS: + _make_module_group(module_groups, tag) + + return module_groups + + +GOVERNANCE_PROCESSORS = { + 'legacy': read_legacy_programs_yaml, + 'early_big_tent': read_early_big_tent_projects_yaml, + 'big_tent': read_big_tent_projects_yaml, +} + + +def process_official_list(releases): + module_groups = _make_default_module_groups() + releases_with_refs = (r for r in releases if r.get('refs')) + + for release in releases_with_refs: + ref_governance = release['refs'].get('governance') + if not ref_governance: + continue + + gov_type = ref_governance['type'] + gov_source = ref_governance['source'] + release_name = release['release_name'].lower() + + LOG.debug('Process governance content from uri: %s', gov_source) + content = yaml.safe_load(utils.read_uri(gov_source)) + + GOVERNANCE_PROCESSORS[gov_type](module_groups, release_name, content) # set ids for module groups for group_id, group in six.iteritems(module_groups): group['id'] = group_id group['modules'].sort() + if 'releases' in group: + for gr in six.itervalues(group['releases']): + gr.sort() + return module_groups diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index fa1b87d0c..987e0986b 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -240,8 +240,10 @@ def apply_corrections(uri, runtime_storage_inst): def process_project_list(runtime_storage_inst, project_list_uri): module_groups = runtime_storage_inst.get_by_key('module_groups') or {} + releases = runtime_storage_inst.get_by_key('releases') or {} + + official_module_groups = governance.process_official_list(releases) - official_module_groups = governance.read_projects_yaml(project_list_uri) LOG.debug('Update module groups with official: %s', official_module_groups) module_groups.update(official_module_groups) diff --git a/stackalytics/tests/unit/test_governance.py b/stackalytics/tests/unit/test_governance.py index ca77a509c..5182e6523 100644 --- a/stackalytics/tests/unit/test_governance.py +++ b/stackalytics/tests/unit/test_governance.py @@ -62,7 +62,7 @@ Sahara: class TestGovernance(testtools.TestCase): @mock.patch('stackalytics.processor.utils.read_uri') - def test_read_official_projects_yaml(self, read_uri): + def test_process_official_list(self, read_uri): read_uri.return_value = SAMPLE expected = { @@ -95,13 +95,21 @@ class TestGovernance(testtools.TestCase): 'openstack-official': { 'id': 'openstack-official', 'module_group_name': 'openstack-official', - 'modules': ['python-saharaclient', 'sahara', - 'sahara-dashboard', 'sahara-extra', - 'sahara-image-elements', 'sahara-specs'], + 'modules': [], + 'releases': { + 'liberty': ['python-saharaclient', 'sahara', + 'sahara-dashboard', 'sahara-extra', + 'sahara-image-elements', 'sahara-specs'], + }, 'tag': 'project_type' }, } - actual = governance.read_projects_yaml('uri') + releases = [{ + 'release_name': 'Liberty', + 'refs': {'governance': {'type': 'big_tent', 'source': 'uri'}} + }] + + actual = governance.process_official_list(releases) self.assertEqual(expected, actual)