From 570887cd576d34876f3ab0164f8ede5411273beb Mon Sep 17 00:00:00 2001 From: Ilya Shakhat Date: Fri, 6 Feb 2015 16:04:36 +0300 Subject: [PATCH] Support of a new format of projects.yaml Recently the program definition file was changed in governance repo. The new format is project oriented, incubation status is removed and integration status is specified as tag. Closes bug 1417385 Change-Id: I06d8c27d7afac2660f09175e17173b8e089485e1 --- etc/default_data.json | 8 +-- etc/stackalytics.conf | 4 +- etc/test_default_data.json | 8 +-- etc/test_programs.yaml | 9 ---- etc/test_projects.yaml | 16 ++++++ .../templates/_macros/module_details.html | 4 +- stackalytics/dashboard/templates/layout.html | 4 +- stackalytics/dashboard/templates/widget.html | 2 +- stackalytics/processor/config.py | 6 +-- stackalytics/processor/main.py | 51 +++++++++---------- 10 files changed, 53 insertions(+), 59 deletions(-) delete mode 100644 etc/test_programs.yaml create mode 100644 etc/test_projects.yaml diff --git a/etc/default_data.json b/etc/default_data.json index 2722220af..7ccc63433 100644 --- a/etc/default_data.json +++ b/etc/default_data.json @@ -11694,7 +11694,7 @@ { "id": "openstack", "title": "OpenStack", - "modules": ["official-integrated", "official-incubated", "official-other", "governance", "python-openstackclient", "api-wg"] + "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"] }, { "id": "integrated", @@ -11702,12 +11702,6 @@ "child": true, "modules": ["official-integrated"] }, - { - "id": "incubated", - "title": "incubated", - "child": true, - "modules": ["official-incubated"] - }, { "id": "other", "title": "other", diff --git a/etc/stackalytics.conf b/etc/stackalytics.conf index 63b2d5dd1..f6a8a1c41 100644 --- a/etc/stackalytics.conf +++ b/etc/stackalytics.conf @@ -38,8 +38,8 @@ # Forcibly read default data and update records # force_update = False -# The address of file with list of programs -# program_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/programs.yaml +# The address of file with the official projects list +# project_list_uri = https://git.openstack.org/cgit/openstack/governance/plain/reference/projects.yaml # The address of DriverLog data # driverlog_data_uri = https://git.openstack.org/cgit/stackforge/driverlog/plain/etc/default_data.json diff --git a/etc/test_default_data.json b/etc/test_default_data.json index d370f3668..17107e10e 100644 --- a/etc/test_default_data.json +++ b/etc/test_default_data.json @@ -183,7 +183,7 @@ { "id": "openstack", "title": "OpenStack", - "modules": ["openstack", "openstack-infra", "openstack-dev"] + "modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"] }, { "id": "integrated", @@ -191,12 +191,6 @@ "child": true, "modules": ["official-integrated"] }, - { - "id": "incubated", - "title": "incubated", - "child": true, - "modules": ["official-incubated"] - }, { "id": "other", "title": "other", diff --git a/etc/test_programs.yaml b/etc/test_programs.yaml deleted file mode 100644 index 0d9ea0f54..000000000 --- a/etc/test_programs.yaml +++ /dev/null @@ -1,9 +0,0 @@ -Image Service: - codename: Glance - ptl: Mark Washenberger (markwash) - url: https://wiki.openstack.org/wiki/Glance - projects: - - repo: openstack/glance - integrated-since: havana - incubated-since: grizzly - - repo: openstack/python-glanceclient diff --git a/etc/test_projects.yaml b/etc/test_projects.yaml new file mode 100644 index 000000000..a11b3bc72 --- /dev/null +++ b/etc/test_projects.yaml @@ -0,0 +1,16 @@ +Sahara: + ptl: Sergey Lukjanov (SergeyLukjanov) + mission: > + To provide a scalable data processing stack and associated management + interfaces. + url: https://wiki.openstack.org/wiki/Sahara + projects: + - repo: openstack/sahara + tags: + - name: integrated-release + since: juno + - repo: openstack/python-saharaclient + - repo: openstack/sahara-dashboard + - repo: openstack/sahara-extra + - repo: openstack/sahara-image-elements + - repo: openstack/sahara-specs diff --git a/stackalytics/dashboard/templates/_macros/module_details.html b/stackalytics/dashboard/templates/_macros/module_details.html index dcf76c6a4..8096dfa29 100644 --- a/stackalytics/dashboard/templates/_macros/module_details.html +++ b/stackalytics/dashboard/templates/_macros/module_details.html @@ -28,8 +28,8 @@

${name}

{%if tag == 'program' %} - The official OpenStack program as defined in - programs.yaml + The official OpenStack project as defined in + projects.yaml {%/if%} {%if tag == 'group' %} Custom module group as defined in diff --git a/stackalytics/dashboard/templates/layout.html b/stackalytics/dashboard/templates/layout.html index e07a3d7bc..34d9e0f9b 100644 --- a/stackalytics/dashboard/templates/layout.html +++ b/stackalytics/dashboard/templates/layout.html @@ -54,12 +54,12 @@ Stackalytics {% if page_title %}| {{ page_title }} {% endif %}
- +
- +
diff --git a/stackalytics/dashboard/templates/widget.html b/stackalytics/dashboard/templates/widget.html index 04c650e77..5d4c1fe90 100644 --- a/stackalytics/dashboard/templates/widget.html +++ b/stackalytics/dashboard/templates/widget.html @@ -121,7 +121,7 @@
- +
diff --git a/stackalytics/processor/config.py b/stackalytics/processor/config.py index d98bb68fc..0be405aa6 100644 --- a/stackalytics/processor/config.py +++ b/stackalytics/processor/config.py @@ -45,10 +45,10 @@ OPTS = [ help='SSH username for gerrit review system access'), cfg.BoolOpt('force-update', default=False, help='Forcibly read default data and update records'), - cfg.StrOpt('program-list-uri', + cfg.StrOpt('project-list-uri', default=('https://git.openstack.org/cgit/' - 'openstack/governance/plain/reference/programs.yaml'), - help='The address of file with list of programs'), + 'openstack/governance/plain/reference/projects.yaml'), + help='The address of file with the official projects list'), cfg.StrOpt('driverlog-data-uri', default='https://git.openstack.org/cgit/' 'stackforge/driverlog/plain/etc/default_data.json', diff --git a/stackalytics/processor/main.py b/stackalytics/processor/main.py index 40c3e65d7..5772afb7e 100644 --- a/stackalytics/processor/main.py +++ b/stackalytics/processor/main.py @@ -240,30 +240,24 @@ def apply_corrections(uri, runtime_storage_inst): runtime_storage_inst.apply_corrections(valid_corrections) -def _read_official_programs_yaml(program_list_uri, release_names): - LOG.debug('Process list of programs from uri: %s', program_list_uri) - content = yaml.safe_load(utils.read_uri(program_list_uri)) +def _read_official_projects_yaml(project_list_uri, release_names): + LOG.debug('Process list of projects from uri: %s', project_list_uri) + content = yaml.safe_load(utils.read_uri(project_list_uri)) module_groups = collections.defaultdict( lambda: {'modules': [], 'releases': collections.defaultdict(list)}) official_integrated = module_groups['official-integrated'] official_integrated['tag'] = 'project_type' official_integrated['module_group_name'] = 'official-integrated' - official_incubated = module_groups['official-incubated'] - official_incubated['tag'] = 'project_type' - official_incubated['module_group_name'] = 'official-incubated' official_other = module_groups['official-other'] official_other['tag'] = 'project_type' official_other['module_group_name'] = 'official-other' for name, info in six.iteritems(content): - # for one program - group_id = name.lower() - if 'codename' in info: - name = '%s (%s)' % (info['codename'], name) - group_id = '%s-group' % info['codename'].lower() + # take one official project - module_groups[group_id]['module_group_name'] = name + group_id = '%s-group' % name.lower() + module_groups[group_id]['module_group_name'] = '%s Official' % name module_groups[group_id]['tag'] = 'program' for module in info['projects']: @@ -271,17 +265,22 @@ def _read_official_programs_yaml(program_list_uri, release_names): module_groups[group_id]['modules'].append(module_name) - if ('integrated-since' in module) or ('incubated-since' in module): - project_type = 'official-other' - for release_name in release_names: - if release_name == module.get('incubated-since'): - project_type = 'official-incubated' - elif release_name == module.get('integrated-since'): - project_type = 'official-integrated' + type_matched = False + if 'tags' in module: + for tag in module.get('tags'): + tag_name = tag.get('name') - module_groups[project_type]['releases'][ - release_name].append(module_name) - else: + if tag_name == 'integrated-release': + type_matched = True # project type is matched here + project_type = 'official-other' + for release_name in release_names: + if release_name == tag.get('since'): + project_type = 'official-integrated' + + module_groups[project_type]['releases'][ + release_name].append(module_name) + + if not type_matched: module_groups['official-other']['modules'].append(module_name) # set ids for module groups @@ -291,13 +290,13 @@ def _read_official_programs_yaml(program_list_uri, release_names): return module_groups -def process_program_list(runtime_storage_inst, program_list_uri): +def process_project_list(runtime_storage_inst, project_list_uri): module_groups = runtime_storage_inst.get_by_key('module_groups') or {} release_names = [r['release_name'].lower() for r in runtime_storage_inst.get_by_key('releases')[1:]] - official_module_groups = _read_official_programs_yaml( - program_list_uri, release_names) + official_module_groups = _read_official_projects_yaml( + project_list_uri, release_names) LOG.debug('Update module groups with official: %s', official_module_groups) module_groups.update(official_module_groups) @@ -335,7 +334,7 @@ def main(): default_data, cfg.CONF.driverlog_data_uri) - process_program_list(runtime_storage_inst, cfg.CONF.program_list_uri) + process_project_list(runtime_storage_inst, cfg.CONF.project_list_uri) update_pids(runtime_storage_inst)