x/stackalytics
Code Issues Proposed changes

Support new format of official projects.yaml

Browse Source 
This patch makes support of latest schema of project.yaml
(introduced by https://review.openstack.org/#/c/202583/)

Change-Id: I3307300711638cd33de9420b8efd7635417926ab
This commit is contained in:
Ilya Shakhat 2015-07-31 17:47:50 +03:00
parent 844f1eb934
commit c563296c17
7 changed files with 195 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
etc/default_data.json
View File

@ -14141,19 +14141,13 @@
{
"id": "openstack",
"title": "OpenStack",
"modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
"modules": ["tc-approved-release", "governance", "api-wg"]
},
{
"id": "integrated",
"title": "integrated",
"id": "tc-approved-release",
"title": "tc-approved",
"child": true,
"modules": ["official-integrated"]
},
{
"id": "other",
"title": "other",
"child": true,
"modules": ["official-other"]
"modules": ["tc-approved-release"]
},
{
"id": "stackforge",

18
etc/test_default_data.json
View File

@ -168,6 +168,10 @@
{
"release_name": "Kilo",
"end_date": "2015-Apr-30"
},
{
"release_name": "Liberty",
"end_date": "2015-Oct-15"
}
],
@ -183,19 +187,13 @@
{
"id": "openstack",
"title": "OpenStack",
"modules": ["official-integrated", "official-other", "governance", "python-openstackclient", "api-wg"]
"modules": ["tc-approved-release", "governance", "python-openstackclient", "api-wg"]
},
{
"id": "integrated",
"title": "integrated",
"id": "tc-approved-release",
"title": "tc-approved",
"child": true,
"modules": ["official-integrated"]
},
{
"id": "other",
"title": "other",
"child": true,
"modules": ["official-other"]
"modules": ["tc-approved-release"]
},
{
"id": "stackforge",

40
etc/test_projects.yaml
View File

@ -1,16 +1,38 @@
Sahara:
ptl: Sergey Lukjanov (SergeyLukjanov)
irc-channel: openstack-sahara
service: Data processing service
mission: >
To provide a scalable data processing stack and associated management
interfaces.
url: https://wiki.openstack.org/wiki/Sahara
projects:
- repo: openstack/sahara
deliverables:
python-saharaclient:
repos:
- openstack/python-saharaclient
tags:
- name: integrated-release
since: juno
- repo: openstack/python-saharaclient
- repo: openstack/sahara-dashboard
- repo: openstack/sahara-extra
- repo: openstack/sahara-image-elements
- repo: openstack/sahara-specs
- release:cycle-with-intermediary
- release:has-stable-branches
- type:library
- release:managed
- vulnerability:managed
sahara:
repos:
- openstack/sahara
- openstack/sahara-extra
- openstack/sahara-image-elements
tags:
- tc-approved-release
- release:managed
- release:cycle-with-milestones
- release:has-stable-branches
- type:service
- vulnerability:managed
sahara-dashboard:
repos:
- openstack/sahara-dashboard
tags:
- type:library
sahara-specs:
repos:
- openstack/sahara-specs

62
stackalytics/processor/governance.py Normal file
View File

@ -0,0 +1,62 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from oslo_log import log as logging
import six
import yaml
from stackalytics.processor import utils
LOG = logging.getLogger(__name__)
TAGS = ['tc-approved-release'] # list of supported tags
def read_projects_yaml(project_list_uri):
LOG.debug('Process list of projects from uri: %s', project_list_uri)
content = yaml.safe_load(utils.read_uri(project_list_uri))
module_groups = collections.defaultdict(lambda: {'modules': []})
for tag in TAGS:
m = module_groups[tag] # object created by defaultdict
m['tag'] = 'project_type'
m['module_group_name'] = tag
for name, project in six.iteritems(content):
group_id = '%s-group' % name.lower()
module_groups[group_id]['module_group_name'] = '%s Official' % name
module_groups[group_id]['tag'] = 'program'
for d_name, deliverable in six.iteritems(project['deliverables']):
for repo in deliverable['repos']:
repo_split = repo.split('/')
if len(repo_split) < 2:
continue # valid repo must be in form of 'org/module'
module_name = repo_split[1]
module_groups[group_id]['modules'].append(module_name)
tags = deliverable.get('tags', [])
for tag in tags:
if tag in TAGS:
module_groups[tag]['modules'].append(module_name)
# set ids for module groups
for group_id, group in six.iteritems(module_groups):
group['id'] = group_id
group['modules'].sort()
return module_groups

64
stackalytics/processor/main.py
View File

@ -13,19 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from oslo_config import cfg
from oslo_log import log as logging
import psutil
import six
from six.moves.urllib import parse
import yaml
from stackalytics.processor import bps
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
from stackalytics.processor import driverlog
from stackalytics.processor import governance
from stackalytics.processor import lp
from stackalytics.processor import mls
from stackalytics.processor import mps
@ -35,7 +32,6 @@ from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
from stackalytics.processor import vcs
LOG = logging.getLogger(__name__)
@ -242,66 +238,10 @@ def apply_corrections(uri, runtime_storage_inst):
runtime_storage_inst.apply_corrections(valid_corrections)
def _read_official_projects_yaml(project_list_uri, release_names):
LOG.debug('Process list of projects from uri: %s', project_list_uri)
content = yaml.safe_load(utils.read_uri(project_list_uri))
module_groups = collections.defaultdict(
lambda: {'modules': [], 'releases': collections.defaultdict(list)})
official_integrated = module_groups['official-integrated']
official_integrated['tag'] = 'project_type'
official_integrated['module_group_name'] = 'official-integrated'
official_other = module_groups['official-other']
official_other['tag'] = 'project_type'
official_other['module_group_name'] = 'official-other'
for name, info in six.iteritems(content):
# take one official project
group_id = '%s-group' % name.lower()
module_groups[group_id]['module_group_name'] = '%s Official' % name
module_groups[group_id]['tag'] = 'program'
for module in info['projects']:
repo_split = module['repo'].split('/')
if len(repo_split) < 2:
continue # valid repo must be in form of 'org/module'
module_name = repo_split[1]
module_groups[group_id]['modules'].append(module_name)
type_matched = False
if 'tags' in module:
for tag in module.get('tags'):
tag_name = tag.get('name')
if tag_name == 'integrated-release':
type_matched = True # project type is matched here
project_type = 'official-other'
for release_name in release_names:
if release_name == tag.get('since'):
project_type = 'official-integrated'
module_groups[project_type]['releases'][
release_name].append(module_name)
if not type_matched:
module_groups['official-other']['modules'].append(module_name)
# set ids for module groups
for group_id, group in six.iteritems(module_groups):
group['id'] = group_id
return module_groups
def process_project_list(runtime_storage_inst, project_list_uri):
module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
release_names = [r['release_name'].lower()
for r in runtime_storage_inst.get_by_key('releases')[1:]]
official_module_groups = _read_official_projects_yaml(
project_list_uri, release_names)
official_module_groups = governance.read_projects_yaml(project_list_uri)
LOG.debug('Update module groups with official: %s', official_module_groups)
module_groups.update(official_module_groups)

1
stackalytics/processor/rcs.py
View File

@ -126,6 +126,7 @@ class Gerrit(Rcs):
start_id=0, last_id=0, is_open=False,
grab_comments=False):
sort_key = start_id
last_id = last_id or 0
while True:
cmd = self._get_cmd(project_organization, module, branch, sort_key,

87
tests/unit/test_governance.py Normal file
View File

@ -0,0 +1,87 @@
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import mock
import testtools
from stackalytics.processor import governance
SAMPLE = """
Sahara:
ptl: Sergey Lukjanov (SergeyLukjanov)
irc-channel: openstack-sahara
service: Data processing service
mission: >
To provide a scalable data processing stack and associated management
interfaces.
url: https://wiki.openstack.org/wiki/Sahara
deliverables:
python-saharaclient:
repos:
- openstack/python-saharaclient
tags:
- release:cycle-with-intermediary
- release:has-stable-branches
- type:library
- release:managed
- vulnerability:managed
sahara:
repos:
- openstack/sahara
- openstack/sahara-extra
- openstack/sahara-image-elements
tags:
- tc-approved-release
- release:managed
- release:cycle-with-milestones
- release:has-stable-branches
- type:service
- vulnerability:managed
sahara-dashboard:
repos:
- openstack/sahara-dashboard
tags:
- type:library
sahara-specs:
repos:
- openstack/sahara-specs
"""
class TestGovernance(testtools.TestCase):
@mock.patch('stackalytics.processor.utils.read_uri')
def test_read_official_projects_yaml(self, read_uri):
read_uri.return_value = SAMPLE
expected = {
'sahara-group': {
'id': 'sahara-group',
'module_group_name': 'Sahara Official',
'modules': ['python-saharaclient', 'sahara',
'sahara-dashboard', 'sahara-extra',
'sahara-image-elements', 'sahara-specs'],
'tag': 'program'
},
'tc-approved-release': {
'id': 'tc-approved-release',
'module_group_name': 'tc-approved-release',
'modules': ['sahara', 'sahara-extra', 'sahara-image-elements'],
'tag': 'project_type'
}
}
actual = governance.read_projects_yaml('uri')
self.assertEqual(expected, actual)