x/stackalytics
Code Issues Proposed changes

Support new format of official projects.yaml

Browse Source 
This patch makes support of latest schema of project.yaml
(introduced by https://review.openstack.org/#/c/202583/)

Change-Id: I3307300711638cd33de9420b8efd7635417926ab
This commit is contained in:
Ilya Shakhat
2015-07-31 17:47:50 +03:00
parent 844f1eb934
commit c563296c17
7 changed files with 195 additions and 91 deletions
Download Patch File Download Diff File Expand all files Collapse all files

64
stackalytics/processor/main.py
View File

@@ -13,19 +13,16 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import collections
from oslo_config import cfg
from oslo_log import log as logging
import psutil
import six
from six.moves.urllib import parse
import yaml
from stackalytics.processor import bps
from stackalytics.processor import config
from stackalytics.processor import default_data_processor
from stackalytics.processor import driverlog
from stackalytics.processor import governance
from stackalytics.processor import lp
from stackalytics.processor import mls
from stackalytics.processor import mps
@@ -35,7 +32,6 @@ from stackalytics.processor import runtime_storage
from stackalytics.processor import utils
from stackalytics.processor import vcs
LOG = logging.getLogger(__name__)
@@ -242,66 +238,10 @@ def apply_corrections(uri, runtime_storage_inst):
runtime_storage_inst.apply_corrections(valid_corrections)
def _read_official_projects_yaml(project_list_uri, release_names):
LOG.debug('Process list of projects from uri: %s', project_list_uri)
content = yaml.safe_load(utils.read_uri(project_list_uri))
module_groups = collections.defaultdict(
lambda: {'modules': [], 'releases': collections.defaultdict(list)})
official_integrated = module_groups['official-integrated']
official_integrated['tag'] = 'project_type'
official_integrated['module_group_name'] = 'official-integrated'
official_other = module_groups['official-other']
official_other['tag'] = 'project_type'
official_other['module_group_name'] = 'official-other'
for name, info in six.iteritems(content):
# take one official project
group_id = '%s-group' % name.lower()
module_groups[group_id]['module_group_name'] = '%s Official' % name
module_groups[group_id]['tag'] = 'program'
for module in info['projects']:
repo_split = module['repo'].split('/')
if len(repo_split) < 2:
continue # valid repo must be in form of 'org/module'
module_name = repo_split[1]
module_groups[group_id]['modules'].append(module_name)
type_matched = False
if 'tags' in module:
for tag in module.get('tags'):
tag_name = tag.get('name')
if tag_name == 'integrated-release':
type_matched = True # project type is matched here
project_type = 'official-other'
for release_name in release_names:
if release_name == tag.get('since'):
project_type = 'official-integrated'
module_groups[project_type]['releases'][
release_name].append(module_name)
if not type_matched:
module_groups['official-other']['modules'].append(module_name)
# set ids for module groups
for group_id, group in six.iteritems(module_groups):
group['id'] = group_id
return module_groups
def process_project_list(runtime_storage_inst, project_list_uri):
module_groups = runtime_storage_inst.get_by_key('module_groups') or {}
release_names = [r['release_name'].lower()
for r in runtime_storage_inst.get_by_key('releases')[1:]]
official_module_groups = _read_official_projects_yaml(
project_list_uri, release_names)
official_module_groups = governance.read_projects_yaml(project_list_uri)
LOG.debug('Update module groups with official: %s', official_module_groups)
module_groups.update(official_module_groups)