Load subproject data from governance

Instead of manually curating a list of all the subprojects in a
project, we should use the governance repo that already includes
most, if not all, of this information.

For the moment, this change just adds the governance data to the
data already in the json files. This is because I'm not sure whether
there is a 1:1 relationship between what is in governance and what
is in the json files. It's possible we could eliminate the json files
entirely at some point, but for the moment this gets us updated
subproject lists for essentially free, since it's likely that many
projects' json files are missing subprojects added in the past few
years.

Change-Id: Ia3992dd30f83294a96ace155691e50838cd5543b
This commit is contained in:
Ben Nemec 2019-04-16 15:12:28 +00:00 committed by Natal Ngétal
parent 1407bba3ee
commit 39b1e03047
4 changed files with 58 additions and 3 deletions

View File

@ -5,3 +5,4 @@ Babel>=1.3
pytz>=2010h
requests>=2.2.0,!=2.4.0
six>=1.7.0
PyYAML>=3.1.0

View File

@ -10,19 +10,23 @@
# License for the specific language governing permissions and limitations
# under the License.
import mock
from reviewstats.tests import base
from reviewstats import utils
@mock.patch('reviewstats.utils.get_remote_data', return_value={})
class TestProjectInfo(base.TestCase):
def test_project_definitions_load(self):
def test_project_definitions_load(self, get_remote_data):
utils.get_projects_info('', True)
def test_get_projects_info_single_name(self):
def test_get_projects_info_single_name(self, get_remote_data):
projects = utils.get_projects_info('nova')
self.assertEqual(1, len(projects))
def test_get_projects_info_single_name_projects_prefixed(self):
def test_get_projects_info_single_name_projects_prefixed(self,
get_remote_data):
projects = utils.get_projects_info('projects/stable.json')
self.assertEqual(1, len(projects))

View File

@ -18,6 +18,7 @@
"""
import glob
import gzip
import json
import logging
import os
@ -25,12 +26,50 @@ import requests
import requests.auth
from six.moves import cPickle as pickle
import time
import yaml
import paramiko
from six.moves import cStringIO
from six.moves import urllib
LOG = logging.getLogger(__name__)
PROJECTS_YAML = ('http://git.openstack.org/cgit/openstack/governance/plain/'
'reference/projects.yaml')
class DataRetrievalFailed(Exception):
pass
# Copied from https://github.com/cybertron/zuul-status/blob/master/app.py
def get_remote_data(address, datatype='json'):
req = urllib.request.Request(address)
req.add_header('Accept-encoding', 'gzip')
try:
remote_data = urllib.request.urlopen(req, timeout=10)
except Exception as e:
msg = 'Failed to retrieve data from %s: %s' % (address, str(e))
raise DataRetrievalFailed(msg)
data = ""
while True:
chunk = remote_data.read()
if not chunk:
break
data += chunk
if remote_data.info().get('Content-Encoding') == 'gzip':
buf = cStringIO.StringIO(data)
f = gzip.GzipFile(fileobj=buf)
data = f.read()
if datatype == 'json':
return json.loads(data)
else:
return yaml.safe_load(data)
def get_projects_info(project=None, all_projects=False,
base_dir='./projects/'):
"""Return the list of project dict objects.
@ -78,6 +117,16 @@ def get_projects_info(project=None, all_projects=False,
raise
if not (all_projects and project.get('unofficial')):
projects.append(project)
# Get base project name
project_name = os.path.splitext(os.path.basename(fn))[0]
project_data = get_remote_data(PROJECTS_YAML, 'yaml')
for name, data in project_data.items():
if name == project_name:
for d, d_data in data['deliverables'].items():
projects[-1]['subprojects'] += d_data['repos']
projects[-1]['subprojects'] = sorted(
list(set(projects[-1]['subprojects']))
)
return projects

View File

@ -6,3 +6,4 @@ python-subunit
testrepository>=0.0.17
testscenarios>=0.4,<0.5
testtools>=0.9.32
mock