Load subproject data from governance

Instead of manually curating a list of all the subprojects in a
project, we should use the governance repo that already includes
most, if not all, of this information.

For the moment, this change just adds the governance data to the
data already in the json files. This is because I'm not sure whether
there is a 1:1 relationship between what is in governance and what
is in the json files. It's possible we could eliminate the json files
entirely at some point, but for the moment this gets us updated
subproject lists for essentially free, since it's likely that many
projects' json files are missing subprojects added in the past few
years.

Change-Id: Ia3992dd30f83294a96ace155691e50838cd5543b

requirements.txt

@@ -5,3 +5,4 @@ Babel>=1.3
 pytz>=2010h
 requests>=2.2.0,!=2.4.0
 six>=1.7.0
+PyYAML>=3.1.0

reviewstats/tests/test_projectinfo.py

@@ -10,19 +10,23 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import mock
+
 from reviewstats.tests import base
 from reviewstats import utils
 
 
+@mock.patch('reviewstats.utils.get_remote_data', return_value={})
 class TestProjectInfo(base.TestCase):
 
-    def test_project_definitions_load(self):
+    def test_project_definitions_load(self, get_remote_data):
         utils.get_projects_info('', True)
 
-    def test_get_projects_info_single_name(self):
+    def test_get_projects_info_single_name(self, get_remote_data):
         projects = utils.get_projects_info('nova')
         self.assertEqual(1, len(projects))
 
-    def test_get_projects_info_single_name_projects_prefixed(self):
+    def test_get_projects_info_single_name_projects_prefixed(self,
+                                                             get_remote_data):
         projects = utils.get_projects_info('projects/stable.json')
         self.assertEqual(1, len(projects))

reviewstats/utils.py

@@ -18,6 +18,7 @@
 """
 
 import glob
+import gzip
 import json
 import logging
 import os
@@ -25,12 +26,50 @@ import requests
 import requests.auth
 from six.moves import cPickle as pickle
 import time
+import yaml
 
 import paramiko
+from six.moves import cStringIO
+from six.moves import urllib
 
 LOG = logging.getLogger(__name__)
 
 
+PROJECTS_YAML = ('http://git.openstack.org/cgit/openstack/governance/plain/'
+                 'reference/projects.yaml')
+
+
+class DataRetrievalFailed(Exception):
+    pass
+
+
+# Copied from https://github.com/cybertron/zuul-status/blob/master/app.py
+def get_remote_data(address, datatype='json'):
+    req = urllib.request.Request(address)
+    req.add_header('Accept-encoding', 'gzip')
+    try:
+        remote_data = urllib.request.urlopen(req, timeout=10)
+    except Exception as e:
+        msg = 'Failed to retrieve data from %s: %s' % (address, str(e))
+        raise DataRetrievalFailed(msg)
+    data = ""
+    while True:
+        chunk = remote_data.read()
+        if not chunk:
+            break
+        data += chunk
+
+    if remote_data.info().get('Content-Encoding') == 'gzip':
+        buf = cStringIO.StringIO(data)
+        f = gzip.GzipFile(fileobj=buf)
+        data = f.read()
+
+    if datatype == 'json':
+        return json.loads(data)
+    else:
+        return yaml.safe_load(data)
+
+
 def get_projects_info(project=None, all_projects=False,
                       base_dir='./projects/'):
     """Return the list of project dict objects.
@@ -78,6 +117,16 @@ def get_projects_info(project=None, all_projects=False,
                     raise
                 if not (all_projects and project.get('unofficial')):
                     projects.append(project)
+        # Get base project name
+        project_name = os.path.splitext(os.path.basename(fn))[0]
+        project_data = get_remote_data(PROJECTS_YAML, 'yaml')
+        for name, data in project_data.items():
+            if name == project_name:
+                for d, d_data in data['deliverables'].items():
+                    projects[-1]['subprojects'] += d_data['repos']
+        projects[-1]['subprojects'] = sorted(
+            list(set(projects[-1]['subprojects']))
+            )
 
     return projects
 

test-requirements.txt

@@ -6,3 +6,4 @@ python-subunit
 testrepository>=0.0.17
 testscenarios>=0.4,<0.5
 testtools>=0.9.32
+mock