Merge "Load subproject data from governance"

requirements.txt

@@ -5,3 +5,4 @@ Babel>=1.3
 pytz>=2010h
 requests>=2.2.0,!=2.4.0
 six>=1.7.0
+PyYAML>=3.1.0

reviewstats/tests/test_projectinfo.py

@@ -10,19 +10,23 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import mock
+
 from reviewstats.tests import base
 from reviewstats import utils
 
 
+@mock.patch('reviewstats.utils.get_remote_data', return_value={})
 class TestProjectInfo(base.TestCase):
 
-    def test_project_definitions_load(self):
+    def test_project_definitions_load(self, get_remote_data):
         utils.get_projects_info('', True)
 
-    def test_get_projects_info_single_name(self):
+    def test_get_projects_info_single_name(self, get_remote_data):
         projects = utils.get_projects_info('nova')
         self.assertEqual(1, len(projects))
 
-    def test_get_projects_info_single_name_projects_prefixed(self):
+    def test_get_projects_info_single_name_projects_prefixed(self,
+                                                             get_remote_data):
         projects = utils.get_projects_info('projects/stable.json')
         self.assertEqual(1, len(projects))

reviewstats/utils.py

@@ -18,6 +18,7 @@
 """
 
 import glob
+import gzip
 import json
 import logging
 import os
@@ -25,12 +26,50 @@ import requests
 import requests.auth
 from six.moves import cPickle as pickle
 import time
+import yaml
 
 import paramiko
+from six.moves import cStringIO
+from six.moves import urllib
 
 LOG = logging.getLogger(__name__)
 
 
+PROJECTS_YAML = ('http://git.openstack.org/cgit/openstack/governance/plain/'
+                 'reference/projects.yaml')
+
+
+class DataRetrievalFailed(Exception):
+    pass
+
+
+# Copied from https://github.com/cybertron/zuul-status/blob/master/app.py
+def get_remote_data(address, datatype='json'):
+    req = urllib.request.Request(address)
+    req.add_header('Accept-encoding', 'gzip')
+    try:
+        remote_data = urllib.request.urlopen(req, timeout=10)
+    except Exception as e:
+        msg = 'Failed to retrieve data from %s: %s' % (address, str(e))
+        raise DataRetrievalFailed(msg)
+    data = ""
+    while True:
+        chunk = remote_data.read()
+        if not chunk:
+            break
+        data += chunk
+
+    if remote_data.info().get('Content-Encoding') == 'gzip':
+        buf = cStringIO.StringIO(data)
+        f = gzip.GzipFile(fileobj=buf)
+        data = f.read()
+
+    if datatype == 'json':
+        return json.loads(data)
+    else:
+        return yaml.safe_load(data)
+
+
 def get_projects_info(project=None, all_projects=False,
                       base_dir='./projects/'):
     """Return the list of project dict objects.
@@ -78,6 +117,16 @@ def get_projects_info(project=None, all_projects=False,
                     raise
                 if not (all_projects and project.get('unofficial')):
                     projects.append(project)
+        # Get base project name
+        project_name = os.path.splitext(os.path.basename(fn))[0]
+        project_data = get_remote_data(PROJECTS_YAML, 'yaml')
+        for name, data in project_data.items():
+            if name == project_name:
+                for d, d_data in data['deliverables'].items():
+                    projects[-1]['subprojects'] += d_data['repos']
+        projects[-1]['subprojects'] = sorted(
+            list(set(projects[-1]['subprojects']))
+            )
 
     return projects
 

test-requirements.txt

@@ -6,3 +6,4 @@ python-subunit
 testrepository>=0.0.17
 testscenarios>=0.4,<0.5
 testtools>=0.9.32
+mock