Merge "selective caching of git repos"

nodepool/elements/openstack-repos/README.rst

@@ -1 +1,19 @@
-Download all repos and packages that a devstack run might need.
+===============
+openstack-repos
+===============
+
+Download all repos and packages that might be needed.
+
+Environment variables:
+----------------------
+
+DIB_CUSTOM_PROJECTS_LIST_URL
+  :Required: No
+  :Default: None
+  :Description: Url to a yaml file contains custom list of repos.
+    The custom yaml file has the same structure as the default file:
+    'https://git.openstack.org/cgit/openstack-infra/project-config/plain/gerrit/projects.yaml'
+    Download only the repos that appear in the custom file rather than
+    downloading all openstack repos that appear in the default file.
+  :Example:
+    DIB_CUSTOM_PROJECTS_LIST_URL='file:///etc//project-config//gerrit//custom_projects.yaml'

nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list

@@ -18,6 +18,7 @@
 
 import os
 import urllib2
+from urllib2 import URLError
 import yaml
 
 URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
@@ -27,9 +28,24 @@ TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
 PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
 GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
 
+CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
+
+def get_project_list(url):
+    try:
+        projects = [f['project'] for f in yaml.load(urllib2.urlopen(url))]
+        return projects
+    except URLError:
+        print "Could not open project list url: '%s'" % url
+        return None
 
 def main():
-    projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
+    projects = []
+    if CUSTOM_PROJECTS_LIST_URL:
+        projects = get_project_list(CUSTOM_PROJECTS_LIST_URL)
+
+    if not projects:
+        projects = get_project_list(URL)
+
     with open(PROJECTS_REPOS, 'w') as projects_list:
         for project in projects:
             # Skip repos that are inactive