From 918a3865963f381506c755e4dfff2d92022e002c Mon Sep 17 00:00:00 2001
From: Isaac Beckman <isaacb@il.ibm.com>
Date: Sun, 29 May 2016 13:36:17 +0300
Subject: [PATCH] selective caching of git repos

Adding the ability to selectively cache only some git repos in
nodepool slave.
This is useful in some environments where it is needless to cache
all the git repos that appear in gerrit.
For example, in a 3rd party CI environment, each nodepool slave
needs only few openstack projects in order to check commits to
a specific project. So it is needless to cache > 1500 projects
currently in gerrit

Change-Id: Ie7302594b24d2bdcc157d3cf64b1f219e7ef0205
---
 nodepool/elements/openstack-repos/README.rst  | 20 ++++++++++++++++++-
 .../extra-data.d/50-create-repo-list          | 18 ++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/nodepool/elements/openstack-repos/README.rst b/nodepool/elements/openstack-repos/README.rst
index 82bcfde8cf..0118ecc0e8 100644
--- a/nodepool/elements/openstack-repos/README.rst
+++ b/nodepool/elements/openstack-repos/README.rst
@@ -1 +1,19 @@
-Download all repos and packages that a devstack run might need.
+===============
+openstack-repos
+===============
+
+Download all repos and packages that might be needed.
+
+Environment variables:
+----------------------
+
+DIB_CUSTOM_PROJECTS_LIST_URL
+  :Required: No
+  :Default: None
+  :Description: Url to a yaml file contains custom list of repos.
+    The custom yaml file has the same structure as the default file:
+    'https://git.openstack.org/cgit/openstack-infra/project-config/plain/gerrit/projects.yaml'
+    Download only the repos that appear in the custom file rather than
+    downloading all openstack repos that appear in the default file.
+  :Example:
+    DIB_CUSTOM_PROJECTS_LIST_URL='file:///etc//project-config//gerrit//custom_projects.yaml'
diff --git a/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list b/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
index 5ab8d27479..2465d47a09 100755
--- a/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
+++ b/nodepool/elements/openstack-repos/extra-data.d/50-create-repo-list
@@ -18,6 +18,7 @@
 
 import os
 import urllib2
+from urllib2 import URLError
 import yaml
 
 URL = ('https://git.openstack.org/cgit/openstack-infra/project-config/'
@@ -27,9 +28,24 @@ TMP_HOOKS_PATH=os.environ['TMP_HOOKS_PATH']
 PROJECTS_REPOS=os.path.join(TMP_HOOKS_PATH, 'source-repository-projects-yaml')
 GIT_BASE=os.environ.get('GIT_BASE', 'git://git.openstack.org')
 
+CUSTOM_PROJECTS_LIST_URL=os.environ.get('DIB_CUSTOM_PROJECTS_LIST_URL')
+
+def get_project_list(url):
+    try:
+        projects = [f['project'] for f in yaml.load(urllib2.urlopen(url))]
+        return projects
+    except URLError:
+        print "Could not open project list url: '%s'" % url
+        return None
 
 def main():
-    projects = [f['project'] for f in yaml.load(urllib2.urlopen(URL))]
+    projects = []
+    if CUSTOM_PROJECTS_LIST_URL:
+        projects = get_project_list(CUSTOM_PROJECTS_LIST_URL)
+
+    if not projects:
+        projects = get_project_list(URL)
+
     with open(PROJECTS_REPOS, 'w') as projects_list:
         for project in projects:
             # Skip repos that are inactive