Nodepool: cache all gerrit repos

Cache all review.openstack.org git repos in /opt/git.

Subsequent changes to gerrit-git-prep and devstack-gate will make
use of these repos (eventually deprecating the devstack-gate repos
is ~jenkins/workspace-cache).

Refactor the devstack cache python script a bit to make common
functions reusable.  Also rename it because underscores are better
than hyphens for python file names.

Change-Id: I4ae40613ebcca62b0dfc2eaba159557a4c71a529

modules/openstack_project/files/nodepool/scripts/cache_devstack.py

@@ -18,24 +18,13 @@
 
 import os
 import sys
-import subprocess
+
+from common import run_local
 
 DEVSTACK = os.path.expanduser('~/workspace-cache/devstack')
 CACHEDIR = os.path.expanduser('~/cache/files')
 
 
-def run_local(cmd, status=False, cwd='.', env={}):
-    print "Running:", cmd
-    newenv = os.environ
-    newenv.update(env)
-    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
-                         stderr=subprocess.STDOUT, env=newenv)
-    (out, nothing) = p.communicate()
-    if status:
-        return (p.returncode, out.strip())
-    return out.strip()
-
-
 def git_branches():
     branches = []
     for branch in run_local(['git', 'branch', '-a'], cwd=DEVSTACK).split("\n"):

modules/openstack_project/files/nodepool/scripts/cache_git_repos.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import urllib2
+
+from common import run_local
+
+URL = ('http://git.openstack.org/cgit/openstack-infra/config/plain/'
+       'modules/openstack_project/templates/review.projects.yaml.erb')
+PROJECT_RE = re.compile('^-?\s+project:\s+(.*)$')
+
+
+def main():
+    # TODO(jeblair): use gerrit rest api when available
+    data = urllib2.urlopen(URL).read()
+    for line in data.split('\n'):
+        # We're regex-parsing YAML so that we don't have to depend on the
+        # YAML module which is not in the stdlib.
+        m = PROJECT_RE.match(line)
+        if m:
+            project = 'https://git.openstack.org/%s' % m.group(1)
+            print run_local(['git', 'clone', project, m.group(1)],
+                            cwd='/opt/git')
+
+
+if __name__ == '__main__':
+    main()

modules/openstack_project/files/nodepool/scripts/common.py

@@ -0,0 +1,32 @@
+#!/usr/bin/env python
+
+# Copyright (C) 2011-2013 OpenStack Foundation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import subprocess
+
+
+def run_local(cmd, status=False, cwd='.', env={}):
+    print "Running:", cmd
+    newenv = os.environ
+    newenv.update(env)
+    p = subprocess.Popen(cmd, stdout=subprocess.PIPE, cwd=cwd,
+                         stderr=subprocess.STDOUT, env=newenv)
+    (out, nothing) = p.communicate()
+    if status:
+        return (p.returncode, out.strip())
+    return out.strip()

modules/openstack_project/files/nodepool/scripts/prepare_devstack.sh

@@ -72,7 +72,7 @@ git clone https://review.openstack.org/p/stackforge/wsme
 DISTRIB_CODENAME=`lsb_release -sc`
 
 cd /opt/nodepool-scripts/
-python ./devstack-cache.py $DISTRIB_CODENAME
+python ./cache_devstack.py $DISTRIB_CODENAME
 
 sync
 sleep 5

modules/openstack_project/files/nodepool/scripts/prepare_node.sh

@@ -31,3 +31,6 @@ else
     sudo puppet apply --modulepath=/root/config/modules:/etc/puppet/modules \
 	-e "class {'openstack_project::slave_template': install_users => false, ssh_key => '$NODEPOOL_SSH_KEY', }"
 fi
+
+sudo mkdir -p /opt/git
+sudo -i python /opt/nodepool-scripts/cache_git_repos.py

modules/openstack_project/files/nodepool/scripts/prepare_tripleo.sh

@@ -46,7 +46,7 @@ bash tripleo-incubator/scripts/pull-tools
 #
 #. /etc/lsb-release
 #cd /opt/nodepool-scripts/
-#python ./devstack-cache.py $DISTRIB_CODENAME
+#python ./cache_devstack.py $DISTRIB_CODENAME
 
 sync
 sleep 5