Make superrepo work with zuul-cloner

zuul-cloner will assemble something similar to what gerrit's
user-submodule capability assembles, but it won't have any submodules or
be a git repository itself. We still have all of the information we need
without that, but we can't force the checkout of the version we pass in.

This way we have both methods, which will be useful for developers who
will be checking out the result of a gerrit user-submodule in order to
reproduce a build, but CI will have zuul-cloner assembling the
components that will eventually become a commit to the gerrit
user-submodule maintained repository.

Change-Id: I5f92d4ce79307258d492a9634e9330e15d056ceb

giftwrap/build_spec.py

@@ -48,6 +48,7 @@ class BuildSpec(object):
             for project in self._manifest['projects']:
                 existing_project_names.add(project['name'])
             # Read all dirs with a setup.py as projects
+            try:
                 repo = git.Repo(self._manifest['superrepo'])
                 try:
                     # Try it as a branch
@@ -55,11 +56,13 @@ class BuildSpec(object):
                 except IndexError:
                     # Nope, detach head
                     repo.head.reference = repo.commit(self.version)
-            for subdir in os.listdir(repo.working_tree_dir):
+                working_tree = repo.working_tree_dir
+            except git.InvalidGitRepositoryError:
+                working_tree = self._manifest['superrepo']
+            for subdir in os.listdir(working_tree):
                 # requirements is special
                 if subdir == 'requirements':
-                    reqdir = os.path.join(
+                    reqdir = os.path.join(working_tree, 'requirements')
-                        repo.working_tree_dir, 'requirements')
                     reqfile = os.path.join(reqdir, 'upper-constraints.txt')
                     if reqfile not in self.settings.constraints:
                         self.settings.constraints.append(reqfile)
@@ -67,14 +70,13 @@ class BuildSpec(object):
                 # Skip any projects explicitly in the manifest
                 if subdir in existing_project_names:
                     continue
-                subpath = os.path.join(repo.working_tree_dir, subdir)
+                subpath = os.path.join(working_tree, subdir)
                 if not os.path.exists(os.path.join(subpath, 'setup.py')):
                     continue
                 # skip non git repos since we won't be able to figure out a
                 # version
                 try:
-                    subrepo = git.Repo(os.path.join(repo.working_tree_dir,
+                    subrepo = git.Repo(os.path.join(working_tree, subdir))
-                                                    subdir))
                 except git.exc.InvalidGitRepositoryError:
                     continue
                 project = {}

giftwrap/tests/test_buildspec.py

@@ -16,6 +16,7 @@
 #    under the License.
 
 import os
+import shutil
 import tempfile
 import unittest2 as unittest
 
@@ -70,6 +71,14 @@ class TestBuildSpec(unittest.TestCase):
         repo.index.add(['setup.py'])
         repo.index.commit('adding setup.py')
 
+    def _populate_reqrepo(self, reqrepo, childname):
+        constraints_path = os.path.join(reqrepo, 'upper-constraints.txt')
+        with open(constraints_path, 'w') as cf:
+            cf.write("foo==1.0\n{}==11.0\n".format(childname))
+        reqrepo = git.Repo(reqrepo)
+        reqrepo.index.add(['upper-constraints.txt'])
+        reqrepo.index.commit('adding upper constraints')
+
     @utils.make_test_repo("parentrepo")
     @utils.make_test_repo("childrepo2")
     @utils.make_test_repo("childrepo")
@@ -99,27 +108,63 @@ class TestBuildSpec(unittest.TestCase):
         parentrepo.create_submodule(child2name, child2name,
                                     url=childrepo2.working_tree_dir)
         parentrepo.index.commit('adding child repos')
-        constraints_path = os.path.join(reqrepo, 'upper-constraints.txt')
+        self._populate_reqrepo(reqrepo, childname)
-        with open(constraints_path, 'w') as cf:
-            cf.write("foo==1.0\n{}==11.0\n".format(childname))
-        reqrepo = git.Repo(reqrepo)
-        reqrepo.index.add(['upper-constraints.txt'])
-        reqrepo.index.commit('adding upper constraints')
         parentrepo.create_submodule('requirements', 'requirements',
-                                    url=reqrepo.working_tree_dir)
+                                    url=reqrepo)
-        parenthash = parentrepo.head.commit.hexsha
+        version = parentrepo.head.commit.hexsha
+        self._test_build_spec(version,
+                              parentrepo.working_tree_dir,
+                              childrepo2,
+                              childrepo)
+
+    @utils.make_test_repo("childrepo2")
+    @utils.make_test_repo("childrepo")
+    @utils.make_test_repo("reqrepo")
+    def test_build_spec_superrepo_no_submodules(self,
+                                                childrepo2,
+                                                childrepo,
+                                                reqrepo):
+        parentdir = None
+        try:
+            parentdir = tempfile.mkdtemp()
+            childname = os.path.basename(childrepo)
+            child2name = os.path.basename(childrepo2)
+            newchildrepo = os.path.join(parentdir, childname)
+            newchildrepo2 = os.path.join(parentdir, child2name)
+            newreqrepo = os.path.join(parentdir, 'requirements')
+            os.rename(childrepo, newchildrepo)
+            os.rename(childrepo2, newchildrepo2)
+            os.rename(reqrepo, newreqrepo)
+            newchildrepo = git.Repo(newchildrepo)
+            self._add_setup_py(newchildrepo)
+            newchildrepo2 = git.Repo(newchildrepo2)
+            self._add_setup_py(newchildrepo2)
+            self._populate_reqrepo(newreqrepo, childname)
+            self._test_build_spec('9999', parentdir, newchildrepo2,
+                                  newchildrepo)
+        finally:
+            if parentdir:
+                shutil.rmtree(parentdir)
+
+    def _test_build_spec(self,
+                         version,
+                         working_tree,
+                         childrepo2,
+                         childrepo):
+        childname = os.path.basename(childrepo.working_tree_dir)
+        child2name = os.path.basename(childrepo2.working_tree_dir)
         childhash = childrepo.head.commit.hexsha
         child2hash = childrepo2.head.commit.hexsha
         child2describe = childrepo2.git.describe(always=True)
         manifest = {
             'settings': {},
-            'superrepo': parentrepo.working_tree_dir,
+            'superrepo': working_tree,
         }
         with tempfile.TemporaryFile(mode='w+') as tf:
             yaml.safe_dump(manifest, tf)
             tf.flush()
             tf.seek(0)
-            bs = build_spec.BuildSpec(tf, parenthash)
+            bs = build_spec.BuildSpec(tf, version)
         self.assertEqual(2, len(bs.projects))
         results = {
             childname: {
@@ -133,11 +178,11 @@ class TestBuildSpec(unittest.TestCase):
         }
         for project in bs.projects:
             child_path = os.path.join(
-                parentrepo.working_tree_dir, project.name)
+                working_tree, project.name)
             self.assertEqual(child_path, project.giturl)
             self.assertEqual(results[project.name]['gitref'], project.gitref)
             self.assertEqual(results[project.name]['version'], project.version)
-        constraints_added = os.path.join(parentrepo.working_tree_dir,
+        constraints_added = os.path.join(working_tree,
                                          'requirements',
                                          'upper-constraints.txt')
         self.assertIn(constraints_added, bs.settings.constraints)

giftwrap/tests/utils.py

@@ -35,6 +35,7 @@ def make_test_repo(name='testrepo'):
                 repo.index.commit('test commit')
                 return test(*args, **kwargs)
             finally:
+                if os.path.isdir(testrepo):
                     shutil.rmtree(testrepo)
         return wrapper
     return decorator