Add shallow_since to parallel git clone

shallow_since controls from which date history repo will be shallowed.
It has prescedence over depth, so when shallow_since is set for the repo
depth won't be used. This is handy
for some repos, like ceph-ansible, that
merge a lot of changes in bunches, which results in bootstrap issues.

Later, shallow_since will be managed with osa CLI and bump will
update this date for the repos in a-r-r.

Change-Id: I8017a01853e8f8950e6e8c7c2337e384af8d7689

playbooks/library/git_requirements.py

@@ -25,6 +25,7 @@ options:
         "version" - git version to checkout
         "refspec" - git refspec to checkout
         "depth" - clone depth level
+        "shallow_since" - get repo history starting from that date
         "force" - require git clone uses "--force"
   default_path:
     description:
@@ -46,7 +47,13 @@ options:
   default_depth:
     description:
       Default clone depth (int) in case not specified
-      on an individual repo basis. Defaults to 10.
+      on an individual repo basis. Defaults to None.
+      Not required.
+  default_shallow_since:
+    description:
+      Default shallow date (str) strating from which
+      git history will be fetched. Defaults to None.
+      Has prescedence over depth.
       Not required.
   retries:
      description:
@@ -84,12 +91,17 @@ def init_signal():
     signal.signal(signal.SIGINT, signal.SIG_IGN)
 
 
-def check_out_version(repo, version, pull=False, force=False,
+def check_out_version(repo, version, pull=False, fetch=True, force=False,
-                      refspec=None, tag=False, depth=10):
+                      refspec=None, tag=False, depth=None,
-    try:
+                      shallow_since=None):
-        repo.git.fetch(tags=tag, force=force, refspec=refspec, depth=depth)
+    if fetch:
-    except Exception as e:
+        try:
-        return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))]
+            if shallow_since:
+                depth = None
+            repo.git.fetch(tags=tag, force=force, refspec=refspec,
+                           depth=depth, shallow_since=shallow_since)
+        except Exception as e:
+            return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))]
 
     try:
         repo.git.checkout(version, force=force)
@@ -164,14 +176,16 @@ def pull_role(info):
             fail = check_out_version(repo, required_version, pull=True,
                                      force=config["force"],
                                      refspec=role["refspec"],
-                                     depth=role["depth"])
+                                     depth=role["depth"],
+                                     shallow_since=role["shallow_since"])
 
         # If we have a hash then reset it to
         elif version_hash:
             fail = check_out_version(repo, required_version,
                                      force=config["force"],
                                      refspec=role["refspec"],
-                                     depth=role["depth"])
+                                     depth=role["depth"],
+                                     shallow_since=role["shallow_since"])
         else:
             # describe can fail in some cases so be careful:
             try:
@@ -186,27 +200,37 @@ def pull_role(info):
                                          force=config["force"],
                                          refspec=role["refspec"],
                                          depth=role["depth"],
+                                         shallow_since=role["shallow_since"],
                                          tag=True)
 
     else:
         try:
             # If we have a hash id then treat this a little differently
+            shallow_since = role.get('shallow_since')
+            if shallow_since:
+                depth = None
+            else:
+                depth = role.get('depth')
+
             if version_hash:
-                git.Repo.clone_from(role["src"], role["dest"],
+                repo = git.Repo.clone_from(role["src"], role["dest"],
-                                    branch='master',
+                                           branch='master',
-                                    no_single_branch=True,
+                                           no_single_branch=True,
-                                    depth=role["depth"])
+                                           depth=depth,
-                repo = get_repo(role["dest"])
+                                           shallow_since=shallow_since,)
                 if not repo:
                     return False  # go to next role
                 fail = check_out_version(repo, required_version,
                                          force=config["force"],
                                          refspec=role["refspec"],
-                                         depth=role["depth"])
+                                         depth=depth,
+                                         fetch=False,
+                                         shallow_since=shallow_since,)
             else:
                 git.Repo.clone_from(role["src"], role["dest"],
                                     branch=required_version,
-                                    depth=role["depth"],
+                                    depth=depth,
+                                    shallow_since=shallow_since,
                                     no_single_branch=True)
                 fail = []
 
@@ -242,7 +266,10 @@ def main():
                             "default": None},
         "default_depth": {"required": False,
                           "type": "int",
-                          "default": 10},
+                          "default": None},
+        "default_shallow_since": {"required": False,
+                                  "type": str,
+                                  "default": None},
         "retries": {"required": False,
                     "type": "int",
                     "default": 1},
@@ -265,6 +292,7 @@ def main():
     defaults = {
         "path": module.params["default_path"],
         "depth": module.params["default_depth"],
+        "shallow_since": module.params["default_shallow_since"],
         "version": module.params["default_version"],
         "refspec": module.params["default_refspec"]
     }
@@ -277,7 +305,7 @@ def main():
 
     # Set up defaults
     for repo in git_repos:
-        for key in ["path", "refspec", "version", "depth"]:
+        for key in ["path", "refspec", "version", "depth", "shallow_since"]:
             set_default(repo, key, defaults)
         if "name" not in repo.keys():
             repo["name"] = os.path.basename(repo["src"])

scripts/get-ansible-role-requirements.yml

@@ -139,7 +139,7 @@
         - name: Clone git repos (parallel)
           git_requirements:
             default_path: "{{ role_path_default }}"
-            default_depth: 20
+            default_depth: "{{ role_clone_default_depth }}"
             default_version: "master"
             repo_info: "{{ clone_roles }}"
             retries: "{{ git_clone_retries }}"
@@ -153,7 +153,7 @@
             dest: "{{ item.path | default(role_path_default) }}/{{ item.name | default(item.src | basename) }}"
             version: "{{ item.version | default('master') }}"
             refspec: "{{ item.refspec | default(omit) }}"
-            depth: "{{ item.depth | default('20') }}"
+            depth: "{{ item.depth | default(role_clone_default_depth| default(omit)) }}"
             update: true
             force: true
           with_items: "{{ clone_roles }}"
@@ -172,3 +172,4 @@
     user_role_path: "{{ lookup('env', 'OSA_CONFIG_DIR') | default('/etc/openstack_deploy', true) ~ '/' ~ (user_role_file|default('')) }}"
     git_clone_retries: 2
     git_clone_retry_delay: 5
+    role_clone_default_depth: 20