d8d6391b03
In this path of the code we clone then checkout the required refspec. The branch name used during the initial clone does not affect the checkout to the required refspec. Hardcoding the initial clone to master has worked when the default branch name for repos has been 'master'. The migration of default branch names to 'main' breaks this assumption. By not specifying a branch name in the initial clone we avoid problems with master/main migration, or with a repo which legitimately uses a non standard default branch name. Change-Id: I7761c2083eec8081cd719f7b285f2d657066d343
335 lines
11 KiB
Python
335 lines
11 KiB
Python
#!/usr/bin/python
|
|
# -*- coding: utf-8 -*-
|
|
|
|
from ansible.module_utils.basic import AnsibleModule
|
|
import git
|
|
import itertools
|
|
import multiprocessing
|
|
import os
|
|
import signal
|
|
import time
|
|
|
|
DOCUMENTATION = """
|
|
---
|
|
module: git_requirements
|
|
short_description: Module to run a multithreaded git clone
|
|
|
|
options:
|
|
repo_info:
|
|
description:
|
|
- List of repo information dictionaries containing at
|
|
a minimum a key entry "src" with the source git URL
|
|
to clone for each repo. In these dictionaries, one
|
|
can further specify:
|
|
"path" - destination clone location
|
|
"version" - git version to checkout
|
|
"refspec" - git refspec to checkout
|
|
"depth" - clone depth level
|
|
"shallow_since" - get repo history starting from that date
|
|
"force" - require git clone uses "--force"
|
|
default_path:
|
|
description:
|
|
Default git clone path (str) in case not
|
|
specified on an individual repo basis in
|
|
repo_info. Defaults to "master". Not
|
|
required.
|
|
default_version:
|
|
description:
|
|
Default git version (str) in case not
|
|
specified on an individual repo basis in
|
|
repo_info. Defaults to "master". Not
|
|
required.
|
|
default_refspec:
|
|
description:
|
|
Default git repo refspec (str) in case not
|
|
specified on an individual repo basis in
|
|
repo_info. Defaults to "". Not required.
|
|
default_depth:
|
|
description:
|
|
Default clone depth (int) in case not specified
|
|
on an individual repo basis. Defaults to None.
|
|
Not required.
|
|
default_shallow_since:
|
|
description:
|
|
Default shallow date (str) strating from which
|
|
git history will be fetched. Defaults to None.
|
|
Has prescedence over depth.
|
|
Not required.
|
|
retries:
|
|
description:
|
|
Integer number of retries allowed in case of git
|
|
clone failure. Defaults to 1. Not required.
|
|
delay:
|
|
description:
|
|
Integer time delay (seconds) between git clone
|
|
retries in case of failure. Defaults to 0. Not
|
|
required.
|
|
force:
|
|
description:
|
|
Boolean. Apply --force flags to git clones wherever
|
|
possible. Defaults to False. Not required.
|
|
core_multiplier:
|
|
description:
|
|
Integer multiplier on the number of cores
|
|
present on the machine to use for
|
|
multithreading. For example, on a 2 core
|
|
machine, a multiplier of 4 would use 8
|
|
threads. Defaults to 4. Not required.
|
|
"""
|
|
|
|
EXAMPLES = r"""
|
|
|
|
- name: Clone repos
|
|
git_requirements:
|
|
repo_info: "[{'src':'https://github.com/ansible/',
|
|
'name': 'ansible'
|
|
'dest': '/etc/opt/ansible'}]"
|
|
"""
|
|
|
|
|
|
def init_signal():
|
|
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
|
|
|
|
def check_out_version(repo, version, pull=False, fetch=True, force=False,
|
|
refspec=None, tag=False, depth=None,
|
|
shallow_since=None):
|
|
if fetch:
|
|
try:
|
|
if shallow_since:
|
|
depth = None
|
|
repo.git.fetch(tags=tag, force=force, refspec=refspec,
|
|
depth=depth, shallow_since=shallow_since)
|
|
except Exception as e:
|
|
return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))]
|
|
|
|
try:
|
|
repo.git.checkout(version, force=force)
|
|
except Exception as e:
|
|
return [
|
|
"Failed to check out version %s for %s\n%s" %
|
|
(version, repo.working_dir, str(e))]
|
|
|
|
if repo.is_dirty(untracked_files=True) and force:
|
|
try:
|
|
repo.git.clean(force=force)
|
|
except Exception as e:
|
|
return [
|
|
"Failed to clean up repository% s\n%s" %
|
|
(repo.working_dir, str(e))]
|
|
|
|
if pull:
|
|
try:
|
|
repo.git.pull(force=force, refspec=refspec, depth=depth)
|
|
except Exception as e:
|
|
return ["Failed to pull repo %s\n%s" % (repo.working_dir, str(e))]
|
|
return []
|
|
|
|
|
|
def pull_wrapper(info):
|
|
role_info = info
|
|
retries = info[1]["retries"]
|
|
delay = info[1]["delay"]
|
|
for i in range(retries):
|
|
success = pull_role(role_info)
|
|
if success:
|
|
return True
|
|
else:
|
|
time.sleep(delay)
|
|
info[2].append(["Role {0} failed after {1} retries\n".format(role_info[0],
|
|
retries)])
|
|
return False
|
|
|
|
|
|
def pull_role(info):
|
|
role, config, failures = info
|
|
|
|
required_version = role["version"]
|
|
version_hash = False
|
|
if 'version' in role:
|
|
# If the version is the length of a hash then treat is as one
|
|
if len(required_version) == 40:
|
|
version_hash = True
|
|
|
|
def get_repo(dest):
|
|
try:
|
|
return git.Repo(dest)
|
|
except Exception:
|
|
failtxt = "Role in {0} is broken/not a git repo.".format(
|
|
role["dest"])
|
|
failtxt += "Please delete or fix it manually"
|
|
failures.append(failtxt)
|
|
return False
|
|
|
|
# if repo exists
|
|
if os.path.exists(role["dest"]):
|
|
repo = get_repo(role["dest"])
|
|
if not repo:
|
|
return False # go to next role
|
|
repo_url = list(repo.remote().urls)[0]
|
|
if repo_url != role["src"]:
|
|
repo.remote().set_url(role["src"])
|
|
|
|
# if they want master then fetch, checkout and pull to stay at latest
|
|
# master
|
|
if required_version == "master":
|
|
fail = check_out_version(repo, required_version, pull=True,
|
|
force=config["force"],
|
|
refspec=role["refspec"],
|
|
depth=role["depth"],
|
|
shallow_since=role["shallow_since"])
|
|
|
|
# If we have a hash then reset it to
|
|
elif version_hash:
|
|
fail = check_out_version(repo, required_version,
|
|
force=config["force"],
|
|
refspec=role["refspec"],
|
|
depth=role["depth"],
|
|
shallow_since=role["shallow_since"])
|
|
else:
|
|
# describe can fail in some cases so be careful:
|
|
try:
|
|
current_version = repo.git.describe(tags=True)
|
|
except Exception:
|
|
current_version = ""
|
|
if current_version == required_version and not config["force"]:
|
|
fail = []
|
|
pass
|
|
else:
|
|
fail = check_out_version(repo, required_version,
|
|
force=config["force"],
|
|
refspec=role["refspec"],
|
|
depth=role["depth"],
|
|
shallow_since=role["shallow_since"],
|
|
tag=True)
|
|
|
|
else:
|
|
try:
|
|
# If we have a hash id then treat this a little differently
|
|
shallow_since = role.get('shallow_since')
|
|
if shallow_since:
|
|
depth = None
|
|
else:
|
|
depth = role.get('depth')
|
|
|
|
if version_hash:
|
|
repo = git.Repo.clone_from(role["src"], role["dest"],
|
|
no_single_branch=True,
|
|
depth=depth,
|
|
shallow_since=shallow_since,)
|
|
if not repo:
|
|
return False # go to next role
|
|
fail = check_out_version(repo, required_version,
|
|
force=config["force"],
|
|
refspec=role["refspec"],
|
|
depth=depth,
|
|
fetch=False,
|
|
shallow_since=shallow_since,)
|
|
else:
|
|
git.Repo.clone_from(role["src"], role["dest"],
|
|
branch=required_version,
|
|
depth=depth,
|
|
shallow_since=shallow_since,
|
|
no_single_branch=True)
|
|
fail = []
|
|
|
|
except Exception as e:
|
|
fail = ('Failed cloning repo %s\n%s' % (role["dest"], str(e)))
|
|
|
|
if fail == []:
|
|
return True
|
|
else:
|
|
failures.append(fail)
|
|
return False
|
|
|
|
|
|
def set_default(dictionary, key, defaults):
|
|
if key not in dictionary.keys():
|
|
dictionary[key] = defaults[key]
|
|
|
|
|
|
def main():
|
|
# Define variables
|
|
failures = multiprocessing.Manager().list()
|
|
|
|
# Data we can pass in to the module
|
|
fields = {
|
|
"repo_info": {"required": True, "type": "list"},
|
|
"default_path": {"required": True,
|
|
"type": "str"},
|
|
"default_version": {"required": False,
|
|
"type": "str",
|
|
"default": "master"},
|
|
"default_refspec": {"required": False,
|
|
"type": "str",
|
|
"default": None},
|
|
"default_depth": {"required": False,
|
|
"type": "int",
|
|
"default": None},
|
|
"default_shallow_since": {"required": False,
|
|
"type": str,
|
|
"default": None},
|
|
"retries": {"required": False,
|
|
"type": "int",
|
|
"default": 1},
|
|
"delay": {"required": False,
|
|
"type": "int",
|
|
"default": 0},
|
|
"force": {"required": False,
|
|
"type": "bool",
|
|
"default": False},
|
|
"core_multiplier": {"required": False,
|
|
"type": "int",
|
|
"default": 4},
|
|
|
|
}
|
|
|
|
# Pull in module fields and pass into variables
|
|
module = AnsibleModule(argument_spec=fields)
|
|
|
|
git_repos = module.params['repo_info']
|
|
defaults = {
|
|
"path": module.params["default_path"],
|
|
"depth": module.params["default_depth"],
|
|
"shallow_since": module.params["default_shallow_since"],
|
|
"version": module.params["default_version"],
|
|
"refspec": module.params["default_refspec"]
|
|
}
|
|
config = {
|
|
"retries": module.params["retries"],
|
|
"delay": module.params["delay"],
|
|
"force": module.params["force"],
|
|
"core_multiplier": module.params["core_multiplier"]
|
|
}
|
|
|
|
# Set up defaults
|
|
for repo in git_repos:
|
|
for key in ["path", "refspec", "version", "depth", "shallow_since"]:
|
|
set_default(repo, key, defaults)
|
|
if "name" not in repo.keys():
|
|
repo["name"] = os.path.basename(repo["src"])
|
|
repo["dest"] = os.path.join(repo["path"], repo["name"])
|
|
|
|
# Define varibles
|
|
failures = multiprocessing.Manager().list()
|
|
core_count = multiprocessing.cpu_count() * config["core_multiplier"]
|
|
|
|
# Load up process and pass in interrupt and core process count
|
|
p = multiprocessing.Pool(core_count, init_signal)
|
|
|
|
clone_success = p.map(pull_wrapper, zip(git_repos,
|
|
itertools.repeat(config),
|
|
itertools.repeat(failures)),
|
|
chunksize=1)
|
|
p.close()
|
|
|
|
success = all(i for i in clone_success)
|
|
if success:
|
|
module.exit_json(msg=str(git_repos), changed=True)
|
|
else:
|
|
module.fail_json(msg=("Module failed"), meta=failures)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|