Move git_requirements plugin to collection
Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/824563 Change-Id: Iabb0386f0d020fbc95ea23701ef2feda01b999f2
This commit is contained in:
Dmitriy Rabotyagov
Dmitriy Rabotyagov
parent
db1555018a
commit
1bb5121e0b
@ -1,335 +0,0 @@
|
||||
#!/usr/bin/python
|
||||
# -*- coding: utf-8 -*-
|
||||
|
||||
from ansible.module_utils.basic import AnsibleModule
|
||||
import git
|
||||
import itertools
|
||||
import multiprocessing
|
||||
import os
|
||||
import signal
|
||||
import time
|
||||
|
||||
DOCUMENTATION = """
|
||||
---
|
||||
module: git_requirements
|
||||
short_description: Module to run a multithreaded git clone
|
||||
|
||||
options:
|
||||
repo_info:
|
||||
description:
|
||||
- List of repo information dictionaries containing at
|
||||
a minimum a key entry "src" with the source git URL
|
||||
to clone for each repo. In these dictionaries, one
|
||||
can further specify:
|
||||
"path" - destination clone location
|
||||
"version" - git version to checkout
|
||||
"refspec" - git refspec to checkout
|
||||
"depth" - clone depth level
|
||||
"shallow_since" - get repo history starting from that date
|
||||
"force" - require git clone uses "--force"
|
||||
default_path:
|
||||
description:
|
||||
Default git clone path (str) in case not
|
||||
specified on an individual repo basis in
|
||||
repo_info. Defaults to "master". Not
|
||||
required.
|
||||
default_version:
|
||||
description:
|
||||
Default git version (str) in case not
|
||||
specified on an individual repo basis in
|
||||
repo_info. Defaults to "master". Not
|
||||
required.
|
||||
default_refspec:
|
||||
description:
|
||||
Default git repo refspec (str) in case not
|
||||
specified on an individual repo basis in
|
||||
repo_info. Defaults to "". Not required.
|
||||
default_depth:
|
||||
description:
|
||||
Default clone depth (int) in case not specified
|
||||
on an individual repo basis. Defaults to None.
|
||||
Not required.
|
||||
default_shallow_since:
|
||||
description:
|
||||
Default shallow date (str) strating from which
|
||||
git history will be fetched. Defaults to None.
|
||||
Has prescedence over depth.
|
||||
Not required.
|
||||
retries:
|
||||
description:
|
||||
Integer number of retries allowed in case of git
|
||||
clone failure. Defaults to 1. Not required.
|
||||
delay:
|
||||
description:
|
||||
Integer time delay (seconds) between git clone
|
||||
retries in case of failure. Defaults to 0. Not
|
||||
required.
|
||||
force:
|
||||
description:
|
||||
Boolean. Apply --force flags to git clones wherever
|
||||
possible. Defaults to False. Not required.
|
||||
core_multiplier:
|
||||
description:
|
||||
Integer multiplier on the number of cores
|
||||
present on the machine to use for
|
||||
multithreading. For example, on a 2 core
|
||||
machine, a multiplier of 4 would use 8
|
||||
threads. Defaults to 4. Not required.
|
||||
"""
|
||||
|
||||
EXAMPLES = r"""
|
||||
|
||||
- name: Clone repos
|
||||
git_requirements:
|
||||
repo_info: "[{'src':'https://github.com/ansible/',
|
||||
'name': 'ansible'
|
||||
'dest': '/etc/opt/ansible'}]"
|
||||
"""
|
||||
|
||||
|
||||
def init_signal():
|
||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
||||
|
||||
|
||||
def check_out_version(repo, version, pull=False, fetch=True, force=False,
|
||||
refspec=None, tag=False, depth=None,
|
||||
shallow_since=None):
|
||||
if fetch:
|
||||
try:
|
||||
if shallow_since:
|
||||
depth = None
|
||||
repo.git.fetch(tags=tag, force=force, refspec=refspec,
|
||||
depth=depth, shallow_since=shallow_since)
|
||||
except Exception as e:
|
||||
return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))]
|
||||
|
||||
try:
|
||||
repo.git.checkout(version, force=force)
|
||||
except Exception as e:
|
||||
return [
|
||||
"Failed to check out version %s for %s\n%s" %
|
||||
(version, repo.working_dir, str(e))]
|
||||
|
||||
if repo.is_dirty(untracked_files=True) and force:
|
||||
try:
|
||||
repo.git.clean(force=force)
|
||||
except Exception as e:
|
||||
return [
|
||||
"Failed to clean up repository% s\n%s" %
|
||||
(repo.working_dir, str(e))]
|
||||
|
||||
if pull:
|
||||
try:
|
||||
repo.git.pull(force=force, refspec=refspec, depth=depth)
|
||||
except Exception as e:
|
||||
return ["Failed to pull repo %s\n%s" % (repo.working_dir, str(e))]
|
||||
return []
|
||||
|
||||
|
||||
def pull_wrapper(info):
|
||||
role_info = info
|
||||
retries = info[1]["retries"]
|
||||
delay = info[1]["delay"]
|
||||
for i in range(retries):
|
||||
success = pull_role(role_info)
|
||||
if success:
|
||||
return True
|
||||
else:
|
||||
time.sleep(delay)
|
||||
info[2].append(["Role {0} failed after {1} retries\n".format(role_info[0],
|
||||
retries)])
|
||||
return False
|
||||
|
||||
|
||||
def pull_role(info):
|
||||
role, config, failures = info
|
||||
|
||||
required_version = role["version"]
|
||||
version_hash = False
|
||||
if 'version' in role:
|
||||
# If the version is the length of a hash then treat is as one
|
||||
if len(required_version) == 40:
|
||||
version_hash = True
|
||||
|
||||
def get_repo(dest):
|
||||
try:
|
||||
return git.Repo(dest)
|
||||
except Exception:
|
||||
failtxt = "Role in {0} is broken/not a git repo.".format(
|
||||
role["dest"])
|
||||
failtxt += "Please delete or fix it manually"
|
||||
failures.append(failtxt)
|
||||
return False
|
||||
|
||||
# if repo exists
|
||||
if os.path.exists(role["dest"]):
|
||||
repo = get_repo(role["dest"])
|
||||
if not repo:
|
||||
return False # go to next role
|
||||
repo_url = list(repo.remote().urls)[0]
|
||||
if repo_url != role["src"]:
|
||||
repo.remote().set_url(role["src"])
|
||||
|
||||
# if they want master then fetch, checkout and pull to stay at latest
|
||||
# master
|
||||
if required_version == "master":
|
||||
fail = check_out_version(repo, required_version, pull=True,
|
||||
force=config["force"],
|
||||
refspec=role["refspec"],
|
||||
depth=role["depth"],
|
||||
shallow_since=role["shallow_since"])
|
||||
|
||||
# If we have a hash then reset it to
|
||||
elif version_hash:
|
||||
fail = check_out_version(repo, required_version,
|
||||
force=config["force"],
|
||||
refspec=role["refspec"],
|
||||
depth=role["depth"],
|
||||
shallow_since=role["shallow_since"])
|
||||
else:
|
||||
# describe can fail in some cases so be careful:
|
||||
try:
|
||||
current_version = repo.git.describe(tags=True)
|
||||
except Exception:
|
||||
current_version = ""
|
||||
if current_version == required_version and not config["force"]:
|
||||
fail = []
|
||||
pass
|
||||
else:
|
||||
fail = check_out_version(repo, required_version,
|
||||
force=config["force"],
|
||||
refspec=role["refspec"],
|
||||
depth=role["depth"],
|
||||
shallow_since=role["shallow_since"],
|
||||
tag=True)
|
||||
|
||||
else:
|
||||
try:
|
||||
# If we have a hash id then treat this a little differently
|
||||
shallow_since = role.get('shallow_since')
|
||||
if shallow_since:
|
||||
depth = None
|
||||
else:
|
||||
depth = role.get('depth')
|
||||
|
||||
if version_hash:
|
||||
repo = git.Repo.clone_from(role["src"], role["dest"],
|
||||
branch='master',
|
||||
no_single_branch=True,
|
||||
depth=depth,
|
||||
shallow_since=shallow_since,)
|
||||
if not repo:
|
||||
return False # go to next role
|
||||
fail = check_out_version(repo, required_version,
|
||||
force=config["force"],
|
||||
refspec=role["refspec"],
|
||||
depth=depth,
|
||||
fetch=False,
|
||||
shallow_since=shallow_since,)
|
||||
else:
|
||||
git.Repo.clone_from(role["src"], role["dest"],
|
||||
branch=required_version,
|
||||
depth=depth,
|
||||
shallow_since=shallow_since,
|
||||
no_single_branch=True)
|
||||
fail = []
|
||||
|
||||
except Exception as e:
|
||||
fail = ('Failed cloning repo %s\n%s' % (role["dest"], str(e)))
|
||||
|
||||
if fail == []:
|
||||
return True
|
||||
else:
|
||||
failures.append(fail)
|
||||
return False
|
||||
|
||||
|
||||
def set_default(dictionary, key, defaults):
|
||||
if key not in dictionary.keys():
|
||||
dictionary[key] = defaults[key]
|
||||
|
||||
|
||||
def main():
|
||||
# Define variables
|
||||
failures = multiprocessing.Manager().list()
|
||||
|
||||
# Data we can pass in to the module
|
||||
fields = {
|
||||
"repo_info": {"required": True, "type": "list"},
|
||||
"default_path": {"required": True,
|
||||
"type": "str"},
|
||||
"default_version": {"required": False,
|
||||
"type": "str",
|
||||
"default": "master"},
|
||||
"default_refspec": {"required": False,
|
||||
"type": "str",
|
||||
"default": None},
|
||||
"default_depth": {"required": False,
|
||||
"type": "int",
|
||||
"default": None},
|
||||
"default_shallow_since": {"required": False,
|
||||
"type": str,
|
||||
"default": None},
|
||||
"retries": {"required": False,
|
||||
"type": "int",
|
||||
"default": 1},
|
||||
"delay": {"required": False,
|
||||
"type": "int",
|
||||
"default": 0},
|
||||
"force": {"required": False,
|
||||
"type": "bool",
|
||||
"default": False},
|
||||
"core_multiplier": {"required": False,
|
||||
"type": "int",
|
||||
"default": 4},
|
||||
|
||||
}
|
||||
|
||||
# Pull in module fields and pass into variables
|
||||
module = AnsibleModule(argument_spec=fields)
|
||||
|
||||
git_repos = module.params['repo_info']
|
||||
defaults = {
|
||||
"path": module.params["default_path"],
|
||||
"depth": module.params["default_depth"],
|
||||
"shallow_since": module.params["default_shallow_since"],
|
||||
"version": module.params["default_version"],
|
||||
"refspec": module.params["default_refspec"]
|
||||
}
|
||||
config = {
|
||||
"retries": module.params["retries"],
|
||||
"delay": module.params["delay"],
|
||||
"force": module.params["force"],
|
||||
"core_multiplier": module.params["core_multiplier"]
|
||||
}
|
||||
|
||||
# Set up defaults
|
||||
for repo in git_repos:
|
||||
for key in ["path", "refspec", "version", "depth", "shallow_since"]:
|
||||
set_default(repo, key, defaults)
|
||||
if "name" not in repo.keys():
|
||||
repo["name"] = os.path.basename(repo["src"])
|
||||
repo["dest"] = os.path.join(repo["path"], repo["name"])
|
||||
|
||||
# Define varibles
|
||||
failures = multiprocessing.Manager().list()
|
||||
core_count = multiprocessing.cpu_count() * config["core_multiplier"]
|
||||
|
||||
# Load up process and pass in interrupt and core process count
|
||||
p = multiprocessing.Pool(core_count, init_signal)
|
||||
|
||||
clone_success = p.map(pull_wrapper, zip(git_repos,
|
||||
itertools.repeat(config),
|
||||
itertools.repeat(failures)),
|
||||
chunksize=1)
|
||||
p.close()
|
||||
|
||||
success = all(i for i in clone_success)
|
||||
if success:
|
||||
module.exit_json(msg=str(git_repos), changed=True)
|
||||
else:
|
||||
module.fail_json(msg=("Module failed"), meta=failures)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@ -137,7 +137,7 @@
|
||||
- name: Clone git repos
|
||||
block:
|
||||
- name: Clone git repos (parallel)
|
||||
git_requirements:
|
||||
openstack.osa.git_requirements:
|
||||
default_path: "{{ role_path_default }}"
|
||||
default_depth: "{{ role_clone_default_depth }}"
|
||||
default_version: "master"
|
||||
|
||||
@ -26,7 +26,7 @@ export ANSIBLE_ROLES_PATH="${ANSIBLE_ROLES_PATH:-/etc/ansible/roles:OSA_PLAYBOOK
|
||||
export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-/etc/ansible}"
|
||||
export ANSIBLE_COLLECTIONS_PATH="${ANSIBLE_COLLECTIONS_PATH:-/etc/ansible}"
|
||||
|
||||
export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-OSA_PLAYBOOK_PATH/library:/etc/ansible/roles/ceph-ansible/library}"
|
||||
export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-/etc/ansible/roles/ceph-ansible/library}"
|
||||
export ANSIBLE_FILTER_PLUGINS="${ANSIBLE_FILTER_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/filter}"
|
||||
export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_ACTION_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/actions}"
|
||||
export ANSIBLE_CALLBACK_PLUGINS="${ANSIBLE_CALLBACK_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/callback}"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user