Move git_requirements plugin to collection
Depends-On: https://review.opendev.org/c/openstack/openstack-ansible-plugins/+/824563 Change-Id: Iabb0386f0d020fbc95ea23701ef2feda01b999f2
This commit is contained in:
Dmitriy Rabotyagov
Dmitriy Rabotyagov
parent
db1555018a
commit
1bb5121e0b
@ -1,335 +0,0 @@
|
|||||||
#!/usr/bin/python
|
|
||||||
# -*- coding: utf-8 -*-
|
|
||||||
|
|
||||||
from ansible.module_utils.basic import AnsibleModule
|
|
||||||
import git
|
|
||||||
import itertools
|
|
||||||
import multiprocessing
|
|
||||||
import os
|
|
||||||
import signal
|
|
||||||
import time
|
|
||||||
|
|
||||||
DOCUMENTATION = """
|
|
||||||
---
|
|
||||||
module: git_requirements
|
|
||||||
short_description: Module to run a multithreaded git clone
|
|
||||||
|
|
||||||
options:
|
|
||||||
repo_info:
|
|
||||||
description:
|
|
||||||
- List of repo information dictionaries containing at
|
|
||||||
a minimum a key entry "src" with the source git URL
|
|
||||||
to clone for each repo. In these dictionaries, one
|
|
||||||
can further specify:
|
|
||||||
"path" - destination clone location
|
|
||||||
"version" - git version to checkout
|
|
||||||
"refspec" - git refspec to checkout
|
|
||||||
"depth" - clone depth level
|
|
||||||
"shallow_since" - get repo history starting from that date
|
|
||||||
"force" - require git clone uses "--force"
|
|
||||||
default_path:
|
|
||||||
description:
|
|
||||||
Default git clone path (str) in case not
|
|
||||||
specified on an individual repo basis in
|
|
||||||
repo_info. Defaults to "master". Not
|
|
||||||
required.
|
|
||||||
default_version:
|
|
||||||
description:
|
|
||||||
Default git version (str) in case not
|
|
||||||
specified on an individual repo basis in
|
|
||||||
repo_info. Defaults to "master". Not
|
|
||||||
required.
|
|
||||||
default_refspec:
|
|
||||||
description:
|
|
||||||
Default git repo refspec (str) in case not
|
|
||||||
specified on an individual repo basis in
|
|
||||||
repo_info. Defaults to "". Not required.
|
|
||||||
default_depth:
|
|
||||||
description:
|
|
||||||
Default clone depth (int) in case not specified
|
|
||||||
on an individual repo basis. Defaults to None.
|
|
||||||
Not required.
|
|
||||||
default_shallow_since:
|
|
||||||
description:
|
|
||||||
Default shallow date (str) strating from which
|
|
||||||
git history will be fetched. Defaults to None.
|
|
||||||
Has prescedence over depth.
|
|
||||||
Not required.
|
|
||||||
retries:
|
|
||||||
description:
|
|
||||||
Integer number of retries allowed in case of git
|
|
||||||
clone failure. Defaults to 1. Not required.
|
|
||||||
delay:
|
|
||||||
description:
|
|
||||||
Integer time delay (seconds) between git clone
|
|
||||||
retries in case of failure. Defaults to 0. Not
|
|
||||||
required.
|
|
||||||
force:
|
|
||||||
description:
|
|
||||||
Boolean. Apply --force flags to git clones wherever
|
|
||||||
possible. Defaults to False. Not required.
|
|
||||||
core_multiplier:
|
|
||||||
description:
|
|
||||||
Integer multiplier on the number of cores
|
|
||||||
present on the machine to use for
|
|
||||||
multithreading. For example, on a 2 core
|
|
||||||
machine, a multiplier of 4 would use 8
|
|
||||||
threads. Defaults to 4. Not required.
|
|
||||||
"""
|
|
||||||
|
|
||||||
EXAMPLES = r"""
|
|
||||||
|
|
||||||
- name: Clone repos
|
|
||||||
git_requirements:
|
|
||||||
repo_info: "[{'src':'https://github.com/ansible/',
|
|
||||||
'name': 'ansible'
|
|
||||||
'dest': '/etc/opt/ansible'}]"
|
|
||||||
"""
|
|
||||||
|
|
||||||
|
|
||||||
def init_signal():
|
|
||||||
signal.signal(signal.SIGINT, signal.SIG_IGN)
|
|
||||||
|
|
||||||
|
|
||||||
def check_out_version(repo, version, pull=False, fetch=True, force=False,
|
|
||||||
refspec=None, tag=False, depth=None,
|
|
||||||
shallow_since=None):
|
|
||||||
if fetch:
|
|
||||||
try:
|
|
||||||
if shallow_since:
|
|
||||||
depth = None
|
|
||||||
repo.git.fetch(tags=tag, force=force, refspec=refspec,
|
|
||||||
depth=depth, shallow_since=shallow_since)
|
|
||||||
except Exception as e:
|
|
||||||
return ["Failed to fetch %s\n%s" % (repo.working_dir, str(e))]
|
|
||||||
|
|
||||||
try:
|
|
||||||
repo.git.checkout(version, force=force)
|
|
||||||
except Exception as e:
|
|
||||||
return [
|
|
||||||
"Failed to check out version %s for %s\n%s" %
|
|
||||||
(version, repo.working_dir, str(e))]
|
|
||||||
|
|
||||||
if repo.is_dirty(untracked_files=True) and force:
|
|
||||||
try:
|
|
||||||
repo.git.clean(force=force)
|
|
||||||
except Exception as e:
|
|
||||||
return [
|
|
||||||
"Failed to clean up repository% s\n%s" %
|
|
||||||
(repo.working_dir, str(e))]
|
|
||||||
|
|
||||||
if pull:
|
|
||||||
try:
|
|
||||||
repo.git.pull(force=force, refspec=refspec, depth=depth)
|
|
||||||
except Exception as e:
|
|
||||||
return ["Failed to pull repo %s\n%s" % (repo.working_dir, str(e))]
|
|
||||||
return []
|
|
||||||
|
|
||||||
|
|
||||||
def pull_wrapper(info):
|
|
||||||
role_info = info
|
|
||||||
retries = info[1]["retries"]
|
|
||||||
delay = info[1]["delay"]
|
|
||||||
for i in range(retries):
|
|
||||||
success = pull_role(role_info)
|
|
||||||
if success:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
time.sleep(delay)
|
|
||||||
info[2].append(["Role {0} failed after {1} retries\n".format(role_info[0],
|
|
||||||
retries)])
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def pull_role(info):
|
|
||||||
role, config, failures = info
|
|
||||||
|
|
||||||
required_version = role["version"]
|
|
||||||
version_hash = False
|
|
||||||
if 'version' in role:
|
|
||||||
# If the version is the length of a hash then treat is as one
|
|
||||||
if len(required_version) == 40:
|
|
||||||
version_hash = True
|
|
||||||
|
|
||||||
def get_repo(dest):
|
|
||||||
try:
|
|
||||||
return git.Repo(dest)
|
|
||||||
except Exception:
|
|
||||||
failtxt = "Role in {0} is broken/not a git repo.".format(
|
|
||||||
role["dest"])
|
|
||||||
failtxt += "Please delete or fix it manually"
|
|
||||||
failures.append(failtxt)
|
|
||||||
return False
|
|
||||||
|
|
||||||
# if repo exists
|
|
||||||
if os.path.exists(role["dest"]):
|
|
||||||
repo = get_repo(role["dest"])
|
|
||||||
if not repo:
|
|
||||||
return False # go to next role
|
|
||||||
repo_url = list(repo.remote().urls)[0]
|
|
||||||
if repo_url != role["src"]:
|
|
||||||
repo.remote().set_url(role["src"])
|
|
||||||
|
|
||||||
# if they want master then fetch, checkout and pull to stay at latest
|
|
||||||
# master
|
|
||||||
if required_version == "master":
|
|
||||||
fail = check_out_version(repo, required_version, pull=True,
|
|
||||||
force=config["force"],
|
|
||||||
refspec=role["refspec"],
|
|
||||||
depth=role["depth"],
|
|
||||||
shallow_since=role["shallow_since"])
|
|
||||||
|
|
||||||
# If we have a hash then reset it to
|
|
||||||
elif version_hash:
|
|
||||||
fail = check_out_version(repo, required_version,
|
|
||||||
force=config["force"],
|
|
||||||
refspec=role["refspec"],
|
|
||||||
depth=role["depth"],
|
|
||||||
shallow_since=role["shallow_since"])
|
|
||||||
else:
|
|
||||||
# describe can fail in some cases so be careful:
|
|
||||||
try:
|
|
||||||
current_version = repo.git.describe(tags=True)
|
|
||||||
except Exception:
|
|
||||||
current_version = ""
|
|
||||||
if current_version == required_version and not config["force"]:
|
|
||||||
fail = []
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
fail = check_out_version(repo, required_version,
|
|
||||||
force=config["force"],
|
|
||||||
refspec=role["refspec"],
|
|
||||||
depth=role["depth"],
|
|
||||||
shallow_since=role["shallow_since"],
|
|
||||||
tag=True)
|
|
||||||
|
|
||||||
else:
|
|
||||||
try:
|
|
||||||
# If we have a hash id then treat this a little differently
|
|
||||||
shallow_since = role.get('shallow_since')
|
|
||||||
if shallow_since:
|
|
||||||
depth = None
|
|
||||||
else:
|
|
||||||
depth = role.get('depth')
|
|
||||||
|
|
||||||
if version_hash:
|
|
||||||
repo = git.Repo.clone_from(role["src"], role["dest"],
|
|
||||||
branch='master',
|
|
||||||
no_single_branch=True,
|
|
||||||
depth=depth,
|
|
||||||
shallow_since=shallow_since,)
|
|
||||||
if not repo:
|
|
||||||
return False # go to next role
|
|
||||||
fail = check_out_version(repo, required_version,
|
|
||||||
force=config["force"],
|
|
||||||
refspec=role["refspec"],
|
|
||||||
depth=depth,
|
|
||||||
fetch=False,
|
|
||||||
shallow_since=shallow_since,)
|
|
||||||
else:
|
|
||||||
git.Repo.clone_from(role["src"], role["dest"],
|
|
||||||
branch=required_version,
|
|
||||||
depth=depth,
|
|
||||||
shallow_since=shallow_since,
|
|
||||||
no_single_branch=True)
|
|
||||||
fail = []
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
fail = ('Failed cloning repo %s\n%s' % (role["dest"], str(e)))
|
|
||||||
|
|
||||||
if fail == []:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
failures.append(fail)
|
|
||||||
return False
|
|
||||||
|
|
||||||
|
|
||||||
def set_default(dictionary, key, defaults):
|
|
||||||
if key not in dictionary.keys():
|
|
||||||
dictionary[key] = defaults[key]
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
|
||||||
# Define variables
|
|
||||||
failures = multiprocessing.Manager().list()
|
|
||||||
|
|
||||||
# Data we can pass in to the module
|
|
||||||
fields = {
|
|
||||||
"repo_info": {"required": True, "type": "list"},
|
|
||||||
"default_path": {"required": True,
|
|
||||||
"type": "str"},
|
|
||||||
"default_version": {"required": False,
|
|
||||||
"type": "str",
|
|
||||||
"default": "master"},
|
|
||||||
"default_refspec": {"required": False,
|
|
||||||
"type": "str",
|
|
||||||
"default": None},
|
|
||||||
"default_depth": {"required": False,
|
|
||||||
"type": "int",
|
|
||||||
"default": None},
|
|
||||||
"default_shallow_since": {"required": False,
|
|
||||||
"type": str,
|
|
||||||
"default": None},
|
|
||||||
"retries": {"required": False,
|
|
||||||
"type": "int",
|
|
||||||
"default": 1},
|
|
||||||
"delay": {"required": False,
|
|
||||||
"type": "int",
|
|
||||||
"default": 0},
|
|
||||||
"force": {"required": False,
|
|
||||||
"type": "bool",
|
|
||||||
"default": False},
|
|
||||||
"core_multiplier": {"required": False,
|
|
||||||
"type": "int",
|
|
||||||
"default": 4},
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
# Pull in module fields and pass into variables
|
|
||||||
module = AnsibleModule(argument_spec=fields)
|
|
||||||
|
|
||||||
git_repos = module.params['repo_info']
|
|
||||||
defaults = {
|
|
||||||
"path": module.params["default_path"],
|
|
||||||
"depth": module.params["default_depth"],
|
|
||||||
"shallow_since": module.params["default_shallow_since"],
|
|
||||||
"version": module.params["default_version"],
|
|
||||||
"refspec": module.params["default_refspec"]
|
|
||||||
}
|
|
||||||
config = {
|
|
||||||
"retries": module.params["retries"],
|
|
||||||
"delay": module.params["delay"],
|
|
||||||
"force": module.params["force"],
|
|
||||||
"core_multiplier": module.params["core_multiplier"]
|
|
||||||
}
|
|
||||||
|
|
||||||
# Set up defaults
|
|
||||||
for repo in git_repos:
|
|
||||||
for key in ["path", "refspec", "version", "depth", "shallow_since"]:
|
|
||||||
set_default(repo, key, defaults)
|
|
||||||
if "name" not in repo.keys():
|
|
||||||
repo["name"] = os.path.basename(repo["src"])
|
|
||||||
repo["dest"] = os.path.join(repo["path"], repo["name"])
|
|
||||||
|
|
||||||
# Define varibles
|
|
||||||
failures = multiprocessing.Manager().list()
|
|
||||||
core_count = multiprocessing.cpu_count() * config["core_multiplier"]
|
|
||||||
|
|
||||||
# Load up process and pass in interrupt and core process count
|
|
||||||
p = multiprocessing.Pool(core_count, init_signal)
|
|
||||||
|
|
||||||
clone_success = p.map(pull_wrapper, zip(git_repos,
|
|
||||||
itertools.repeat(config),
|
|
||||||
itertools.repeat(failures)),
|
|
||||||
chunksize=1)
|
|
||||||
p.close()
|
|
||||||
|
|
||||||
success = all(i for i in clone_success)
|
|
||||||
if success:
|
|
||||||
module.exit_json(msg=str(git_repos), changed=True)
|
|
||||||
else:
|
|
||||||
module.fail_json(msg=("Module failed"), meta=failures)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
|
||||||
main()
|
|
||||||
@ -137,7 +137,7 @@
|
|||||||
- name: Clone git repos
|
- name: Clone git repos
|
||||||
block:
|
block:
|
||||||
- name: Clone git repos (parallel)
|
- name: Clone git repos (parallel)
|
||||||
git_requirements:
|
openstack.osa.git_requirements:
|
||||||
default_path: "{{ role_path_default }}"
|
default_path: "{{ role_path_default }}"
|
||||||
default_depth: "{{ role_clone_default_depth }}"
|
default_depth: "{{ role_clone_default_depth }}"
|
||||||
default_version: "master"
|
default_version: "master"
|
||||||
|
|||||||
@ -26,7 +26,7 @@ export ANSIBLE_ROLES_PATH="${ANSIBLE_ROLES_PATH:-/etc/ansible/roles:OSA_PLAYBOOK
|
|||||||
export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-/etc/ansible}"
|
export ANSIBLE_COLLECTIONS_PATHS="${ANSIBLE_COLLECTIONS_PATHS:-/etc/ansible}"
|
||||||
export ANSIBLE_COLLECTIONS_PATH="${ANSIBLE_COLLECTIONS_PATH:-/etc/ansible}"
|
export ANSIBLE_COLLECTIONS_PATH="${ANSIBLE_COLLECTIONS_PATH:-/etc/ansible}"
|
||||||
|
|
||||||
export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-OSA_PLAYBOOK_PATH/library:/etc/ansible/roles/ceph-ansible/library}"
|
export ANSIBLE_LIBRARY="${ANSIBLE_LIBRARY:-/etc/ansible/roles/ceph-ansible/library}"
|
||||||
export ANSIBLE_FILTER_PLUGINS="${ANSIBLE_FILTER_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/filter}"
|
export ANSIBLE_FILTER_PLUGINS="${ANSIBLE_FILTER_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/filter}"
|
||||||
export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_ACTION_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/actions}"
|
export ANSIBLE_ACTION_PLUGINS="${ANSIBLE_ACTION_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/actions}"
|
||||||
export ANSIBLE_CALLBACK_PLUGINS="${ANSIBLE_CALLBACK_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/callback}"
|
export ANSIBLE_CALLBACK_PLUGINS="${ANSIBLE_CALLBACK_PLUGINS:-/etc/ansible/roles/ceph-ansible/plugins/callback}"
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user