108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
import logging
|
|
import os
|
|
import sys
|
|
|
|
from concurrent import futures
|
|
import git
|
|
|
|
from fuel_ccp import config
|
|
|
|
|
|
CONF = config.CONF
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
FETCH_TIMEOUT = 2 ** 16 # in seconds
|
|
|
|
ALREADY_EXISTED_STATUS = 'AlreadyExist'
|
|
CLONE_FAILED_STATUS = 'CloneFailure'
|
|
CHECKOUT_FAILED_STATUS = 'CheckoutFailed'
|
|
FETCH_SUCCEEDED_STATUS = 'Success'
|
|
|
|
|
|
def fetch_repository(repository_def):
|
|
name = repository_def['name']
|
|
dest_dir = os.path.join(CONF.repositories.path, name)
|
|
result = {'name': name, 'status': FETCH_SUCCEEDED_STATUS}
|
|
if os.path.isdir(dest_dir):
|
|
LOG.debug('%s: Repository is already cloned, skipping', name)
|
|
result["status"] = ALREADY_EXISTED_STATUS
|
|
return result
|
|
git_url = repository_def['git_url']
|
|
git_ref = repository_def.get('git_ref')
|
|
LOG.debug('%s: Cloning repository %s to %s', name, git_url, dest_dir)
|
|
try:
|
|
repo = git.Repo.clone_from(git_url, dest_dir)
|
|
except git.exc.GitCommandError:
|
|
LOG.error('%s: Repository does not exist', name)
|
|
result["status"] = CLONE_FAILED_STATUS
|
|
return result
|
|
if git_ref and repo:
|
|
LOG.debug('%s: Changing reference to "%s"', name, git_ref)
|
|
try:
|
|
repo.git.checkout(git_ref)
|
|
except git.exc.GitCommandError:
|
|
LOG.error('%s: Failed to checkout %s', name, git_ref)
|
|
result["status"] = CHECKOUT_FAILED_STATUS
|
|
return result
|
|
LOG.info('%s: Repository has been cloned', name)
|
|
return result
|
|
|
|
|
|
def _get_summary(fetch_info):
|
|
LOG.info('#' * 50)
|
|
LOG.info('Summary:')
|
|
fetch_succeeded = [info['name'] for info in fetch_info
|
|
if info['status'] == FETCH_SUCCEEDED_STATUS]
|
|
clone_failed = [info['name'] for info in fetch_info
|
|
if info['status'] == CLONE_FAILED_STATUS]
|
|
checkout_failed = [info['name'] for info in fetch_info
|
|
if info['status'] == CHECKOUT_FAILED_STATUS]
|
|
alredy_existed = [info['name'] for info in fetch_info
|
|
if info['status'] == ALREADY_EXISTED_STATUS]
|
|
if fetch_succeeded:
|
|
LOG.info('%d repository(s) cloning succeeded: %s' % (
|
|
len(fetch_succeeded), ', '.join(fetch_succeeded)))
|
|
if alredy_existed:
|
|
LOG.info('%d repository(s) is(are) already cloned: %s' % (
|
|
len(alredy_existed), ', '.join(alredy_existed)))
|
|
if clone_failed:
|
|
LOG.info('%d repository(s) cloning failed: %s' % (
|
|
len(clone_failed), ', '.join(clone_failed)))
|
|
if checkout_failed:
|
|
LOG.info('%d repository(s) checkout failed: %s' % (
|
|
len(checkout_failed), ', '.join(checkout_failed)))
|
|
LOG.info('#' * 50)
|
|
if clone_failed or checkout_failed:
|
|
return False
|
|
return True
|
|
|
|
|
|
def fetch_repositories(repository_defs=None):
|
|
if repository_defs is None:
|
|
repository_defs = CONF.repositories.repos
|
|
|
|
LOG.info('Cloning repositories into %s', CONF.repositories.path)
|
|
|
|
with futures.ThreadPoolExecutor(
|
|
max_workers=CONF.repositories.clone_concurrency) as executor:
|
|
future_list = []
|
|
try:
|
|
for repository_def in repository_defs:
|
|
future_list.append(executor.submit(
|
|
fetch_repository, repository_def
|
|
))
|
|
fetch_statuses = []
|
|
for future in future_list:
|
|
# we need to use timeout because in this case python
|
|
# thread wakes up time to time to check timeout and don't
|
|
# block signal processing
|
|
fetch_statuses.append(future.result(timeout=FETCH_TIMEOUT))
|
|
except SystemExit:
|
|
for future in future_list:
|
|
future.cancel()
|
|
raise
|
|
fetch_succeeded = _get_summary(fetch_statuses)
|
|
if not fetch_succeeded:
|
|
sys.exit(1)
|