Cache status of stages in manage-projects
Implement a trivial caching system that will allow us to skip even attempting gerrit and github remote operations for projects that we've processed successfully in previous passes. This should hopefully reduce the amount of time we spend processing the projects.yaml file. In the future, we could extend this to tracking more specific actions- such as the description set in github, so that we could know whether or not they have changed and thus need to be processed again. In this form, acls files will always be processed. Change-Id: I07b13c8663e6f9ee1255a4e56caf556ea49fb51b
This commit is contained in:
parent
961da2b3b4
commit
1c7264f925
@ -52,6 +52,7 @@
|
||||
|
||||
import argparse
|
||||
import ConfigParser
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
@ -288,12 +289,31 @@ def make_ssh_wrapper(gerrit_user, gerrit_key):
|
||||
|
||||
def create_update_github_project(
|
||||
default_has_issues, default_has_downloads, default_has_wiki,
|
||||
github_secure_config, options, project, description, homepage):
|
||||
github_secure_config, options, project, description, homepage,
|
||||
project_cache):
|
||||
created = False
|
||||
has_issues = 'has-issues' in options or default_has_issues
|
||||
has_downloads = 'has-downloads' in options or default_has_downloads
|
||||
has_wiki = 'has-wiki' in options or default_has_wiki
|
||||
|
||||
needs_update = False
|
||||
if not project_cache[project].get('created-in-github', False):
|
||||
needs_update = True
|
||||
if not project_cache[project].get('gerrit-in-team', False):
|
||||
needs_update = True
|
||||
if project_cache[project].get('description') != description:
|
||||
needs_update = True
|
||||
if project_cache[project].get('homepage') != homepage:
|
||||
needs_update = True
|
||||
if project_cache[project].get('has_issues') != has_issues:
|
||||
needs_update = True
|
||||
if project_cache[project].get('has_downloads') != has_downloads:
|
||||
needs_update = True
|
||||
if project_cache[project].get('has_wiki') != has_wiki:
|
||||
needs_update = True
|
||||
if not needs_update:
|
||||
return False
|
||||
|
||||
secure_config = ConfigParser.ConfigParser()
|
||||
secure_config.read(github_secure_config)
|
||||
|
||||
@ -302,6 +322,7 @@ def create_update_github_project(
|
||||
else:
|
||||
ghub = github.Github(secure_config.get("github", "username"),
|
||||
secure_config.get("github", "password"))
|
||||
|
||||
orgs = ghub.get_user().get_orgs()
|
||||
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
|
||||
|
||||
@ -324,14 +345,19 @@ def create_update_github_project(
|
||||
# If necessary, update project on Github
|
||||
if description and description != repo.description:
|
||||
repo.edit(repo_name, description=description)
|
||||
project_cache[project]['description'] = description
|
||||
if homepage and homepage != repo.homepage:
|
||||
repo.edit(repo_name, homepage=homepage)
|
||||
project_cache[project]['homepage'] = homepage
|
||||
if has_issues != repo.has_issues:
|
||||
repo.edit(repo_name, has_issues=has_issues)
|
||||
project_cache[project]['has_issues'] = has_issues
|
||||
if has_downloads != repo.has_downloads:
|
||||
repo.edit(repo_name, has_downloads=has_downloads)
|
||||
project_cache[project]['has_downloads'] = has_downloads
|
||||
if has_wiki != repo.has_wiki:
|
||||
repo.edit(repo_name, has_wiki=has_wiki)
|
||||
project_cache[project]['has_wiki'] = has_wiki
|
||||
|
||||
except github.GithubException:
|
||||
repo = org.create_repo(repo_name,
|
||||
@ -339,19 +365,28 @@ def create_update_github_project(
|
||||
has_issues=has_issues,
|
||||
has_downloads=has_downloads,
|
||||
has_wiki=has_wiki)
|
||||
project_cache[project]['created-in-github'] = True
|
||||
project_cache[project]['has_wiki'] = has_wiki
|
||||
project_cache[project]['has_downloads'] = has_downloads
|
||||
project_cache[project]['has_issues'] = has_issues
|
||||
|
||||
if description:
|
||||
repo.edit(repo_name, description=description)
|
||||
project_cache[project]['description'] = description
|
||||
if homepage:
|
||||
repo.edit(repo_name, homepage=homepage)
|
||||
project_cache[project]['homepage'] = homepage
|
||||
repo.edit(repo_name, has_issues=has_issues,
|
||||
has_downloads=has_downloads,
|
||||
has_wiki=has_wiki)
|
||||
created = True
|
||||
|
||||
if project_cache[project].get('gerrit-in-team', False):
|
||||
if 'gerrit' not in [team.name for team in repo.get_teams()]:
|
||||
teams = org.get_teams()
|
||||
teams_dict = dict(zip([t.name.lower() for t in teams], teams))
|
||||
teams_dict['gerrit'].add_to_repos(repo)
|
||||
created = True
|
||||
project_cache[project]['gerrit-in-team'] = True
|
||||
|
||||
return created
|
||||
|
||||
@ -606,6 +641,10 @@ def main():
|
||||
GITHUB_SECURE_CONFIG = registry.get_defaults(
|
||||
'github-config',
|
||||
'/etc/github/github-projects.secure.config')
|
||||
PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache')
|
||||
project_cache = {}
|
||||
if os.path.exists(PROJECT_CACHE_FILE):
|
||||
project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read())
|
||||
|
||||
gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST,
|
||||
GERRIT_USER,
|
||||
@ -647,12 +686,17 @@ def main():
|
||||
acl_config = section.get(
|
||||
'acl-config',
|
||||
'%s.config' % os.path.join(ACL_DIR, project))
|
||||
project_cache.setdefault(project, {})
|
||||
|
||||
# Create the project in Gerrit first, since it will fail
|
||||
# spectacularly if its project directory or local replica
|
||||
# already exist on disk
|
||||
project_created = project_cache[project].get(
|
||||
'project-created', False)
|
||||
if not project_created:
|
||||
project_created = create_gerrit_project(
|
||||
project, project_list, gerrit)
|
||||
project_cache[project]['project-created'] = project_created
|
||||
|
||||
# Create the repo for the local git mirror
|
||||
create_local_mirror(
|
||||
@ -679,8 +723,13 @@ def main():
|
||||
find_description_override(repo_path) or description)
|
||||
|
||||
if project_created:
|
||||
pushed_to_gerrit = project_cache[project].get(
|
||||
'pushed-to-gerrit', False)
|
||||
if not pushed_to_gerrit:
|
||||
push_to_gerrit(
|
||||
repo_path, project, push_string, remote_url, ssh_env)
|
||||
repo_path, project, push_string,
|
||||
remote_url, ssh_env)
|
||||
project_cache[project]['pushed-to-gerrit'] = True
|
||||
if GERRIT_REPLICATE:
|
||||
gerrit.replicate(project)
|
||||
|
||||
@ -699,7 +748,7 @@ def main():
|
||||
created = create_update_github_project(
|
||||
DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS,
|
||||
DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG,
|
||||
options, project, description, homepage)
|
||||
options, project, description, homepage, project_cache)
|
||||
if created and GERRIT_REPLICATE:
|
||||
gerrit.replicate(project)
|
||||
|
||||
@ -708,6 +757,9 @@ def main():
|
||||
"Problems creating %s, moving on." % project)
|
||||
continue
|
||||
finally:
|
||||
with open(PROJECT_CACHE_FILE, 'w') as cache_out:
|
||||
cache_out.write(json.dumps(
|
||||
project_cache, sort_keys=True, indent=2))
|
||||
os.unlink(ssh_env['GIT_SSH'])
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
Loading…
Reference in New Issue
Block a user