From 1c7264f9252232574e6498c12fe96a0d18fad062 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 7 Feb 2017 08:44:00 -0600 Subject: [PATCH] Cache status of stages in manage-projects Implement a trivial caching system that will allow us to skip even attempting gerrit and github remote operations for projects that we've processed successfully in previous passes. This should hopefully reduce the amount of time we spend processing the projects.yaml file. In the future, we could extend this to tracking more specific actions- such as the description set in github, so that we could know whether or not they have changed and thus need to be processed again. In this form, acls files will always be processed. Change-Id: I07b13c8663e6f9ee1255a4e56caf556ea49fb51b --- jeepyb/cmd/manage_projects.py | 70 ++++++++++++++++++++++++++++++----- 1 file changed, 61 insertions(+), 9 deletions(-) diff --git a/jeepyb/cmd/manage_projects.py b/jeepyb/cmd/manage_projects.py index eef72ea..15a635b 100644 --- a/jeepyb/cmd/manage_projects.py +++ b/jeepyb/cmd/manage_projects.py @@ -52,6 +52,7 @@ import argparse import ConfigParser +import json import logging import os import re @@ -288,12 +289,31 @@ def make_ssh_wrapper(gerrit_user, gerrit_key): def create_update_github_project( default_has_issues, default_has_downloads, default_has_wiki, - github_secure_config, options, project, description, homepage): + github_secure_config, options, project, description, homepage, + project_cache): created = False has_issues = 'has-issues' in options or default_has_issues has_downloads = 'has-downloads' in options or default_has_downloads has_wiki = 'has-wiki' in options or default_has_wiki + needs_update = False + if not project_cache[project].get('created-in-github', False): + needs_update = True + if not project_cache[project].get('gerrit-in-team', False): + needs_update = True + if project_cache[project].get('description') != description: + needs_update = True + if project_cache[project].get('homepage') != homepage: + needs_update = True + if project_cache[project].get('has_issues') != has_issues: + needs_update = True + if project_cache[project].get('has_downloads') != has_downloads: + needs_update = True + if project_cache[project].get('has_wiki') != has_wiki: + needs_update = True + if not needs_update: + return False + secure_config = ConfigParser.ConfigParser() secure_config.read(github_secure_config) @@ -302,6 +322,7 @@ def create_update_github_project( else: ghub = github.Github(secure_config.get("github", "username"), secure_config.get("github", "password")) + orgs = ghub.get_user().get_orgs() orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs)) @@ -324,14 +345,19 @@ def create_update_github_project( # If necessary, update project on Github if description and description != repo.description: repo.edit(repo_name, description=description) + project_cache[project]['description'] = description if homepage and homepage != repo.homepage: repo.edit(repo_name, homepage=homepage) + project_cache[project]['homepage'] = homepage if has_issues != repo.has_issues: repo.edit(repo_name, has_issues=has_issues) + project_cache[project]['has_issues'] = has_issues if has_downloads != repo.has_downloads: repo.edit(repo_name, has_downloads=has_downloads) + project_cache[project]['has_downloads'] = has_downloads if has_wiki != repo.has_wiki: repo.edit(repo_name, has_wiki=has_wiki) + project_cache[project]['has_wiki'] = has_wiki except github.GithubException: repo = org.create_repo(repo_name, @@ -339,19 +365,28 @@ def create_update_github_project( has_issues=has_issues, has_downloads=has_downloads, has_wiki=has_wiki) + project_cache[project]['created-in-github'] = True + project_cache[project]['has_wiki'] = has_wiki + project_cache[project]['has_downloads'] = has_downloads + project_cache[project]['has_issues'] = has_issues + if description: repo.edit(repo_name, description=description) + project_cache[project]['description'] = description if homepage: repo.edit(repo_name, homepage=homepage) + project_cache[project]['homepage'] = homepage repo.edit(repo_name, has_issues=has_issues, has_downloads=has_downloads, has_wiki=has_wiki) + created = True + if project_cache[project].get('gerrit-in-team', False): if 'gerrit' not in [team.name for team in repo.get_teams()]: teams = org.get_teams() teams_dict = dict(zip([t.name.lower() for t in teams], teams)) teams_dict['gerrit'].add_to_repos(repo) - created = True + project_cache[project]['gerrit-in-team'] = True return created @@ -606,6 +641,10 @@ def main(): GITHUB_SECURE_CONFIG = registry.get_defaults( 'github-config', '/etc/github/github-projects.secure.config') + PROJECT_CACHE_FILE = os.path.join(JEEPYB_CACHE_DIR, 'project.cache') + project_cache = {} + if os.path.exists(PROJECT_CACHE_FILE): + project_cache = json.loads(open(PROJECT_CACHE_FILE, 'r').read()) gerrit = gerritlib.gerrit.Gerrit(GERRIT_HOST, GERRIT_USER, @@ -647,12 +686,17 @@ def main(): acl_config = section.get( 'acl-config', '%s.config' % os.path.join(ACL_DIR, project)) + project_cache.setdefault(project, {}) # Create the project in Gerrit first, since it will fail # spectacularly if its project directory or local replica # already exist on disk - project_created = create_gerrit_project( - project, project_list, gerrit) + project_created = project_cache[project].get( + 'project-created', False) + if not project_created: + project_created = create_gerrit_project( + project, project_list, gerrit) + project_cache[project]['project-created'] = project_created # Create the repo for the local git mirror create_local_mirror( @@ -679,10 +723,15 @@ def main(): find_description_override(repo_path) or description) if project_created: - push_to_gerrit( - repo_path, project, push_string, remote_url, ssh_env) - if GERRIT_REPLICATE: - gerrit.replicate(project) + pushed_to_gerrit = project_cache[project].get( + 'pushed-to-gerrit', False) + if not pushed_to_gerrit: + push_to_gerrit( + repo_path, project, push_string, + remote_url, ssh_env) + project_cache[project]['pushed-to-gerrit'] = True + if GERRIT_REPLICATE: + gerrit.replicate(project) # If we're configured to track upstream, make sure we have # upstream's refs, and then push them to the appropriate @@ -699,7 +748,7 @@ def main(): created = create_update_github_project( DEFAULT_HAS_ISSUES, DEFAULT_HAS_DOWNLOADS, DEFAULT_HAS_WIKI, GITHUB_SECURE_CONFIG, - options, project, description, homepage) + options, project, description, homepage, project_cache) if created and GERRIT_REPLICATE: gerrit.replicate(project) @@ -708,6 +757,9 @@ def main(): "Problems creating %s, moving on." % project) continue finally: + with open(PROJECT_CACHE_FILE, 'w') as cache_out: + cache_out.write(json.dumps( + project_cache, sort_keys=True, indent=2)) os.unlink(ssh_env['GIT_SSH']) if __name__ == "__main__":