Reduce impact of check revisions task
Now that change refs are stored locally, the only time a local repo should be missing revisions is when it is newly cloned. Handle the case where a user deletes the local repo automatically (when gertty starts and a repo does not exist for a subscribed project, perform a missing refs check on that project). If the user knows something is wrong, they can supply the --fetch-missing-refs flag at startup and an exhaustive check will be run of all projects. Otherwise, do nothing. Change-Id: Iedebbe4b05ec1293b82c651587e42004444dc4a8
This commit is contained in:
parent
2a50ba2053
commit
f47275022b
|
@ -100,7 +100,7 @@ class SearchDialog(mywid.ButtonDialog):
|
||||||
|
|
||||||
class App(object):
|
class App(object):
|
||||||
def __init__(self, server=None, palette='default', keymap='default',
|
def __init__(self, server=None, palette='default', keymap='default',
|
||||||
debug=False, disable_sync=False):
|
debug=False, disable_sync=False, fetch_missing_refs=False):
|
||||||
self.server = server
|
self.server = server
|
||||||
self.config = config.Config(server, palette, keymap)
|
self.config = config.Config(server, palette, keymap)
|
||||||
if debug:
|
if debug:
|
||||||
|
@ -112,6 +112,8 @@ class App(object):
|
||||||
level=level)
|
level=level)
|
||||||
self.log = logging.getLogger('gertty.App')
|
self.log = logging.getLogger('gertty.App')
|
||||||
self.log.debug("Starting")
|
self.log.debug("Starting")
|
||||||
|
|
||||||
|
self.fetch_missing_refs = fetch_missing_refs
|
||||||
self.config.keymap.updateCommandMap()
|
self.config.keymap.updateCommandMap()
|
||||||
self.search = search.SearchCompiler(self)
|
self.search = search.SearchCompiler(self)
|
||||||
self.db = db.Database(self)
|
self.db = db.Database(self)
|
||||||
|
@ -362,6 +364,9 @@ def main():
|
||||||
help='enable debug logging')
|
help='enable debug logging')
|
||||||
parser.add_argument('--no-sync', dest='no_sync', action='store_true',
|
parser.add_argument('--no-sync', dest='no_sync', action='store_true',
|
||||||
help='disable remote syncing')
|
help='disable remote syncing')
|
||||||
|
parser.add_argument('--fetch-missing-refs', dest='fetch_missing_refs',
|
||||||
|
action='store_true',
|
||||||
|
help='fetch any refs missing from local repos')
|
||||||
parser.add_argument('-p', dest='palette', default='default',
|
parser.add_argument('-p', dest='palette', default='default',
|
||||||
help='Color palette to use')
|
help='Color palette to use')
|
||||||
parser.add_argument('-k', dest='keymap', default='default',
|
parser.add_argument('-k', dest='keymap', default='default',
|
||||||
|
@ -369,7 +374,8 @@ def main():
|
||||||
parser.add_argument('server', nargs='?',
|
parser.add_argument('server', nargs='?',
|
||||||
help='the server to use (as specified in config file)')
|
help='the server to use (as specified in config file)')
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
g = App(args.server, args.palette, args.keymap, args.debug, args.no_sync)
|
g = App(args.server, args.palette, args.keymap, args.debug, args.no_sync,
|
||||||
|
args.fetch_missing_refs)
|
||||||
g.run()
|
g.run()
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -232,6 +232,9 @@ class Repo(object):
|
||||||
self.differ = difflib.Differ()
|
self.differ = difflib.Differ()
|
||||||
if not os.path.exists(path):
|
if not os.path.exists(path):
|
||||||
git.Repo.clone_from(self.url, self.path)
|
git.Repo.clone_from(self.url, self.path)
|
||||||
|
self.newly_cloned = True
|
||||||
|
else:
|
||||||
|
self.newly_cloned = False
|
||||||
|
|
||||||
def hasCommit(self, sha):
|
def hasCommit(self, sha):
|
||||||
repo = git.Repo(self.path)
|
repo = git.Repo(self.path)
|
||||||
|
|
|
@ -1,4 +1,5 @@
|
||||||
# Copyright 2014 OpenStack Foundation
|
# Copyright 2014 OpenStack Foundation
|
||||||
|
# Copyright 2014 Hewlett-Packard Development Company, L.P.
|
||||||
#
|
#
|
||||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
# not use this file except in compliance with the License. You may obtain
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
@ -431,25 +432,48 @@ class SyncChangeTask(Task):
|
||||||
self.log.debug("git fetch %s %s" % (url, ref))
|
self.log.debug("git fetch %s %s" % (url, ref))
|
||||||
repo.fetch(url, ref)
|
repo.fetch(url, ref)
|
||||||
|
|
||||||
class CheckRevisionsTask(Task):
|
class CheckReposTask(Task):
|
||||||
|
# on startup, check all projects
|
||||||
|
# for any newly cloned project, run checkrevisionstask on that project
|
||||||
|
# if --fetch-missing-refs is supplied, run crt on every project
|
||||||
def __repr__(self):
|
def __repr__(self):
|
||||||
return '<CheckRevisionsTask>'
|
return '<CheckReposTask>'
|
||||||
|
|
||||||
|
def run(self, sync):
|
||||||
|
app = sync.app
|
||||||
|
to_check = []
|
||||||
|
with app.db.getSession() as session:
|
||||||
|
for project in session.getProjects(subscribed=True):
|
||||||
|
try:
|
||||||
|
repo = app.getRepo(project.name)
|
||||||
|
if repo.newly_cloned or app.fetch_missing_refs:
|
||||||
|
to_check.append(project.key)
|
||||||
|
except Exception:
|
||||||
|
self.log.exception("Exception checking repo %s" % (project.name,))
|
||||||
|
for key in to_check:
|
||||||
|
sync.submitTask(CheckRevisionsTask(key, priority=LOW_PRIORITY))
|
||||||
|
|
||||||
|
class CheckRevisionsTask(Task):
|
||||||
|
def __init__(self, project_key, priority=NORMAL_PRIORITY):
|
||||||
|
super(CheckRevisionsTask, self).__init__(priority)
|
||||||
|
self.project_key = project_key
|
||||||
|
|
||||||
|
def __repr__(self):
|
||||||
|
return '<CheckRevisionsTask %s>' % (self.project_key,)
|
||||||
|
|
||||||
def run(self, sync):
|
def run(self, sync):
|
||||||
app = sync.app
|
app = sync.app
|
||||||
to_fetch = collections.defaultdict(list)
|
to_fetch = collections.defaultdict(list)
|
||||||
with app.db.getSession() as session:
|
with app.db.getSession() as session:
|
||||||
for project in session.getProjects():
|
project = session.getProject(self.project_key)
|
||||||
if not project.open_changes:
|
repo = app.getRepo(project.name)
|
||||||
continue
|
for change in project.open_changes:
|
||||||
repo = app.getRepo(project.name)
|
for revision in change.revisions:
|
||||||
for change in project.open_changes:
|
if not (repo.hasCommit(revision.parent) and
|
||||||
for revision in change.revisions:
|
repo.hasCommit(revision.commit)):
|
||||||
if not (repo.hasCommit(revision.parent) and
|
if revision.fetch_ref:
|
||||||
repo.hasCommit(revision.commit)):
|
to_fetch[(project.name, revision.fetch_auth)
|
||||||
if revision.fetch_ref:
|
].append(revision.fetch_ref)
|
||||||
to_fetch[(project.name, revision.fetch_auth)
|
|
||||||
].append(revision.fetch_ref)
|
|
||||||
for (name, auth), refs in to_fetch.items():
|
for (name, auth), refs in to_fetch.items():
|
||||||
sync.submitTask(FetchRefTask(name, refs, auth, priority=self.priority))
|
sync.submitTask(FetchRefTask(name, refs, auth, priority=self.priority))
|
||||||
|
|
||||||
|
@ -547,10 +571,10 @@ class Sync(object):
|
||||||
self.auth = requests.auth.HTTPDigestAuth(
|
self.auth = requests.auth.HTTPDigestAuth(
|
||||||
self.app.config.username, self.app.config.password)
|
self.app.config.username, self.app.config.password)
|
||||||
self.submitTask(SyncOwnAccountTask(HIGH_PRIORITY))
|
self.submitTask(SyncOwnAccountTask(HIGH_PRIORITY))
|
||||||
|
self.submitTask(CheckReposTask(HIGH_PRIORITY))
|
||||||
self.submitTask(UploadReviewsTask(HIGH_PRIORITY))
|
self.submitTask(UploadReviewsTask(HIGH_PRIORITY))
|
||||||
self.submitTask(SyncProjectListTask(HIGH_PRIORITY))
|
self.submitTask(SyncProjectListTask(HIGH_PRIORITY))
|
||||||
self.submitTask(SyncSubscribedProjectsTask(HIGH_PRIORITY))
|
self.submitTask(SyncSubscribedProjectsTask(HIGH_PRIORITY))
|
||||||
self.submitTask(CheckRevisionsTask(LOW_PRIORITY))
|
|
||||||
self.periodic_thread = threading.Thread(target=self.periodicSync)
|
self.periodic_thread = threading.Thread(target=self.periodicSync)
|
||||||
self.periodic_thread.daemon = True
|
self.periodic_thread.daemon = True
|
||||||
self.periodic_thread.start()
|
self.periodic_thread.start()
|
||||||
|
|
Loading…
Reference in New Issue