Reduce impact of check revisions task

Now that change refs are stored locally, the only time a local repo
should be missing revisions is when it is newly cloned.  Handle the
case where a user deletes the local repo automatically (when gertty
starts and a repo does not exist for a subscribed project, perform
a missing refs check on that project).

If the user knows something is wrong, they can supply the
--fetch-missing-refs flag at startup and an exhaustive check will
be run of all projects.

Otherwise, do nothing.

Change-Id: Iedebbe4b05ec1293b82c651587e42004444dc4a8

gertty/app.py

@@ -100,7 +100,7 @@ class SearchDialog(mywid.ButtonDialog):
 
 class App(object):
     def __init__(self, server=None, palette='default', keymap='default',
-                 debug=False, disable_sync=False):
+                 debug=False, disable_sync=False, fetch_missing_refs=False):
         self.server = server
         self.config = config.Config(server, palette, keymap)
         if debug:
@@ -112,6 +112,8 @@ class App(object):
                             level=level)
         self.log = logging.getLogger('gertty.App')
         self.log.debug("Starting")
+
+        self.fetch_missing_refs = fetch_missing_refs
         self.config.keymap.updateCommandMap()
         self.search = search.SearchCompiler(self)
         self.db = db.Database(self)
@@ -362,6 +364,9 @@ def main():
                         help='enable debug logging')
     parser.add_argument('--no-sync', dest='no_sync', action='store_true',
                         help='disable remote syncing')
+    parser.add_argument('--fetch-missing-refs', dest='fetch_missing_refs',
+                        action='store_true',
+                        help='fetch any refs missing from local repos')
     parser.add_argument('-p', dest='palette', default='default',
                         help='Color palette to use')
     parser.add_argument('-k', dest='keymap', default='default',
@@ -369,7 +374,8 @@ def main():
     parser.add_argument('server', nargs='?',
                         help='the server to use (as specified in config file)')
     args = parser.parse_args()
-    g = App(args.server, args.palette, args.keymap, args.debug, args.no_sync)
+    g = App(args.server, args.palette, args.keymap, args.debug, args.no_sync,
+            args.fetch_missing_refs)
     g.run()
 
 

gertty/gitrepo.py

@@ -232,6 +232,9 @@ class Repo(object):
         self.differ = difflib.Differ()
         if not os.path.exists(path):
             git.Repo.clone_from(self.url, self.path)
+            self.newly_cloned = True
+        else:
+            self.newly_cloned = False
 
     def hasCommit(self, sha):
         repo = git.Repo(self.path)

gertty/sync.py

@@ -1,4 +1,5 @@
 # Copyright 2014 OpenStack Foundation
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -431,25 +432,48 @@ class SyncChangeTask(Task):
                     self.log.debug("git fetch %s %s" % (url, ref))
                     repo.fetch(url, ref)
 
-class CheckRevisionsTask(Task):
+class CheckReposTask(Task):
+    # on startup, check all projects
+    #   for any newly cloned project, run checkrevisionstask on that project
+    #   if --fetch-missing-refs is supplied, run crt on every project
     def __repr__(self):
-        return '<CheckRevisionsTask>'
+        return '<CheckReposTask>'
+
+    def run(self, sync):
+        app = sync.app
+        to_check = []
+        with app.db.getSession() as session:
+            for project in session.getProjects(subscribed=True):
+                try:
+                    repo = app.getRepo(project.name)
+                    if repo.newly_cloned or app.fetch_missing_refs:
+                        to_check.append(project.key)
+                except Exception:
+                    self.log.exception("Exception checking repo %s" % (project.name,))
+        for key in to_check:
+            sync.submitTask(CheckRevisionsTask(key, priority=LOW_PRIORITY))
+
+class CheckRevisionsTask(Task):
+    def __init__(self, project_key, priority=NORMAL_PRIORITY):
+        super(CheckRevisionsTask, self).__init__(priority)
+        self.project_key = project_key
+
+    def __repr__(self):
+        return '<CheckRevisionsTask %s>' % (self.project_key,)
 
     def run(self, sync):
         app = sync.app
         to_fetch = collections.defaultdict(list)
         with app.db.getSession() as session:
-            for project in session.getProjects():
-                if not project.open_changes:
-                    continue
-                repo = app.getRepo(project.name)
-                for change in project.open_changes:
-                    for revision in change.revisions:
-                        if not (repo.hasCommit(revision.parent) and
-                                repo.hasCommit(revision.commit)):
-                            if revision.fetch_ref:
-                                to_fetch[(project.name, revision.fetch_auth)
-                                    ].append(revision.fetch_ref)
+            project = session.getProject(self.project_key)
+            repo = app.getRepo(project.name)
+            for change in project.open_changes:
+                for revision in change.revisions:
+                    if not (repo.hasCommit(revision.parent) and
+                            repo.hasCommit(revision.commit)):
+                        if revision.fetch_ref:
+                            to_fetch[(project.name, revision.fetch_auth)
+                                     ].append(revision.fetch_ref)
         for (name, auth), refs in to_fetch.items():
             sync.submitTask(FetchRefTask(name, refs, auth, priority=self.priority))
 
@@ -547,10 +571,10 @@ class Sync(object):
         self.auth = requests.auth.HTTPDigestAuth(
             self.app.config.username, self.app.config.password)
         self.submitTask(SyncOwnAccountTask(HIGH_PRIORITY))
+        self.submitTask(CheckReposTask(HIGH_PRIORITY))
         self.submitTask(UploadReviewsTask(HIGH_PRIORITY))
         self.submitTask(SyncProjectListTask(HIGH_PRIORITY))
         self.submitTask(SyncSubscribedProjectsTask(HIGH_PRIORITY))
-        self.submitTask(CheckRevisionsTask(LOW_PRIORITY))
         self.periodic_thread = threading.Thread(target=self.periodicSync)
         self.periodic_thread.daemon = True
         self.periodic_thread.start()