Enable commit processing for branches other than master

* Change config to allow to specify branch name for release
* Process all branches specified in config, not only master

TBD: update of default data and its schema

Partially fixes bug 1239816

Change-Id: I960a493b833414148befc5d6339df30706ffa38b

etc/test_default_data.json

@@ -55,7 +55,6 @@
 
     "repos": [
         {
-            "branches": ["master"],
             "project_group": "core",
             "releases": [
                 {
@@ -64,12 +63,14 @@
                     "tag_from": "2011.3"
                 },
                 {
-                    "tag_to": "2012.2",
+                    "branch": "stable/folsom",
+                    "tag_to": "HEAD",
                     "release_name": "Folsom",
                     "tag_from": "2012.1"
                 },
                 {
-                    "tag_to": "2013.1",
+                    "branch": "stable/grizzly",
+                    "tag_to": "HEAD",
                     "release_name": "Grizzly",
                     "tag_from": "2012.2"
                 },
@@ -85,7 +86,6 @@
             "project_type": "openstack"
         },
         {
-            "branches": ["master"],
             "module": "python-glanceclient",
             "project_group": "core",
             "project_type": "openstack",
@@ -93,7 +93,6 @@
             "uri": "git://github.com/openstack/python-glanceclient.git"
         },
         {
-            "branches": ["master"],
             "module": "stackalytics",
             "project_type": "stackforge",
             "organization": "stackforge",

stackalytics/processor/main.py

@@ -92,7 +92,12 @@ def process_repo(repo, runtime_storage_inst, record_processor_inst):
     rcs_inst.setup(key_filename=cfg.CONF.ssh_key_filename,
                    username=cfg.CONF.ssh_username)
 
-    for branch in repo['branches']:
+    branches = set(['master'])
+    for release in repo.get('releases'):
+        if 'branch' in release:
+            branches.add(release['branch'])
+
+    for branch in branches:
         LOG.debug('Processing repo %s, branch %s', uri, branch)
 
         vcs_key = 'vcs:' + str(urllib.quote_plus(uri) + ':' + branch)

stackalytics/processor/vcs.py

@@ -97,7 +97,7 @@ class Git(Vcs):
         else:
             os.chdir(self.folder)
             try:
-                sh.git('pull', 'origin')
+                sh.git('fetch')
             except sh.ErrorReturnCode as e:
                 LOG.error('Unable to pull git repo. Ignore it')
                 LOG.exception(e)
@@ -113,6 +113,13 @@ class Git(Vcs):
         if not self.release_index:
             for release in self.repo['releases']:
                 release_name = release['release_name'].lower()
+
+                if 'branch' in release:
+                    branch = release['branch']
+                else:
+                    branch = 'master'
+                sh.git('checkout', 'origin/' + branch)
+
                 if 'tag_from' in release:
                     tag_range = release['tag_from'] + '..' + release['tag_to']
                 else:
@@ -127,7 +134,7 @@ class Git(Vcs):
         LOG.debug('Parsing git log for repo uri %s', self.repo['uri'])
 
         os.chdir(self.folder)
-        sh.git('checkout', '%s' % branch)
+        sh.git('checkout', 'origin/' + branch)
         commit_range = 'HEAD'
         if head_commit_id:
             commit_range = head_commit_id + '..HEAD'
@@ -186,7 +193,7 @@ class Git(Vcs):
         LOG.debug('Get head commit for repo uri: %s', self.repo['uri'])
 
         os.chdir(self.folder)
-        sh.git('checkout', '%s' % branch)
+        sh.git('checkout', 'origin/' + branch)
         return str(sh.git('rev-parse', 'HEAD')).strip()