Refactor configuration handling

This change should make it a bit easier to see when configuration is
read in, and would allow to further centralize parsing and loading of
the conf file.

We should not 'import config' each time when we need to get access to
the settings. Instead, the settings should be loaded once, and then
explicitly passed around. This patch does not make that happen yet, but
is the first step.

Change-Id: If9ad1222ceb806337a745845ee6e76605bab5fd3

ciwatch/config.py

@@ -17,22 +17,31 @@ import os
 from iniparse import INIConfig
 
 
-def get_config():
-    this_file = os.path.dirname(os.path.realpath(__file__))
-    this_dir = os.path.dirname(this_file)
-    conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
-                  '/etc/ciwatch/ci-watch.conf']
-    # Read first existing conf file, ignore the rest
-    for conf_file in conf_files:
-        if os.path.exists(conf_file):
-            return INIConfig(open(conf_file))
-    raise Exception('Could not read configuration from %s' % conf_files)
+class Config(object):
 
-cfg = get_config()
+    def __init__(self):
+        self.cfg = self.get_config()
+        if self.cfg.Data.data_dir:
+            self.DATA_DIR = self.cfg.Data.data_dir
+        else:
+            self.DATA_DIR = os.path.dirname(
+                os.path.dirname(os.path.abspath(__file__))) + '/data'
 
+    def get_config(self):
+        this_file = os.path.dirname(os.path.realpath(__file__))
+        this_dir = os.path.dirname(this_file)
+        conf_files = [os.path.join(this_dir, 'ci-watch.conf'),
+                      '/etc/ciwatch/ci-watch.conf']
+        # Read first existing conf file, ignore the rest
+        for conf_file in conf_files:
+            if os.path.exists(conf_file):
+                return INIConfig(open(conf_file))
+        else:
+            raise Exception(
+                'Could not read configuration from %s' % conf_files)
 
-def get_projects():
-    projects = []
-    for name in cfg.misc.projects.split(','):
-        projects.append(name)
-    return projects
+    def get_projects(self):
+        projects = []
+        for name in self.cfg.misc.projects.split(','):
+            projects.append(name)
+        return projects

ciwatch/db.py

@@ -15,12 +15,12 @@
 from sqlalchemy import create_engine
 from sqlalchemy.orm import sessionmaker
 
-from ciwatch.config import cfg
-from ciwatch.config import get_projects
+from ciwatch.config import Config
 from ciwatch import models
 
 
-engine = create_engine(cfg.database.connection)
+config = Config()
+engine = create_engine(config.cfg.database.connection)
 Session = sessionmaker()
 Session.configure(bind=engine)
 models.Base.metadata.create_all(engine)
@@ -28,7 +28,7 @@ session = Session()
 
 
 def create_projects():
-    for name in get_projects():
+    for name in config.get_projects():
         get_or_create(models.Project,
                       commit_=False,
                       name=name)

ciwatch/events.py

@@ -18,14 +18,10 @@ import paramiko
 import re
 import time
 
+from ciwatch.config import Config
 from ciwatch import db
-from ciwatch import models
-
-from ciwatch.log import DATA_DIR
 from ciwatch.log import logger
-
-from ciwatch.config import cfg
-from ciwatch.config import get_projects
+from ciwatch import models
 
 
 def _process_project_name(project_name):
@@ -56,18 +52,18 @@ def _is_ci_user(name):
 
 
 # Check if this is a third party CI event
-def _is_valid(event):
+def _is_valid(event, projects):
     if (event.get('type', 'nill') == 'comment-added' and
             _is_ci_user(event['author'].get('name', '')) and
             _process_project_name(
-                event['change']['project']) in get_projects() and
+                event['change']['project']) in projects and
             event['change']['branch'] == 'master'):
         return True
     return False
 
 
-def _store_event(event):
-    with open(DATA_DIR + '/third-party-ci.log', 'a') as f:
+def _store_event(event, datadir):
+    with open(datadir + '/third-party-ci.log', 'a') as f:
         json.dump(event, f)
         f.write('\n')
     add_event_to_db(event)
@@ -75,7 +71,7 @@ def _store_event(event):
 
 
 class GerritEventStream(object):
-    def __init__(self):
+    def __init__(self, cfg):
 
         logger.debug('Connecting to %(host)s:%(port)d as '
                      '%(user)s using %(key)s',
@@ -110,14 +106,14 @@ class GerritEventStream(object):
         return self.stdout.readline()
 
 
-def parse_json_event(event):
+def parse_json_event(event, projects):
     try:
         event = json.loads(event)
     except Exception as ex:
         logger.error('Failed json.loads on event: %s', event)
         logger.exception(ex)
         return None
-    if _is_valid(event):
+    if _is_valid(event, projects):
         _process_event(event)
         logger.info('Parsed valid event: %s', event)
         return event
@@ -163,17 +159,18 @@ def add_event_to_db(event, commit_=True):
 
 
 def main():
+    config = Config()
     db.create_projects()  # This will make sure the database has projects in it
     while True:
         try:
-            events = GerritEventStream()
+            events = GerritEventStream(config.cfg)
         except paramiko.SSHException as ex:
             logger.exception('Error connecting to Gerrit: %s', ex)
             time.sleep(60)
         for event in events:
-            event = parse_json_event(event)
+            event = parse_json_event(event, config.get_projects())
             if event is not None:
-                _store_event(event)
+                _store_event(event, config.DATA_DIR)
 
 
 if __name__ == '__main__':

ciwatch/log.py

@@ -14,9 +14,9 @@
 
 import logging
 from logging import handlers
-import os
 
-from ciwatch.config import cfg
+from ciwatch.config import Config
+config = Config()
 
 
 def setup_logger(name):
@@ -38,9 +38,4 @@ def setup_logger(name):
     return logger
 
 
-DATA_DIR =\
-    os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + '/data'
-if cfg.Data.data_dir:
-    DATA_DIR = cfg.Data.data_dir
-
-logger = setup_logger(DATA_DIR + '/ci-watch.log')
+logger = setup_logger(config.DATA_DIR + '/ci-watch.log')

ciwatch/populate.py

@@ -14,32 +14,35 @@
 
 import os
 
+from ciwatch.config import Config
 from ciwatch import db
 from ciwatch.events import add_event_to_db
 from ciwatch.events import parse_json_event
-from ciwatch.log import DATA_DIR
 
 
-def get_data():
+def get_data(datafile, projects):
     data = []
-    with open(os.path.join(DATA_DIR, 'third-party-ci.log')) as file_:
+    with open(datafile) as file_:
         for line in file_:
-            event = parse_json_event(line)
+            event = parse_json_event(line, projects)
             if event is not None:
                 data.append(event)
     return data
 
 
-def load_data():
-    data = get_data()
+def load_data(data):
     for event in data:
         add_event_to_db(event, commit_=False)
     db.session.commit()
 
 
 def main():
+    config = Config()
+    projects = config.get_projects()
+    datafile = os.path.join(config.DATA_DIR, 'third-party-ci.log')
     db.create_projects()
-    load_data()
+    data = get_data(datafile, projects)
+    load_data(data)
 
 
 if __name__ == '__main__':