Browse Source

Use zuul gerrit event listener implementation

Current event listener implementation does not handle network problems
well. E.g. if ssh stream connection is lost, it would not reconnect or
recover on its own. Instead of fixing the implementation, use
well-tested gerrit listener used by zuul. Explicitly specify version of
zuul to be 2.1.0 to avoid accidental breakages due to changes in zuul
lib.

Downside is that we need to install zuul and its dependencies just to
use gerrit listener.

parse_json_event function had to be changed, because zuul gerrit event
listener provides object, not json string. We still need to create
event from json in populate_db.py, so that part of the function has
been moved there.

Closes-Bug: #1516820

Change-Id: I8aa7a18460b58998f6c378e9d9c0d783032eca21
changes/19/248919/4
Mikhail S Medvedev 4 years ago
parent
commit
56e474d725
3 changed files with 26 additions and 59 deletions
  1. +13
    -55
      ciwatch/events.py
  2. +12
    -4
      ciwatch/populate.py
  3. +1
    -0
      requirements.txt

+ 13
- 55
ciwatch/events.py View File

@@ -14,14 +14,13 @@

from datetime import datetime
import json
import paramiko
import re
import time

from ciwatch.config import Config
from ciwatch import db
from ciwatch.log import logger
from ciwatch import models
from zuul.lib.gerrit import Gerrit


def _process_project_name(project_name):
@@ -70,49 +69,7 @@ def _store_event(event, datadir):
return event


class GerritEventStream(object):
def __init__(self, cfg):

logger.debug('Connecting to %(host)s:%(port)d as '
'%(user)s using %(key)s',
{'user': cfg.AccountInfo.gerrit_username,
'key': cfg.AccountInfo.gerrit_ssh_key,
'host': cfg.AccountInfo.gerrit_host,
'port': int(cfg.AccountInfo.gerrit_port)})

self.ssh = paramiko.SSHClient()
self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())

connected = False
while not connected:
try:
self.ssh.connect(cfg.AccountInfo.gerrit_host,
int(cfg.AccountInfo.gerrit_port),
cfg.AccountInfo.gerrit_username,
key_filename=cfg.AccountInfo.gerrit_ssh_key)
connected = True
except paramiko.SSHException as e:
logger.error('%s', e)
logger.warn('Gerrit may be down, will pause and retry...')
time.sleep(10)

self.stdin, self.stdout, self.stderr =\
self.ssh.exec_command("gerrit stream-events")

def __iter__(self):
return self

def next(self):
return self.stdout.readline()


def parse_json_event(event, projects):
try:
event = json.loads(event)
except Exception as ex:
logger.error('Failed json.loads on event: %s', event)
logger.exception(ex)
return None
def parse_event(event, projects):
if _is_valid(event, projects):
_process_event(event)
logger.info('Parsed valid event: %s', event)
@@ -162,17 +119,18 @@ def add_event_to_db(event, commit_=True):
def main():
config = Config()
db.create_projects() # This will make sure the database has projects in it
gerrit = Gerrit(
hostname=config.cfg.AccountInfo.gerrit_host,
username=config.cfg.AccountInfo.gerrit_username,
port=int(config.cfg.AccountInfo.gerrit_port),
keyfile=config.cfg.AccountInfo.gerrit_ssh_key
)
gerrit.startWatching()
while True:
try:
events = GerritEventStream(config.cfg)
except paramiko.SSHException as ex:
logger.exception('Error connecting to Gerrit: %s', ex)
time.sleep(60)
for event in events:
event = parse_json_event(event, config.get_projects())
if event is not None:
_store_event(event, config.DATA_DIR)

event = gerrit.getEvent()[1]
parsed_event = parse_event(event, config.get_projects())
if parsed_event is not None:
_store_event(parsed_event, config.DATA_DIR)

if __name__ == '__main__':
main()

+ 12
- 4
ciwatch/populate.py View File

@@ -12,21 +12,29 @@
# License for the specific language governing permissions and limitations
# under the License.

import json
import os

from ciwatch.config import Config
from ciwatch import db
from ciwatch.events import add_event_to_db
from ciwatch.events import parse_json_event
from ciwatch.events import parse_event
from ciwatch.log import logger


def get_data(datafile, projects):
data = []
with open(datafile) as file_:
for line in file_:
event = parse_json_event(line, projects)
if event is not None:
data.append(event)
try:
event = json.loads(line)
except Exception as ex:
logger.error('Failed json.loads on event: %s', event)
logger.exception(ex)
continue
parsed_event = parse_event(event, projects)
if parsed_event is not None:
data.append(parsed_event)
return data



+ 1
- 0
requirements.txt View File

@@ -7,3 +7,4 @@ flask>=0.10
sqlalchemy>=1.0
iniparse>=0.4
paramiko>=1.15
zuul==2.1.0

Loading…
Cancel
Save