Browse Source

Use zuul gerrit event listener implementation

Current event listener implementation does not handle network problems
well. E.g. if ssh stream connection is lost, it would not reconnect or
recover on its own. Instead of fixing the implementation, use
well-tested gerrit listener used by zuul. Explicitly specify version of
zuul to be 2.1.0 to avoid accidental breakages due to changes in zuul
lib.

Downside is that we need to install zuul and its dependencies just to
use gerrit listener.

parse_json_event function had to be changed, because zuul gerrit event
listener provides object, not json string. We still need to create
event from json in populate_db.py, so that part of the function has
been moved there.

Closes-Bug: #1516820

Change-Id: I8aa7a18460b58998f6c378e9d9c0d783032eca21
changes/19/248919/4
Mikhail S Medvedev 3 years ago
parent
commit
56e474d725
3 changed files with 26 additions and 59 deletions
  1. 13
    55
      ciwatch/events.py
  2. 12
    4
      ciwatch/populate.py
  3. 1
    0
      requirements.txt

+ 13
- 55
ciwatch/events.py View File

@@ -14,14 +14,13 @@
14 14
 
15 15
 from datetime import datetime
16 16
 import json
17
-import paramiko
18 17
 import re
19
-import time
20 18
 
21 19
 from ciwatch.config import Config
22 20
 from ciwatch import db
23 21
 from ciwatch.log import logger
24 22
 from ciwatch import models
23
+from zuul.lib.gerrit import Gerrit
25 24
 
26 25
 
27 26
 def _process_project_name(project_name):
@@ -70,49 +69,7 @@ def _store_event(event, datadir):
70 69
     return event
71 70
 
72 71
 
73
-class GerritEventStream(object):
74
-    def __init__(self, cfg):
75
-
76
-        logger.debug('Connecting to %(host)s:%(port)d as '
77
-                     '%(user)s using %(key)s',
78
-                     {'user': cfg.AccountInfo.gerrit_username,
79
-                      'key': cfg.AccountInfo.gerrit_ssh_key,
80
-                      'host': cfg.AccountInfo.gerrit_host,
81
-                      'port': int(cfg.AccountInfo.gerrit_port)})
82
-
83
-        self.ssh = paramiko.SSHClient()
84
-        self.ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
85
-
86
-        connected = False
87
-        while not connected:
88
-            try:
89
-                self.ssh.connect(cfg.AccountInfo.gerrit_host,
90
-                                 int(cfg.AccountInfo.gerrit_port),
91
-                                 cfg.AccountInfo.gerrit_username,
92
-                                 key_filename=cfg.AccountInfo.gerrit_ssh_key)
93
-                connected = True
94
-            except paramiko.SSHException as e:
95
-                logger.error('%s', e)
96
-                logger.warn('Gerrit may be down, will pause and retry...')
97
-                time.sleep(10)
98
-
99
-        self.stdin, self.stdout, self.stderr =\
100
-            self.ssh.exec_command("gerrit stream-events")
101
-
102
-    def __iter__(self):
103
-        return self
104
-
105
-    def next(self):
106
-        return self.stdout.readline()
107
-
108
-
109
-def parse_json_event(event, projects):
110
-    try:
111
-        event = json.loads(event)
112
-    except Exception as ex:
113
-        logger.error('Failed json.loads on event: %s', event)
114
-        logger.exception(ex)
115
-        return None
72
+def parse_event(event, projects):
116 73
     if _is_valid(event, projects):
117 74
         _process_event(event)
118 75
         logger.info('Parsed valid event: %s', event)
@@ -162,17 +119,18 @@ def add_event_to_db(event, commit_=True):
162 119
 def main():
163 120
     config = Config()
164 121
     db.create_projects()  # This will make sure the database has projects in it
122
+    gerrit = Gerrit(
123
+        hostname=config.cfg.AccountInfo.gerrit_host,
124
+        username=config.cfg.AccountInfo.gerrit_username,
125
+        port=int(config.cfg.AccountInfo.gerrit_port),
126
+        keyfile=config.cfg.AccountInfo.gerrit_ssh_key
127
+    )
128
+    gerrit.startWatching()
165 129
     while True:
166
-        try:
167
-            events = GerritEventStream(config.cfg)
168
-        except paramiko.SSHException as ex:
169
-            logger.exception('Error connecting to Gerrit: %s', ex)
170
-            time.sleep(60)
171
-        for event in events:
172
-            event = parse_json_event(event, config.get_projects())
173
-            if event is not None:
174
-                _store_event(event, config.DATA_DIR)
175
-
130
+        event = gerrit.getEvent()[1]
131
+        parsed_event = parse_event(event, config.get_projects())
132
+        if parsed_event is not None:
133
+            _store_event(parsed_event, config.DATA_DIR)
176 134
 
177 135
 if __name__ == '__main__':
178 136
     main()

+ 12
- 4
ciwatch/populate.py View File

@@ -12,21 +12,29 @@
12 12
 #    License for the specific language governing permissions and limitations
13 13
 #    under the License.
14 14
 
15
+import json
15 16
 import os
16 17
 
17 18
 from ciwatch.config import Config
18 19
 from ciwatch import db
19 20
 from ciwatch.events import add_event_to_db
20
-from ciwatch.events import parse_json_event
21
+from ciwatch.events import parse_event
22
+from ciwatch.log import logger
21 23
 
22 24
 
23 25
 def get_data(datafile, projects):
24 26
     data = []
25 27
     with open(datafile) as file_:
26 28
         for line in file_:
27
-            event = parse_json_event(line, projects)
28
-            if event is not None:
29
-                data.append(event)
29
+            try:
30
+                event = json.loads(line)
31
+            except Exception as ex:
32
+                logger.error('Failed json.loads on event: %s', event)
33
+                logger.exception(ex)
34
+                continue
35
+            parsed_event = parse_event(event, projects)
36
+            if parsed_event is not None:
37
+                data.append(parsed_event)
30 38
     return data
31 39
 
32 40
 

+ 1
- 0
requirements.txt View File

@@ -7,3 +7,4 @@ flask>=0.10
7 7
 sqlalchemy>=1.0
8 8
 iniparse>=0.4
9 9
 paramiko>=1.15
10
+zuul==2.1.0

Loading…
Cancel
Save