Switch to Zookeeper backed trigger event queues
Trigger events will now be dispatched via Zookeeper. The event queues are namespaced by tenant since the event processing will later require a tenant lock in a multi scheduler deployment. Gitlab events hold their labels as a non-serializable set attribute; this change adjusts them to be held in a list (but set operations are still used for de-duplication). Change-Id: Ie54fc16488ab8cbc15f97d003f36c12b8a648ed4
This commit is contained in:
parent
be8d216629
commit
2e6cfff818
|
@ -4032,7 +4032,6 @@ class SchedulerTestApp:
|
||||||
|
|
||||||
self.event_queues = [
|
self.event_queues = [
|
||||||
self.sched.result_event_queue,
|
self.sched.result_event_queue,
|
||||||
self.sched.trigger_event_queue,
|
|
||||||
self.sched.management_event_queue
|
self.sched.management_event_queue
|
||||||
]
|
]
|
||||||
|
|
||||||
|
@ -4041,7 +4040,6 @@ class SchedulerTestApp:
|
||||||
self.sched.executor.gearman.waitForServer()
|
self.sched.executor.gearman.waitForServer()
|
||||||
self.sched.reconfigure(
|
self.sched.reconfigure(
|
||||||
self.config, validate_tenants=validate_tenants)
|
self.config, validate_tenants=validate_tenants)
|
||||||
self.sched.wakeUp()
|
|
||||||
|
|
||||||
def fullReconfigure(self):
|
def fullReconfigure(self):
|
||||||
try:
|
try:
|
||||||
|
@ -4916,6 +4914,18 @@ class ZuulTestCase(BaseTestCase):
|
||||||
for event_queue in self.additional_event_queues:
|
for event_queue in self.additional_event_queues:
|
||||||
event_queue.join()
|
event_queue.join()
|
||||||
|
|
||||||
|
def __areZooKeeperEventQueuesEmpty(self, matcher=None) -> bool:
|
||||||
|
for sched in map(lambda app: app.sched, self.scheds.filter(matcher)):
|
||||||
|
if sched.trigger_events.hasEvents():
|
||||||
|
return False
|
||||||
|
for tenant in sched.abide.tenants.values():
|
||||||
|
for pipeline_name in tenant.layout.pipelines:
|
||||||
|
if sched.pipeline_trigger_events[tenant.name][
|
||||||
|
pipeline_name
|
||||||
|
].hasEvents():
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
def waitUntilSettled(self, msg="", matcher=None) -> None:
|
def waitUntilSettled(self, msg="", matcher=None) -> None:
|
||||||
self.log.debug("Waiting until settled... (%s)", msg)
|
self.log.debug("Waiting until settled... (%s)", msg)
|
||||||
start = time.time()
|
start = time.time()
|
||||||
|
@ -4928,6 +4938,10 @@ class ZuulTestCase(BaseTestCase):
|
||||||
for event_queue in self.__event_queues(matcher):
|
for event_queue in self.__event_queues(matcher):
|
||||||
self.log.error(" %s: %s" %
|
self.log.error(" %s: %s" %
|
||||||
(event_queue, event_queue.empty()))
|
(event_queue, event_queue.empty()))
|
||||||
|
self.log.error(
|
||||||
|
"All ZK event queues empty: %s",
|
||||||
|
self.__areZooKeeperEventQueuesEmpty(matcher),
|
||||||
|
)
|
||||||
self.log.error("All builds waiting: %s" %
|
self.log.error("All builds waiting: %s" %
|
||||||
(self.__areAllBuildsWaiting(matcher),))
|
(self.__areAllBuildsWaiting(matcher),))
|
||||||
self.log.error("All merge jobs waiting: %s" %
|
self.log.error("All merge jobs waiting: %s" %
|
||||||
|
@ -4953,7 +4967,8 @@ class ZuulTestCase(BaseTestCase):
|
||||||
self.__eventQueuesJoin(matcher)
|
self.__eventQueuesJoin(matcher)
|
||||||
self.scheds.execute(
|
self.scheds.execute(
|
||||||
lambda app: app.sched.run_handler_lock.acquire())
|
lambda app: app.sched.run_handler_lock.acquire())
|
||||||
if (self.__areAllMergeJobsWaiting(matcher) and
|
if (self.__areZooKeeperEventQueuesEmpty(matcher) and
|
||||||
|
self.__areAllMergeJobsWaiting(matcher) and
|
||||||
self.__haveAllBuildsReported(matcher) and
|
self.__haveAllBuildsReported(matcher) and
|
||||||
self.__areAllBuildsWaiting(matcher) and
|
self.__areAllBuildsWaiting(matcher) and
|
||||||
self.__areAllNodeRequestsComplete(matcher) and
|
self.__areAllNodeRequestsComplete(matcher) and
|
||||||
|
|
|
@ -12,6 +12,8 @@
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
|
|
||||||
|
from unittest import mock
|
||||||
|
|
||||||
from tests.base import ZuulTestCase, ZuulGithubAppTestCase
|
from tests.base import ZuulTestCase, ZuulGithubAppTestCase
|
||||||
from zuul.driver.zuul.zuulmodel import ZuulTriggerEvent
|
from zuul.driver.zuul.zuulmodel import ZuulTriggerEvent
|
||||||
|
|
||||||
|
@ -63,29 +65,24 @@ class TestZuulTriggerParentChangeEnqueued(ZuulTestCase):
|
||||||
# Now directly enqueue a change into the check. As no pipeline reacts
|
# Now directly enqueue a change into the check. As no pipeline reacts
|
||||||
# on parent-change-enqueued from pipeline check no
|
# on parent-change-enqueued from pipeline check no
|
||||||
# parent-change-enqueued event is expected.
|
# parent-change-enqueued event is expected.
|
||||||
zuultrigger_event_count = 0
|
_add_trigger_event = self.scheds.first.sched.addTriggerEvent
|
||||||
|
|
||||||
def counting_put(*args, **kwargs):
|
def addTriggerEvent(driver_name, event):
|
||||||
nonlocal zuultrigger_event_count
|
self.assertNotIsInstance(event, ZuulTriggerEvent)
|
||||||
if isinstance(args[0], ZuulTriggerEvent):
|
_add_trigger_event(driver_name, event)
|
||||||
zuultrigger_event_count += 1
|
|
||||||
self.scheds.first.sched.trigger_event_queue\
|
|
||||||
.put_orig(*args, **kwargs)
|
|
||||||
|
|
||||||
self.scheds.first.sched.trigger_event_queue.put_orig = \
|
with mock.patch.object(
|
||||||
self.scheds.first.sched.trigger_event_queue.put
|
self.scheds.first.sched, "addTriggerEvent", addTriggerEvent
|
||||||
self.scheds.first.sched.trigger_event_queue.put = counting_put
|
):
|
||||||
|
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
|
||||||
|
C.addApproval('Verified', -1)
|
||||||
|
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
|
||||||
|
D.addApproval('Verified', -1)
|
||||||
|
D.setDependsOn(C, 1)
|
||||||
|
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
|
||||||
|
|
||||||
C = self.fake_gerrit.addFakeChange('org/project', 'master', 'C')
|
self.waitUntilSettled()
|
||||||
C.addApproval('Verified', -1)
|
self.assertEqual(len(self.history), 4)
|
||||||
D = self.fake_gerrit.addFakeChange('org/project', 'master', 'D')
|
|
||||||
D.addApproval('Verified', -1)
|
|
||||||
D.setDependsOn(C, 1)
|
|
||||||
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
|
|
||||||
|
|
||||||
self.waitUntilSettled()
|
|
||||||
self.assertEqual(len(self.history), 4)
|
|
||||||
self.assertEqual(zuultrigger_event_count, 0)
|
|
||||||
|
|
||||||
|
|
||||||
class TestZuulTriggerParentChangeEnqueuedGithub(ZuulGithubAppTestCase):
|
class TestZuulTriggerParentChangeEnqueuedGithub(ZuulGithubAppTestCase):
|
||||||
|
@ -145,31 +142,31 @@ class TestZuulTriggerParentChangeEnqueuedGithub(ZuulGithubAppTestCase):
|
||||||
# on parent-change-enqueued from pipeline check no
|
# on parent-change-enqueued from pipeline check no
|
||||||
# parent-change-enqueued event is expected.
|
# parent-change-enqueued event is expected.
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
zuultrigger_event_count = 0
|
|
||||||
|
|
||||||
def counting_put(*args, **kwargs):
|
_add_trigger_event = self.scheds.first.sched.addTriggerEvent
|
||||||
nonlocal zuultrigger_event_count
|
|
||||||
if isinstance(args[0], ZuulTriggerEvent):
|
|
||||||
zuultrigger_event_count += 1
|
|
||||||
self.scheds.first.sched.trigger_event_queue\
|
|
||||||
.put_orig(*args, **kwargs)
|
|
||||||
|
|
||||||
self.scheds.first.sched.trigger_event_queue.put_orig = \
|
def addTriggerEvent(driver_name, event):
|
||||||
self.scheds.first.sched.trigger_event_queue.put
|
self.assertNotIsInstance(event, ZuulTriggerEvent)
|
||||||
self.scheds.first.sched.trigger_event_queue.put = counting_put
|
_add_trigger_event(driver_name, event)
|
||||||
|
|
||||||
C = self.fake_github.openFakePullRequest('org/project', 'master', 'C')
|
with mock.patch.object(
|
||||||
C.addLabel('for-check') # should go to check
|
self.scheds.first.sched, "addTriggerEvent", addTriggerEvent
|
||||||
|
):
|
||||||
|
C = self.fake_github.openFakePullRequest(
|
||||||
|
'org/project', 'master', 'C'
|
||||||
|
)
|
||||||
|
C.addLabel('for-check') # should go to check
|
||||||
|
|
||||||
msg = "Depends-On: https://github.com/org/project1/pull/%s" % C.number
|
msg = "Depends-On: https://github.com/org/project1/pull/{}".format(
|
||||||
D = self.fake_github.openFakePullRequest(
|
C.number
|
||||||
'org/project', 'master', 'D', body=msg)
|
)
|
||||||
D.addLabel('for-check') # should go to check
|
D = self.fake_github.openFakePullRequest(
|
||||||
self.fake_github.emitEvent(C.getPullRequestOpenedEvent())
|
'org/project', 'master', 'D', body=msg)
|
||||||
|
D.addLabel('for-check') # should go to check
|
||||||
|
self.fake_github.emitEvent(C.getPullRequestOpenedEvent())
|
||||||
|
|
||||||
self.waitUntilSettled()
|
self.waitUntilSettled()
|
||||||
self.assertEqual(len(self.history), 4)
|
self.assertEqual(len(self.history), 4)
|
||||||
self.assertEqual(zuultrigger_event_count, 0)
|
|
||||||
|
|
||||||
# After starting recording installation containing org2/project
|
# After starting recording installation containing org2/project
|
||||||
# should not be contacted
|
# should not be contacted
|
||||||
|
|
|
@ -71,7 +71,6 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
||||||
self.log.exception("Reconfiguration failed:")
|
self.log.exception("Reconfiguration failed:")
|
||||||
|
|
||||||
def exit_handler(self, signum, frame):
|
def exit_handler(self, signum, frame):
|
||||||
self.sched.exit()
|
|
||||||
self.sched.join()
|
self.sched.join()
|
||||||
self.stop_gear_server()
|
self.stop_gear_server()
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
@ -146,7 +145,6 @@ class Scheduler(zuul.cmd.ZuulDaemonApp):
|
||||||
self.sched.start()
|
self.sched.start()
|
||||||
self.sched.reconfigure(self.config,
|
self.sched.reconfigure(self.config,
|
||||||
validate_tenants=self.args.validate_tenants)
|
validate_tenants=self.args.validate_tenants)
|
||||||
self.sched.wakeUp()
|
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Error starting Zuul:")
|
self.log.exception("Error starting Zuul:")
|
||||||
# TODO(jeblair): If we had all threads marked as daemon,
|
# TODO(jeblair): If we had all threads marked as daemon,
|
||||||
|
|
|
@ -258,7 +258,9 @@ class GerritEventConnector(threading.Thread):
|
||||||
|
|
||||||
self._getChange(event)
|
self._getChange(event)
|
||||||
self.connection.logEvent(event)
|
self.connection.logEvent(event)
|
||||||
self.connection.sched.addEvent(event)
|
self.connection.sched.addTriggerEvent(
|
||||||
|
self.connection.driver_name, event
|
||||||
|
)
|
||||||
|
|
||||||
def _getChange(self, event):
|
def _getChange(self, event):
|
||||||
# Grab the change if we are managing the project or if it exists in the
|
# Grab the change if we are managing the project or if it exists in the
|
||||||
|
|
|
@ -151,7 +151,7 @@ class GitConnection(BaseConnection):
|
||||||
self.getChange(event)
|
self.getChange(event)
|
||||||
self.logEvent(event)
|
self.logEvent(event)
|
||||||
# Pass the event to the scheduler
|
# Pass the event to the scheduler
|
||||||
self.sched.addEvent(event)
|
self.sched.addTriggerEvent(self.driver_name, event)
|
||||||
|
|
||||||
def onLoad(self):
|
def onLoad(self):
|
||||||
self.log.debug("Starting Git Watcher")
|
self.log.debug("Starting Git Watcher")
|
||||||
|
|
|
@ -694,13 +694,7 @@ class GithubEventProcessor(object):
|
||||||
return event
|
return event
|
||||||
|
|
||||||
def _get_sender(self, body):
|
def _get_sender(self, body):
|
||||||
login = body.get('sender').get('login')
|
return body.get('sender').get('login')
|
||||||
if login:
|
|
||||||
# TODO(tobiash): it might be better to plumb in the installation id
|
|
||||||
project = body.get('repository', {}).get('full_name')
|
|
||||||
user = self.connection.getUser(login, project)
|
|
||||||
self.log.debug("Got user %s", user)
|
|
||||||
return user
|
|
||||||
|
|
||||||
|
|
||||||
class GithubEventConnector:
|
class GithubEventConnector:
|
||||||
|
@ -756,9 +750,15 @@ class GithubEventConnector:
|
||||||
if future is None:
|
if future is None:
|
||||||
return
|
return
|
||||||
event = future.result()
|
event = future.result()
|
||||||
if event:
|
if not event:
|
||||||
self.connection.logEvent(event)
|
return
|
||||||
self.connection.sched.addEvent(event)
|
self.connection.logEvent(event)
|
||||||
|
if isinstance(event, DequeueEvent):
|
||||||
|
self.connection.sched.addManagementEvent(event)
|
||||||
|
else:
|
||||||
|
self.connection.sched.addTriggerEvent(
|
||||||
|
self.connection.driver_name, event
|
||||||
|
)
|
||||||
except Exception:
|
except Exception:
|
||||||
self.log.exception("Exception moving GitHub event:")
|
self.log.exception("Exception moving GitHub event:")
|
||||||
finally:
|
finally:
|
||||||
|
|
|
@ -146,7 +146,7 @@ class GitlabEventConnector(threading.Thread):
|
||||||
label["title"] for
|
label["title"] for
|
||||||
label in body["changes"]["labels"]["current"]]
|
label in body["changes"]["labels"]["current"]]
|
||||||
new_labels = set(current_labels) - set(previous_labels)
|
new_labels = set(current_labels) - set(previous_labels)
|
||||||
event.labels = new_labels
|
event.labels = list(new_labels)
|
||||||
elif attrs['action'] in ('approved', 'unapproved'):
|
elif attrs['action'] in ('approved', 'unapproved'):
|
||||||
event.action = attrs['action']
|
event.action = attrs['action']
|
||||||
else:
|
else:
|
||||||
|
@ -237,7 +237,9 @@ class GitlabEventConnector(threading.Thread):
|
||||||
self.connection.checkBranchCache(event.project_name, event)
|
self.connection.checkBranchCache(event.project_name, event)
|
||||||
|
|
||||||
self.connection.logEvent(event)
|
self.connection.logEvent(event)
|
||||||
self.connection.sched.addEvent(event)
|
self.connection.sched.addTriggerEvent(
|
||||||
|
self.connection.driver_name, event
|
||||||
|
)
|
||||||
|
|
||||||
def run(self):
|
def run(self):
|
||||||
while True:
|
while True:
|
||||||
|
|
|
@ -88,7 +88,7 @@ class GitlabTriggerEvent(TriggerEvent):
|
||||||
r = [super(GitlabTriggerEvent, self)._repr()]
|
r = [super(GitlabTriggerEvent, self)._repr()]
|
||||||
if self.action:
|
if self.action:
|
||||||
r.append("action:%s" % self.action)
|
r.append("action:%s" % self.action)
|
||||||
r.append("project:%s" % self.canonical_project_name)
|
r.append("project:%s" % self.project_name)
|
||||||
if self.change_number:
|
if self.change_number:
|
||||||
r.append("mr:%s" % self.change_number)
|
r.append("mr:%s" % self.change_number)
|
||||||
if self.labels:
|
if self.labels:
|
||||||
|
|
|
@ -255,7 +255,9 @@ class PagureEventConnector(threading.Thread):
|
||||||
event=event)
|
event=event)
|
||||||
event.project_hostname = self.connection.canonical_hostname
|
event.project_hostname = self.connection.canonical_hostname
|
||||||
self.connection.logEvent(event)
|
self.connection.logEvent(event)
|
||||||
self.connection.sched.addEvent(event)
|
self.connection.sched.addTriggerEvent(
|
||||||
|
self.connection.driver_name, event
|
||||||
|
)
|
||||||
|
|
||||||
def _event_base(self, body, pull_data_field='pullrequest'):
|
def _event_base(self, body, pull_data_field='pullrequest'):
|
||||||
event = PagureTriggerEvent()
|
event = PagureTriggerEvent()
|
||||||
|
|
|
@ -130,7 +130,7 @@ class TimerDriver(Driver, TriggerInterface):
|
||||||
event.timestamp = time.time()
|
event.timestamp = time.time()
|
||||||
log = get_annotated_logger(self.log, event)
|
log = get_annotated_logger(self.log, event)
|
||||||
log.debug("Adding event")
|
log.debug("Adding event")
|
||||||
self.sched.addEvent(event)
|
self.sched.addTriggerEvent(self.name, event)
|
||||||
|
|
||||||
def stop(self):
|
def stop(self):
|
||||||
if self.apsched:
|
if self.apsched:
|
||||||
|
|
|
@ -96,7 +96,7 @@ class ZuulDriver(Driver, TriggerInterface):
|
||||||
event.ref = change.ref
|
event.ref = change.ref
|
||||||
event.zuul_event_id = str(uuid4().hex)
|
event.zuul_event_id = str(uuid4().hex)
|
||||||
event.timestamp = time.time()
|
event.timestamp = time.time()
|
||||||
self.sched.addEvent(event)
|
self.sched.addTriggerEvent(self.name, event)
|
||||||
|
|
||||||
def _createParentChangeEnqueuedEvents(self, change, pipeline, tenant,
|
def _createParentChangeEnqueuedEvents(self, change, pipeline, tenant,
|
||||||
event):
|
event):
|
||||||
|
@ -133,7 +133,7 @@ class ZuulDriver(Driver, TriggerInterface):
|
||||||
event.patch_number = change.patchset
|
event.patch_number = change.patchset
|
||||||
event.ref = change.ref
|
event.ref = change.ref
|
||||||
event.zuul_event_id = str(uuid4().hex)
|
event.zuul_event_id = str(uuid4().hex)
|
||||||
self.sched.addEvent(event)
|
self.sched.addTriggerEvent(self.name, event)
|
||||||
|
|
||||||
def getTrigger(self, connection_name, config=None):
|
def getTrigger(self, connection_name, config=None):
|
||||||
return zuultrigger.ZuulTrigger(self, config)
|
return zuultrigger.ZuulTrigger(self, config)
|
||||||
|
|
|
@ -18,7 +18,6 @@
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import pickle
|
|
||||||
import re
|
import re
|
||||||
import queue
|
import queue
|
||||||
import socket
|
import socket
|
||||||
|
@ -53,19 +52,22 @@ from zuul.model import (
|
||||||
EnqueueEvent,
|
EnqueueEvent,
|
||||||
FilesChangesCompletedEvent,
|
FilesChangesCompletedEvent,
|
||||||
HoldRequest,
|
HoldRequest,
|
||||||
ManagementEvent,
|
|
||||||
MergeCompletedEvent,
|
MergeCompletedEvent,
|
||||||
NodesProvisionedEvent,
|
NodesProvisionedEvent,
|
||||||
PromoteEvent,
|
PromoteEvent,
|
||||||
ReconfigureEvent,
|
ReconfigureEvent,
|
||||||
ResultEvent,
|
|
||||||
SmartReconfigureEvent,
|
SmartReconfigureEvent,
|
||||||
Tenant,
|
Tenant,
|
||||||
TenantReconfigureEvent,
|
TenantReconfigureEvent,
|
||||||
TriggerEvent,
|
|
||||||
)
|
)
|
||||||
from zuul.zk import ZooKeeperClient
|
from zuul.zk import ZooKeeperClient
|
||||||
from zuul.zk.components import ZooKeeperComponentRegistry
|
from zuul.zk.components import ZooKeeperComponentRegistry
|
||||||
|
from zuul.zk.event_queues import (
|
||||||
|
GlobalEventWatcher,
|
||||||
|
GlobalTriggerEventQueue,
|
||||||
|
PipelineEventWatcher,
|
||||||
|
PipelineTriggerEventQueue,
|
||||||
|
)
|
||||||
from zuul.zk.nodepool import ZooKeeperNodepool
|
from zuul.zk.nodepool import ZooKeeperNodepool
|
||||||
|
|
||||||
COMMANDS = ['full-reconfigure', 'smart-reconfigure', 'stop', 'repl', 'norepl']
|
COMMANDS = ['full-reconfigure', 'smart-reconfigure', 'stop', 'repl', 'norepl']
|
||||||
|
@ -113,7 +115,6 @@ class Scheduler(threading.Thread):
|
||||||
'repl': self.start_repl,
|
'repl': self.start_repl,
|
||||||
'norepl': self.stop_repl,
|
'norepl': self.stop_repl,
|
||||||
}
|
}
|
||||||
self._hibernate = False
|
|
||||||
self._stopped = False
|
self._stopped = False
|
||||||
|
|
||||||
self._zuul_app = app
|
self._zuul_app = app
|
||||||
|
@ -143,9 +144,22 @@ class Scheduler(threading.Thread):
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
self.trigger_event_queue = queue.Queue()
|
|
||||||
self.result_event_queue = queue.Queue()
|
self.result_event_queue = queue.Queue()
|
||||||
self.management_event_queue = zuul.lib.queue.MergedQueue()
|
self.management_event_queue = zuul.lib.queue.MergedQueue()
|
||||||
|
self.global_watcher = GlobalEventWatcher(
|
||||||
|
self.zk_client, self.wake_event.set
|
||||||
|
)
|
||||||
|
self.pipeline_watcher = PipelineEventWatcher(
|
||||||
|
self.zk_client, self.wake_event.set
|
||||||
|
)
|
||||||
|
self.trigger_events = GlobalTriggerEventQueue(
|
||||||
|
self.zk_client, self.connections
|
||||||
|
)
|
||||||
|
self.pipeline_trigger_events = (
|
||||||
|
PipelineTriggerEventQueue.createRegistry(
|
||||||
|
self.zk_client, self.connections
|
||||||
|
)
|
||||||
|
)
|
||||||
self.abide = model.Abide()
|
self.abide = model.Abide()
|
||||||
self.unparsed_abide = model.UnparsedAbideConfig()
|
self.unparsed_abide = model.UnparsedAbideConfig()
|
||||||
|
|
||||||
|
@ -298,38 +312,23 @@ class Scheduler(threading.Thread):
|
||||||
self.statsd.gauge('zuul.mergers.jobs_running', merge_running)
|
self.statsd.gauge('zuul.mergers.jobs_running', merge_running)
|
||||||
self.statsd.gauge('zuul.mergers.jobs_queued', merge_queue)
|
self.statsd.gauge('zuul.mergers.jobs_queued', merge_queue)
|
||||||
self.statsd.gauge('zuul.scheduler.eventqueues.trigger',
|
self.statsd.gauge('zuul.scheduler.eventqueues.trigger',
|
||||||
self.trigger_event_queue.qsize())
|
len(self.trigger_events))
|
||||||
self.statsd.gauge('zuul.scheduler.eventqueues.result',
|
self.statsd.gauge('zuul.scheduler.eventqueues.result',
|
||||||
self.result_event_queue.qsize())
|
self.result_event_queue.qsize())
|
||||||
self.statsd.gauge('zuul.scheduler.eventqueues.management',
|
self.statsd.gauge('zuul.scheduler.eventqueues.management',
|
||||||
self.management_event_queue.qsize())
|
self.management_event_queue.qsize())
|
||||||
|
|
||||||
def addEvent(self, event):
|
def addTriggerEvent(self, driver_name, event):
|
||||||
# Check the event type and put it in the corresponding queue
|
|
||||||
if isinstance(event, TriggerEvent):
|
|
||||||
return self._addTriggerEvent(event)
|
|
||||||
|
|
||||||
if isinstance(event, ManagementEvent):
|
|
||||||
return self._addManagementEvent(event)
|
|
||||||
|
|
||||||
if isinstance(event, ResultEvent):
|
|
||||||
return self._addResultEvent(event)
|
|
||||||
|
|
||||||
self.log.warning(
|
|
||||||
"Unable to found appropriate queue for event %s", event
|
|
||||||
)
|
|
||||||
|
|
||||||
def _addTriggerEvent(self, event):
|
|
||||||
event.arrived_at_scheduler_timestamp = time.time()
|
event.arrived_at_scheduler_timestamp = time.time()
|
||||||
self.trigger_event_queue.put(event)
|
self.trigger_events.put(driver_name, event)
|
||||||
self.wake_event.set()
|
self.wake_event.set()
|
||||||
|
|
||||||
def _addManagementEvent(self, event):
|
def addManagementEvent(self, event):
|
||||||
self.management_event_queue.put(event)
|
self.management_event_queue.put(event)
|
||||||
self.wake_event.set()
|
self.wake_event.set()
|
||||||
event.wait()
|
event.wait()
|
||||||
|
|
||||||
def _addResultEvent(self, event):
|
def addResultEvent(self, event):
|
||||||
self.result_event_queue.put(event)
|
self.result_event_queue.put(event)
|
||||||
self.wake_event.set()
|
self.wake_event.set()
|
||||||
|
|
||||||
|
@ -570,17 +569,6 @@ class Scheduler(threading.Thread):
|
||||||
event.wait()
|
event.wait()
|
||||||
self.log.debug("Enqueue complete")
|
self.log.debug("Enqueue complete")
|
||||||
|
|
||||||
def exit(self):
|
|
||||||
self.log.debug("Prepare to exit")
|
|
||||||
self._hibernate = True
|
|
||||||
self.wake_event.set()
|
|
||||||
self.log.debug("Waiting for exit")
|
|
||||||
|
|
||||||
def _get_queue_pickle_file(self):
|
|
||||||
state_dir = get_default(self.config, 'scheduler', 'state_dir',
|
|
||||||
'/var/lib/zuul', expand_user=True)
|
|
||||||
return os.path.join(state_dir, 'queue.pickle')
|
|
||||||
|
|
||||||
def _get_time_database_dir(self):
|
def _get_time_database_dir(self):
|
||||||
state_dir = get_default(self.config, 'scheduler', 'state_dir',
|
state_dir = get_default(self.config, 'scheduler', 'state_dir',
|
||||||
'/var/lib/zuul', expand_user=True)
|
'/var/lib/zuul', expand_user=True)
|
||||||
|
@ -602,60 +590,6 @@ class Scheduler(threading.Thread):
|
||||||
"current mode is %o" % (key_dir, mode))
|
"current mode is %o" % (key_dir, mode))
|
||||||
return key_dir
|
return key_dir
|
||||||
|
|
||||||
def _save_queue(self) -> None:
|
|
||||||
# TODO JK: Remove when queues in ZK
|
|
||||||
pickle_file = self._get_queue_pickle_file()
|
|
||||||
events = []
|
|
||||||
while not self.trigger_event_queue.empty():
|
|
||||||
events.append(self.trigger_event_queue.get())
|
|
||||||
self.log.debug("Queue length is %s" % len(events))
|
|
||||||
if events:
|
|
||||||
self.log.debug("Saving queue")
|
|
||||||
pickle.dump(events, open(pickle_file, 'wb'))
|
|
||||||
|
|
||||||
def _load_queue(self) -> None:
|
|
||||||
# TODO JK: Remove when queues in ZK
|
|
||||||
pickle_file = self._get_queue_pickle_file()
|
|
||||||
if os.path.exists(pickle_file):
|
|
||||||
self.log.debug("Loading queue")
|
|
||||||
events = pickle.load(open(pickle_file, 'rb'))
|
|
||||||
self.log.debug("Queue length is %s" % len(events))
|
|
||||||
for event in events:
|
|
||||||
self.trigger_event_queue.put(event)
|
|
||||||
else:
|
|
||||||
self.log.debug("No queue file found")
|
|
||||||
|
|
||||||
def _delete_queue(self) -> None:
|
|
||||||
# TODO JK: Remove when queues in ZK
|
|
||||||
pickle_file = self._get_queue_pickle_file()
|
|
||||||
if os.path.exists(pickle_file):
|
|
||||||
self.log.debug("Deleting saved queue")
|
|
||||||
os.unlink(pickle_file)
|
|
||||||
|
|
||||||
def wakeUp(self) -> None:
|
|
||||||
"""
|
|
||||||
Wakes up scheduler by loading pickled queue.
|
|
||||||
|
|
||||||
TODO JK: Remove when queues in ZK
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
self._load_queue()
|
|
||||||
except Exception:
|
|
||||||
self.log.exception("Unable to load queue")
|
|
||||||
try:
|
|
||||||
self._delete_queue()
|
|
||||||
except Exception:
|
|
||||||
self.log.exception("Unable to delete saved queue")
|
|
||||||
self.log.debug("Resuming queue processing")
|
|
||||||
self.wake_event.set()
|
|
||||||
|
|
||||||
def _doHibernate(self) -> None:
|
|
||||||
# TODO JK: Remove when queues in ZK
|
|
||||||
if self._hibernate:
|
|
||||||
self.log.debug("Exiting")
|
|
||||||
self._save_queue()
|
|
||||||
os._exit(0)
|
|
||||||
|
|
||||||
def _checkTenantSourceConf(self, config):
|
def _checkTenantSourceConf(self, config):
|
||||||
tenant_config = None
|
tenant_config = None
|
||||||
script = False
|
script = False
|
||||||
|
@ -1109,13 +1043,11 @@ class Scheduler(threading.Thread):
|
||||||
not self._stopped):
|
not self._stopped):
|
||||||
self.process_result_queue()
|
self.process_result_queue()
|
||||||
|
|
||||||
if not self._hibernate:
|
if not self._stopped:
|
||||||
while (not self.trigger_event_queue.empty() and
|
self.process_global_trigger_queue()
|
||||||
not self._stopped):
|
|
||||||
self.process_event_queue()
|
|
||||||
|
|
||||||
if self._hibernate and self._areAllBuildsComplete():
|
if not self._stopped:
|
||||||
self._doHibernate()
|
self.process_trigger_queue()
|
||||||
|
|
||||||
for tenant in self.abide.tenants.values():
|
for tenant in self.abide.tenants.values():
|
||||||
for pipeline in tenant.layout.pipelines.values():
|
for pipeline in tenant.layout.pipelines.values():
|
||||||
|
@ -1154,72 +1086,126 @@ class Scheduler(threading.Thread):
|
||||||
"End maintain connection cache for: %s" % connection)
|
"End maintain connection cache for: %s" % connection)
|
||||||
self.log.debug("Connection cache size: %s" % len(relevant))
|
self.log.debug("Connection cache size: %s" % len(relevant))
|
||||||
|
|
||||||
def process_event_queue(self):
|
def process_global_trigger_queue(self):
|
||||||
self.log.debug("Fetching trigger event")
|
for event in self.trigger_events:
|
||||||
event = self.trigger_event_queue.get()
|
log = get_annotated_logger(
|
||||||
log = get_annotated_logger(self.log, event.zuul_event_id)
|
self.log, event.zuul_event_id
|
||||||
log.debug("Processing trigger event %s" % event)
|
)
|
||||||
|
log.debug("Forwarding trigger event %s", event)
|
||||||
|
try:
|
||||||
|
for tenant in self.abide.tenants.values():
|
||||||
|
self._forward_trigger_event(event, tenant)
|
||||||
|
finally:
|
||||||
|
self.trigger_events.ack(event)
|
||||||
|
|
||||||
|
def _forward_trigger_event(self, event, tenant):
|
||||||
|
log = get_annotated_logger(
|
||||||
|
self.log, event.zuul_event_id
|
||||||
|
)
|
||||||
|
_, project = tenant.getProject(
|
||||||
|
event.canonical_project_name
|
||||||
|
)
|
||||||
|
if project is None:
|
||||||
|
return
|
||||||
|
|
||||||
try:
|
try:
|
||||||
full_project_name = ('/'.join([event.project_hostname,
|
change = project.source.getChange(event)
|
||||||
event.project_name]))
|
except exceptions.ChangeNotFound as e:
|
||||||
for tenant in self.abide.tenants.values():
|
log.debug("Unable to get change %s from source %s",
|
||||||
(trusted, project) = tenant.getProject(full_project_name)
|
e.change, project.source)
|
||||||
if project is None:
|
return
|
||||||
continue
|
|
||||||
try:
|
|
||||||
change = project.source.getChange(event)
|
|
||||||
except exceptions.ChangeNotFound as e:
|
|
||||||
log.debug("Unable to get change %s from source %s",
|
|
||||||
e.change, project.source)
|
|
||||||
continue
|
|
||||||
reconfigure_tenant = False
|
|
||||||
if ((event.branch_updated and
|
|
||||||
hasattr(change, 'files') and
|
|
||||||
change.updatesConfig(tenant)) or
|
|
||||||
(event.branch_deleted and
|
|
||||||
self.abide.hasUnparsedBranchCache(project.canonical_name,
|
|
||||||
event.branch))):
|
|
||||||
reconfigure_tenant = True
|
|
||||||
|
|
||||||
# The branch_created attribute is also true when a tag is
|
reconfigure_tenant = False
|
||||||
# created. Since we load config only from branches only trigger
|
if ((event.branch_updated and
|
||||||
# a tenant reconfiguration if the branch is set as well.
|
hasattr(change, 'files') and
|
||||||
if event.branch_created and event.branch:
|
change.updatesConfig(tenant)) or
|
||||||
reconfigure_tenant = True
|
(event.branch_deleted and
|
||||||
|
self.abide.hasUnparsedBranchCache(project.canonical_name,
|
||||||
|
event.branch))):
|
||||||
|
reconfigure_tenant = True
|
||||||
|
|
||||||
# If the driver knows the branch but we don't have a config, we
|
# The branch_created attribute is also true when a tag is
|
||||||
# also need to reconfigure. This happens if a GitHub branch
|
# created. Since we load config only from branches only trigger
|
||||||
# was just configured as protected without a push in between.
|
# a tenant reconfiguration if the branch is set as well.
|
||||||
if (event.branch in project.source.getProjectBranches(
|
if event.branch_created and event.branch:
|
||||||
project, tenant)
|
reconfigure_tenant = True
|
||||||
and not self.abide.hasUnparsedBranchCache(
|
|
||||||
project.canonical_name, event.branch)):
|
|
||||||
reconfigure_tenant = True
|
|
||||||
|
|
||||||
# If the branch is unprotected and unprotected branches
|
# If the driver knows the branch but we don't have a config, we
|
||||||
# are excluded from the tenant for that project skip reconfig.
|
# also need to reconfigure. This happens if a GitHub branch
|
||||||
if (reconfigure_tenant and not
|
# was just configured as protected without a push in between.
|
||||||
event.branch_protected and
|
if (event.branch in project.source.getProjectBranches(
|
||||||
tenant.getExcludeUnprotectedBranches(project)):
|
project, tenant)
|
||||||
|
and not self.abide.hasUnparsedBranchCache(
|
||||||
|
project.canonical_name, event.branch)):
|
||||||
|
reconfigure_tenant = True
|
||||||
|
|
||||||
reconfigure_tenant = False
|
# If the branch is unprotected and unprotected branches
|
||||||
|
# are excluded from the tenant for that project skip reconfig.
|
||||||
|
if (reconfigure_tenant and not
|
||||||
|
event.branch_protected and
|
||||||
|
tenant.getExcludeUnprotectedBranches(project)):
|
||||||
|
|
||||||
if reconfigure_tenant:
|
reconfigure_tenant = False
|
||||||
# The change that just landed updates the config
|
|
||||||
# or a branch was just created or deleted. Clear
|
if reconfigure_tenant:
|
||||||
# out cached data for this project and perform a
|
# The change that just landed updates the config
|
||||||
# reconfiguration.
|
# or a branch was just created or deleted. Clear
|
||||||
self.reconfigureTenant(tenant, change.project, event)
|
# out cached data for this project and perform a
|
||||||
for pipeline in tenant.layout.pipelines.values():
|
# reconfiguration.
|
||||||
if event.isPatchsetCreated():
|
self.reconfigureTenant(tenant, change.project, event)
|
||||||
pipeline.manager.removeOldVersionsOfChange(
|
|
||||||
change, event)
|
for pipeline in tenant.layout.pipelines.values():
|
||||||
elif event.isChangeAbandoned():
|
if (
|
||||||
pipeline.manager.removeAbandonedChange(change, event)
|
pipeline.manager.eventMatches(event, change)
|
||||||
if pipeline.manager.eventMatches(event, change):
|
or pipeline.manager.isChangeAlreadyInPipeline(change)
|
||||||
pipeline.manager.addChange(change, event)
|
or pipeline.manager.findOldVersionOfChangeAlreadyInQueue(
|
||||||
finally:
|
change
|
||||||
self.trigger_event_queue.task_done()
|
)
|
||||||
|
):
|
||||||
|
self.pipeline_trigger_events[tenant.name][
|
||||||
|
pipeline.name
|
||||||
|
].put(event.driver_name, event)
|
||||||
|
|
||||||
|
def process_trigger_queue(self):
|
||||||
|
for tenant in self.abide.tenants.values():
|
||||||
|
for pipeline in tenant.layout.pipelines.values():
|
||||||
|
for event in self.pipeline_trigger_events[tenant.name][
|
||||||
|
pipeline.name
|
||||||
|
]:
|
||||||
|
if self._stopped:
|
||||||
|
return
|
||||||
|
log = get_annotated_logger(
|
||||||
|
self.log, event.zuul_event_id
|
||||||
|
)
|
||||||
|
log.debug("Processing trigger event %s", event)
|
||||||
|
try:
|
||||||
|
self._process_trigger_event(tenant, pipeline, event)
|
||||||
|
finally:
|
||||||
|
self.pipeline_trigger_events[tenant.name][
|
||||||
|
pipeline.name
|
||||||
|
].ack(event)
|
||||||
|
|
||||||
|
def _process_trigger_event(self, tenant, pipeline, event):
|
||||||
|
log = get_annotated_logger(
|
||||||
|
self.log, event.zuul_event_id
|
||||||
|
)
|
||||||
|
trusted, project = tenant.getProject(event.canonical_project_name)
|
||||||
|
if project is None:
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
change = project.source.getChange(event)
|
||||||
|
except exceptions.ChangeNotFound as e:
|
||||||
|
log.debug("Unable to get change %s from source %s",
|
||||||
|
e.change, project.source)
|
||||||
|
return
|
||||||
|
|
||||||
|
if event.isPatchsetCreated():
|
||||||
|
pipeline.manager.removeOldVersionsOfChange(
|
||||||
|
change, event)
|
||||||
|
elif event.isChangeAbandoned():
|
||||||
|
pipeline.manager.removeAbandonedChange(change, event)
|
||||||
|
if pipeline.manager.eventMatches(event, change):
|
||||||
|
pipeline.manager.addChange(change, event)
|
||||||
|
|
||||||
def process_management_queue(self):
|
def process_management_queue(self):
|
||||||
self.log.debug("Fetching management event")
|
self.log.debug("Fetching management event")
|
||||||
|
@ -1545,14 +1531,8 @@ class Scheduler(threading.Thread):
|
||||||
data['zuul_version'] = self.zuul_version
|
data['zuul_version'] = self.zuul_version
|
||||||
websocket_url = get_default(self.config, 'web', 'websocket_url', None)
|
websocket_url = get_default(self.config, 'web', 'websocket_url', None)
|
||||||
|
|
||||||
if self._hibernate:
|
|
||||||
data['message'] = 'Queue only mode: preparing to hibernate,' \
|
|
||||||
' queue length: %s'\
|
|
||||||
% self.trigger_event_queue.qsize()
|
|
||||||
|
|
||||||
data['trigger_event_queue'] = {}
|
data['trigger_event_queue'] = {}
|
||||||
data['trigger_event_queue']['length'] = \
|
data['trigger_event_queue']['length'] = len(self.trigger_events)
|
||||||
self.trigger_event_queue.qsize()
|
|
||||||
data['result_event_queue'] = {}
|
data['result_event_queue'] = {}
|
||||||
data['result_event_queue']['length'] = \
|
data['result_event_queue']['length'] = \
|
||||||
self.result_event_queue.qsize()
|
self.result_event_queue.qsize()
|
||||||
|
|
Loading…
Reference in New Issue