Store tenant layout state in Zookeeper
To coordinate reconfigurations across schedulers we need to store the current layout state in Zookeeper. We store the hostname of the scheduler that processed the reconfiguration event as well as the human readable timestamp of the reconfig. The ltime of the layout state is the last modified transaction ID of the corresponding znode in Zookeeper. It will later be used to detect layout changes on other schedulers. Change-Id: I243aa4f54b038fbad78e331fbeeb508989e34b5f
This commit is contained in:
Simon Westphahl
Clark Boylan
parent
c946c26cef
commit
32f3d5fe35
|
|
@ -27,6 +27,7 @@ import github3.exceptions
|
|||
|
||||
from tests.fakegithub import FakeGithubEnterpriseClient
|
||||
from zuul.driver.github.githubconnection import GithubShaCache
|
||||
from zuul.zk.layout import LayoutState
|
||||
import zuul.rpcclient
|
||||
from zuul.lib import strings
|
||||
|
||||
|
|
@ -35,6 +36,8 @@ from tests.base import (AnsibleZuulTestCase, BaseTestCase,
|
|||
simple_layout, random_sha1)
|
||||
from tests.base import ZuulWebFixture
|
||||
|
||||
EMPTY_LAYOUT_STATE = LayoutState("", "", 0)
|
||||
|
||||
|
||||
class TestGithubDriver(ZuulTestCase):
|
||||
config_file = 'zuul-github-driver.conf'
|
||||
|
|
@ -222,8 +225,8 @@ class TestGithubDriver(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
# Record previous tenant reconfiguration time
|
||||
before = self.scheds.first.sched.tenant_last_reconfigured.get(
|
||||
'tenant-one', 0)
|
||||
before = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
self.fake_github.emitEvent(
|
||||
self.fake_github.getPushEvent('org/project', 'refs/tags/newtag',
|
||||
|
|
@ -231,8 +234,8 @@ class TestGithubDriver(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
# Make sure the tenant hasn't been reconfigured due to the new tag
|
||||
after = self.scheds.first.sched.tenant_last_reconfigured.get(
|
||||
'tenant-one', 0)
|
||||
after = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.assertEqual(before, after)
|
||||
|
||||
build_params = self.builds[0].parameters
|
||||
|
|
@ -968,8 +971,8 @@ class TestGithubDriver(ZuulTestCase):
|
|||
removed_files=removed_files)
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
if expected_cat_jobs is not None:
|
||||
|
|
@ -979,8 +982,8 @@ class TestGithubDriver(ZuulTestCase):
|
|||
|
||||
self.fake_github.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
if expect_reconfigure:
|
||||
# New timestamp should be greater than the old timestamp
|
||||
|
|
@ -1434,14 +1437,14 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
modified_files=['zuul.yaml'])
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.fake_github.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We don't expect a reconfiguration because the push was to an
|
||||
# unprotected branch
|
||||
|
|
@ -1454,8 +1457,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_github.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We now expect that zuul reconfigured itself
|
||||
self.assertLess(old, new)
|
||||
|
|
@ -1479,8 +1482,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Delete the branch
|
||||
|
|
@ -1493,8 +1496,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_github.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We now expect that zuul reconfigured itself as we deleted a protected
|
||||
# branch
|
||||
|
|
@ -1554,8 +1557,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
removed_files=removed_files)
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
if expected_cat_jobs is not None:
|
||||
|
|
@ -1565,8 +1568,8 @@ class TestGithubUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_github.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
if expect_reconfigure:
|
||||
# New timestamp should be greater than the old timestamp
|
||||
|
|
|
|||
|
|
@ -20,12 +20,15 @@ import socket
|
|||
|
||||
import zuul.rpcclient
|
||||
from zuul.lib import strings
|
||||
from zuul.zk.layout import LayoutState
|
||||
|
||||
from tests.base import random_sha1, simple_layout
|
||||
from tests.base import ZuulTestCase, ZuulWebFixture
|
||||
|
||||
from testtools.matchers import MatchesRegex
|
||||
|
||||
EMPTY_LAYOUT_STATE = LayoutState("", "", 0)
|
||||
|
||||
|
||||
class TestGitlabWebhook(ZuulTestCase):
|
||||
config_file = 'zuul-gitlab-driver.conf'
|
||||
|
|
@ -294,12 +297,12 @@ class TestGitlabDriver(ZuulTestCase):
|
|||
event = self.fake_gitlab.getPushEvent(
|
||||
'org/project', branch='refs/heads/stable-1.0',
|
||||
before='0' * 40, after=newrev)
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured.get(
|
||||
'tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.fake_gitlab.emitEvent(event)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured.get(
|
||||
'tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
# New timestamp should be greater than the old timestamp
|
||||
self.assertLess(old, new)
|
||||
self.assertEqual(1, len(self.history))
|
||||
|
|
@ -383,8 +386,8 @@ class TestGitlabDriver(ZuulTestCase):
|
|||
def test_ref_updated_and_tenant_reconfigure(self):
|
||||
|
||||
self.waitUntilSettled()
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
zuul_yaml = [
|
||||
{'job': {
|
||||
|
|
@ -410,8 +413,8 @@ class TestGitlabDriver(ZuulTestCase):
|
|||
self.fake_gitlab.emitEvent(event)
|
||||
self.waitUntilSettled()
|
||||
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
# New timestamp should be greater than the old timestamp
|
||||
self.assertLess(old, new)
|
||||
|
||||
|
|
@ -837,14 +840,14 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
branch='refs/heads/master')
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.fake_gitlab.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We don't expect a reconfiguration because the push was to an
|
||||
# unprotected branch
|
||||
|
|
@ -855,8 +858,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_gitlab.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We now expect that zuul reconfigured itself
|
||||
self.assertLess(old, new)
|
||||
|
|
@ -881,8 +884,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
# Delete the branch
|
||||
|
|
@ -895,8 +898,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_gitlab.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
# We now expect that zuul reconfigured itself as we deleted a protected
|
||||
# branch
|
||||
|
|
@ -955,8 +958,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
after=new_sha)
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
if expected_cat_jobs is not None:
|
||||
|
|
@ -966,8 +969,8 @@ class TestGitlabUnprotectedBranches(ZuulTestCase):
|
|||
|
||||
self.fake_gitlab.emitEvent(pevent)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
if expect_reconfigure:
|
||||
# New timestamp should be greater than the old timestamp
|
||||
|
|
|
|||
|
|
@ -22,10 +22,13 @@ from testtools.matchers import MatchesRegex
|
|||
|
||||
import zuul.rpcclient
|
||||
from zuul.lib import strings
|
||||
from zuul.zk.layout import LayoutState
|
||||
|
||||
from tests.base import ZuulTestCase, simple_layout
|
||||
from tests.base import ZuulWebFixture
|
||||
|
||||
EMPTY_LAYOUT_STATE = LayoutState("", "", 0)
|
||||
|
||||
|
||||
class TestPagureDriver(ZuulTestCase):
|
||||
config_file = 'zuul-pagure-driver.conf'
|
||||
|
|
@ -215,12 +218,12 @@ class TestPagureDriver(ZuulTestCase):
|
|||
newrev = repo.commit('refs/heads/stable-1.0').hexsha
|
||||
event = self.fake_pagure.getGitBranchEvent(
|
||||
'org/project', 'stable-1.0', 'creation', newrev)
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.fake_pagure.emitEvent(event)
|
||||
self.waitUntilSettled()
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
# New timestamp should be greater than the old timestamp
|
||||
self.assertLess(old, new)
|
||||
self.assertEqual(1, len(self.history))
|
||||
|
|
@ -248,8 +251,8 @@ class TestPagureDriver(ZuulTestCase):
|
|||
def test_ref_updated_and_tenant_reconfigure(self):
|
||||
|
||||
self.waitUntilSettled()
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
zuul_yaml = [
|
||||
{'job': {
|
||||
|
|
@ -275,8 +278,8 @@ class TestPagureDriver(ZuulTestCase):
|
|||
self.fake_pagure.emitEvent(event)
|
||||
self.waitUntilSettled()
|
||||
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
# New timestamp should be greater than the old timestamp
|
||||
self.assertLess(old, new)
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,9 @@ from tests.base import (
|
|||
RecordingExecutorServer,
|
||||
TestConnectionRegistry,
|
||||
)
|
||||
from zuul.zk.layout import LayoutState
|
||||
|
||||
EMPTY_LAYOUT_STATE = LayoutState("", "", 0)
|
||||
|
||||
|
||||
class TestSchedulerSSL(SSLZuulTestCase):
|
||||
|
|
@ -3655,9 +3658,9 @@ class TestScheduler(ZuulTestCase):
|
|||
def test_live_reconfiguration_command_socket(self):
|
||||
"Test that live reconfiguration via command socket works"
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
# record previous tenant reconfiguration state, which may not be set
|
||||
old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
command_socket = self.scheds.first.config.get(
|
||||
|
|
@ -3673,8 +3676,8 @@ class TestScheduler(ZuulTestCase):
|
|||
while True:
|
||||
if time.time() - start > 15:
|
||||
raise Exception("Timeout waiting for full reconfiguration")
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
if old < new:
|
||||
break
|
||||
else:
|
||||
|
|
@ -8794,10 +8797,10 @@ class TestSchedulerSmartReconfiguration(ZuulTestCase):
|
|||
self.fake_gerrit.addEvent(C.getPatchsetCreatedEvent(1))
|
||||
|
||||
# record previous tenant reconfiguration time, which may not be set
|
||||
old_one = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
old_two = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-two', 0)
|
||||
old_one = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
old_two = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-two', EMPTY_LAYOUT_STATE)
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.newTenantConfig('config/multi-tenant/main-reconfig.yaml')
|
||||
|
|
@ -8813,8 +8816,8 @@ class TestSchedulerSmartReconfiguration(ZuulTestCase):
|
|||
while True:
|
||||
if time.time() - start > 15:
|
||||
raise Exception("Timeout waiting for smart reconfiguration")
|
||||
new_two = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-two', 0)
|
||||
new_two = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-two', EMPTY_LAYOUT_STATE)
|
||||
if old_two < new_two:
|
||||
break
|
||||
else:
|
||||
|
|
@ -8822,8 +8825,8 @@ class TestSchedulerSmartReconfiguration(ZuulTestCase):
|
|||
|
||||
# Ensure that tenant-one has not been reconfigured
|
||||
self.waitUntilSettled()
|
||||
new_one = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new_one = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
self.assertEqual(old_one, new_one)
|
||||
|
||||
self.executor_server.hold_jobs_in_build = False
|
||||
|
|
@ -8843,9 +8846,10 @@ class TestSchedulerSmartReconfiguration(ZuulTestCase):
|
|||
self.assertEqual(expected_tenants,
|
||||
self.scheds.first.sched.abide.tenants.keys())
|
||||
|
||||
self.assertIsNotNone(self.scheds.first.sched.tenant_last_reconfigured
|
||||
.get('tenant-four'),
|
||||
'Tenant tenant-four should exist now.')
|
||||
self.assertIsNotNone(
|
||||
self.scheds.first.sched.tenant_layout_state.get('tenant-four'),
|
||||
'Tenant tenant-four should exist now.'
|
||||
)
|
||||
|
||||
# Test that the new tenant-four actually works
|
||||
D = self.fake_gerrit.addFakeChange('org/project4', 'master', 'D')
|
||||
|
|
@ -8871,8 +8875,8 @@ class TestSchedulerSmartReconfiguration(ZuulTestCase):
|
|||
class TestReconfigureBranch(ZuulTestCase):
|
||||
|
||||
def _setupTenantReconfigureTime(self):
|
||||
self.old = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
self.old = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
|
||||
def _createBranch(self):
|
||||
self.create_branch('org/project1', 'stable')
|
||||
|
|
@ -8889,8 +8893,8 @@ class TestReconfigureBranch(ZuulTestCase):
|
|||
self.waitUntilSettled()
|
||||
|
||||
def _expectReconfigure(self, doReconfigure):
|
||||
new = self.scheds.first.sched.tenant_last_reconfigured\
|
||||
.get('tenant-one', 0)
|
||||
new = self.scheds.first.sched.tenant_layout_state.get(
|
||||
'tenant-one', EMPTY_LAYOUT_STATE)
|
||||
if doReconfigure:
|
||||
self.assertLess(self.old, new)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -23,6 +23,7 @@ from zuul.zk import ZooKeeperClient
|
|||
from zuul.zk.config_cache import UnparsedConfigCache
|
||||
from zuul.zk.exceptions import LockException
|
||||
from zuul.zk.executor import ExecutorApi, BuildRequestEvent
|
||||
from zuul.zk.layout import LayoutStateStore, LayoutState
|
||||
from zuul.zk.locks import locked
|
||||
from zuul.zk.nodepool import ZooKeeperNodepool
|
||||
from zuul.zk.sharding import (
|
||||
|
|
@ -662,3 +663,14 @@ class TestLocks(ZooKeeperBaseTestCase):
|
|||
with locked(lock, blocking=False):
|
||||
pass
|
||||
self.assertFalse(lock.is_acquired)
|
||||
|
||||
|
||||
class TestLayoutStore(ZooKeeperBaseTestCase):
|
||||
|
||||
def test_layout_state(self):
|
||||
store = LayoutStateStore(self.zk_client)
|
||||
state = LayoutState("tenant", "hostname", 0)
|
||||
store["tenant"] = state
|
||||
self.assertEqual(state, store["tenant"])
|
||||
self.assertNotEqual(state.ltime, -1)
|
||||
self.assertNotEqual(store["tenant"].ltime, -1)
|
||||
|
|
|
|||
|
|
@ -82,6 +82,7 @@ from zuul.zk.event_queues import (
|
|||
PipelineTriggerEventQueue,
|
||||
TENANT_ROOT,
|
||||
)
|
||||
from zuul.zk.layout import LayoutState, LayoutStateStore
|
||||
from zuul.zk.nodepool import ZooKeeperNodepool
|
||||
|
||||
COMMANDS = ['full-reconfigure', 'smart-reconfigure', 'stop', 'repl', 'norepl']
|
||||
|
|
@ -191,6 +192,7 @@ class Scheduler(threading.Thread):
|
|||
|
||||
self.abide = Abide()
|
||||
self.unparsed_abide = UnparsedAbideConfig()
|
||||
self.tenant_layout_state = LayoutStateStore(self.zk_client)
|
||||
|
||||
if not testonly:
|
||||
time_dir = self._get_time_database_dir()
|
||||
|
|
@ -207,7 +209,6 @@ class Scheduler(threading.Thread):
|
|||
else:
|
||||
self.zuul_version = zuul_version.release_string
|
||||
self.last_reconfigured = None
|
||||
self.tenant_last_reconfigured = {}
|
||||
self.use_relative_priority = False
|
||||
if self.config.has_option('scheduler', 'relative_priority'):
|
||||
if self.config.getboolean('scheduler', 'relative_priority'):
|
||||
|
|
@ -1132,7 +1133,14 @@ class Scheduler(threading.Thread):
|
|||
trigger.postConfig(pipeline)
|
||||
for reporter in pipeline.actions:
|
||||
reporter.postConfig()
|
||||
self.tenant_last_reconfigured[tenant.name] = time.time()
|
||||
|
||||
layout_state = LayoutState(
|
||||
tenant_name=tenant.name,
|
||||
hostname=self.hostname,
|
||||
last_reconfigured=int(time.time()),
|
||||
)
|
||||
self.tenant_layout_state[tenant.name] = layout_state
|
||||
|
||||
if self.statsd:
|
||||
try:
|
||||
for pipeline in tenant.layout.pipelines.values():
|
||||
|
|
@ -1887,8 +1895,9 @@ class Scheduler(threading.Thread):
|
|||
'length': len(queue),
|
||||
}
|
||||
|
||||
if self.last_reconfigured:
|
||||
data['last_reconfigured'] = self.last_reconfigured * 1000
|
||||
layout_state = self.tenant_layout_state.get(tenant_name)
|
||||
if layout_state:
|
||||
data['last_reconfigured'] = layout_state.last_reconfigured * 1000
|
||||
|
||||
pipelines = []
|
||||
data['pipelines'] = pipelines
|
||||
|
|
|
|||
|
|
@ -0,0 +1,133 @@
|
|||
# Copyright 2020 BMW Group
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import json
|
||||
from collections.abc import MutableMapping
|
||||
from functools import total_ordering
|
||||
from uuid import uuid4
|
||||
|
||||
from kazoo.exceptions import NoNodeError
|
||||
|
||||
from zuul.zk import ZooKeeperBase
|
||||
|
||||
|
||||
@total_ordering
|
||||
class LayoutState:
|
||||
"""Representation of a tenant's layout state.
|
||||
|
||||
The layout state holds information about a certain version of a
|
||||
tenant's layout. It is used to coordinate reconfigurations across
|
||||
multiple schedulers by comparing a local tenant layout state
|
||||
against the current version in Zookeeper. In case it detects that
|
||||
a local layout state is outdated, this scheduler is not allowed to
|
||||
process this tenant (events, pipelines, ...) until the layout is
|
||||
updated.
|
||||
|
||||
The important information of the layout state is the logical
|
||||
timestamp (ltime) that is used to detect if the layout on a
|
||||
scheduler needs to be updated. The ltime is the last modified
|
||||
transaction ID (mzxid) of the corresponding Znode in Zookeeper.
|
||||
|
||||
The hostname of the scheduler creating the new layout state and the
|
||||
timestamp of the last reconfiguration are only informational and
|
||||
may aid in debugging.
|
||||
"""
|
||||
|
||||
def __init__(self, tenant_name, hostname, last_reconfigured, uuid=None,
|
||||
ltime=-1):
|
||||
self.uuid = uuid or uuid4().hex
|
||||
self.ltime = ltime
|
||||
self.tenant_name = tenant_name
|
||||
self.hostname = hostname
|
||||
self.last_reconfigured = last_reconfigured
|
||||
|
||||
def toDict(self):
|
||||
return {
|
||||
"tenant_name": self.tenant_name,
|
||||
"hostname": self.hostname,
|
||||
"last_reconfigured": self.last_reconfigured,
|
||||
"uuid": self.uuid,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def fromDict(cls, data):
|
||||
return cls(
|
||||
data["tenant_name"],
|
||||
data["hostname"],
|
||||
data["last_reconfigured"],
|
||||
data.get("uuid"),
|
||||
data.get("ltime", -1),
|
||||
)
|
||||
|
||||
def __eq__(self, other):
|
||||
if not isinstance(other, LayoutState):
|
||||
return False
|
||||
return self.uuid == other.uuid
|
||||
|
||||
def __gt__(self, other):
|
||||
if not isinstance(other, LayoutState):
|
||||
return False
|
||||
return self.ltime > other.ltime
|
||||
|
||||
def __repr__(self):
|
||||
return (
|
||||
f"<{self.__class__.__name__} {self.tenant_name}: "
|
||||
f"ltime={self.ltime}, "
|
||||
f"hostname={self.hostname}, "
|
||||
f"last_reconfigured={self.last_reconfigured}>"
|
||||
)
|
||||
|
||||
|
||||
class LayoutStateStore(ZooKeeperBase, MutableMapping):
|
||||
|
||||
layout_root = "/zuul/layout"
|
||||
|
||||
def __getitem__(self, tenant_name):
|
||||
try:
|
||||
data, zstat = self.kazoo_client.get(
|
||||
f"{self.layout_root}/{tenant_name}")
|
||||
except NoNodeError:
|
||||
raise KeyError(tenant_name)
|
||||
|
||||
return LayoutState.fromDict({
|
||||
"ltime": zstat.last_modified_transaction_id,
|
||||
**json.loads(data)
|
||||
})
|
||||
|
||||
def __setitem__(self, tenant_name, state):
|
||||
path = f"{self.layout_root}/{tenant_name}"
|
||||
self.kazoo_client.ensure_path(path)
|
||||
data = json.dumps(state.toDict()).encode("utf-8")
|
||||
zstat = self.kazoo_client.set(path, data)
|
||||
# Set correct ltime of the layout in Zookeeper
|
||||
state.ltime = zstat.last_modified_transaction_id
|
||||
|
||||
def __delitem__(self, tenant_name):
|
||||
try:
|
||||
self.kazoo_client.delete(f"{self.layout_root}/{tenant_name}")
|
||||
except NoNodeError:
|
||||
raise KeyError(tenant_name)
|
||||
|
||||
def __iter__(self):
|
||||
try:
|
||||
tenant_names = self.kazoo_client.get_children(self.layout_root)
|
||||
except NoNodeError:
|
||||
return
|
||||
yield from tenant_names
|
||||
|
||||
def __len__(self):
|
||||
zstat = self.kazoo_client.exists(self.layout_root)
|
||||
if zstat is None:
|
||||
return 0
|
||||
return zstat.children_count
|
||||
Loading…
Reference in New Issue