fa274fcf25
Since we require deleting the state (except config cache) for the circular dependency upgrade, we can now remove any zookeeper backwards compatability handling not in the config cache. This change does so. It leaves upgrade handling for two attributes in the branch cache (even though they are old) because it is plausible that even systems that have been upgrading regularly may have non-upgraded data in the branch cache for infrequently used branches, and the cost for retaining the upgrade handling is small. Change-Id: I881578159b06c8c7b38a9fa0980aa0626dddad31
219 lines
7.5 KiB
Python
219 lines
7.5 KiB
Python
# Copyright 2020 BMW Group
|
|
# Copyright 2022 Acme Gating, LLC
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import json
|
|
from collections.abc import MutableMapping
|
|
from contextlib import suppress
|
|
from functools import total_ordering
|
|
import logging
|
|
import time
|
|
|
|
from kazoo.exceptions import NoNodeError
|
|
|
|
from zuul.zk import sharding, ZooKeeperBase
|
|
|
|
|
|
@total_ordering
|
|
class LayoutState:
|
|
"""Representation of a tenant's layout state.
|
|
|
|
The layout state holds information about a certain version of a
|
|
tenant's layout. It is used to coordinate reconfigurations across
|
|
multiple schedulers by comparing a local tenant layout state
|
|
against the current version in Zookeeper. In case it detects that
|
|
a local layout state is outdated, this scheduler is not allowed to
|
|
process this tenant (events, pipelines, ...) until the layout is
|
|
updated.
|
|
|
|
The important information of the layout state is the logical
|
|
timestamp (ltime) that is used to detect if the layout on a
|
|
scheduler needs to be updated. The ltime is the last modified
|
|
transaction ID (mzxid) of the corresponding Znode in Zookeeper.
|
|
|
|
The hostname of the scheduler creating the new layout state and the
|
|
timestamp of the last reconfiguration are only informational and
|
|
may aid in debugging.
|
|
"""
|
|
|
|
def __init__(self, tenant_name, hostname, last_reconfigured, uuid,
|
|
branch_cache_min_ltimes, last_reconfigure_event_ltime,
|
|
ltime=-1):
|
|
self.uuid = uuid
|
|
self.ltime = ltime
|
|
self.tenant_name = tenant_name
|
|
self.hostname = hostname
|
|
self.last_reconfigured = last_reconfigured
|
|
self.last_reconfigure_event_ltime =\
|
|
last_reconfigure_event_ltime
|
|
self.branch_cache_min_ltimes = branch_cache_min_ltimes
|
|
|
|
def toDict(self):
|
|
return {
|
|
"tenant_name": self.tenant_name,
|
|
"hostname": self.hostname,
|
|
"last_reconfigured": self.last_reconfigured,
|
|
"last_reconfigure_event_ltime":
|
|
self.last_reconfigure_event_ltime,
|
|
"uuid": self.uuid,
|
|
"branch_cache_min_ltimes": self.branch_cache_min_ltimes,
|
|
}
|
|
|
|
@classmethod
|
|
def fromDict(cls, data):
|
|
return cls(
|
|
data["tenant_name"],
|
|
data["hostname"],
|
|
data["last_reconfigured"],
|
|
data.get("uuid"),
|
|
data.get("branch_cache_min_ltimes"),
|
|
data.get("last_reconfigure_event_ltime", -1),
|
|
data.get("ltime", -1),
|
|
)
|
|
|
|
def __eq__(self, other):
|
|
if not isinstance(other, LayoutState):
|
|
return False
|
|
return self.uuid == other.uuid
|
|
|
|
def __gt__(self, other):
|
|
if not isinstance(other, LayoutState):
|
|
return False
|
|
return self.ltime > other.ltime
|
|
|
|
def __repr__(self):
|
|
return (
|
|
f"<{self.__class__.__name__} {self.tenant_name}: "
|
|
f"ltime={self.ltime}, "
|
|
f"hostname={self.hostname}, "
|
|
f"last_reconfigured={self.last_reconfigured}>"
|
|
)
|
|
|
|
|
|
class LayoutStateStore(ZooKeeperBase, MutableMapping):
|
|
log = logging.getLogger("zuul.LayoutStore")
|
|
|
|
layout_root = "/zuul/layout"
|
|
layout_data_root = "/zuul/layout-data"
|
|
|
|
def __init__(self, client, callback):
|
|
super().__init__(client)
|
|
self._watched_tenants = set()
|
|
self._callback = callback
|
|
self.kazoo_client.ensure_path(self.layout_root)
|
|
self.kazoo_client.ChildrenWatch(self.layout_root, self._layoutCallback)
|
|
|
|
def _layoutCallback(self, tenant_list, event=None):
|
|
new_tenants = set(tenant_list) - self._watched_tenants
|
|
for tenant_name in new_tenants:
|
|
self.kazoo_client.DataWatch(f"{self.layout_root}/{tenant_name}",
|
|
self._callbackWrapper)
|
|
|
|
def _callbackWrapper(self, data, stat, event):
|
|
self._callback()
|
|
|
|
def __getitem__(self, tenant_name):
|
|
try:
|
|
data, zstat = self.kazoo_client.get(
|
|
f"{self.layout_root}/{tenant_name}")
|
|
except NoNodeError:
|
|
raise KeyError(tenant_name)
|
|
|
|
if not data:
|
|
raise KeyError(tenant_name)
|
|
|
|
return LayoutState.fromDict({
|
|
"ltime": zstat.last_modified_transaction_id,
|
|
**json.loads(data)
|
|
})
|
|
|
|
def __setitem__(self, tenant_name, state):
|
|
path = f"{self.layout_root}/{tenant_name}"
|
|
data = json.dumps(state.toDict(), sort_keys=True).encode("utf-8")
|
|
if self.kazoo_client.exists(path):
|
|
zstat = self.kazoo_client.set(path, data)
|
|
else:
|
|
_, zstat = self.kazoo_client.create(path, data, include_data=True)
|
|
# Set correct ltime of the layout in Zookeeper
|
|
state.ltime = zstat.last_modified_transaction_id
|
|
|
|
def __delitem__(self, tenant_name):
|
|
try:
|
|
self.kazoo_client.delete(f"{self.layout_root}/{tenant_name}")
|
|
except NoNodeError:
|
|
raise KeyError(tenant_name)
|
|
|
|
def __iter__(self):
|
|
try:
|
|
tenant_names = self.kazoo_client.get_children(self.layout_root)
|
|
except NoNodeError:
|
|
return
|
|
yield from tenant_names
|
|
|
|
def __len__(self):
|
|
zstat = self.kazoo_client.exists(self.layout_root)
|
|
if zstat is None:
|
|
return 0
|
|
return zstat.children_count
|
|
|
|
def getMinLtimes(self, layout_state):
|
|
try:
|
|
path = f"{self.layout_data_root}/{layout_state.uuid}"
|
|
with sharding.BufferedShardReader(
|
|
self.kazoo_client, path) as stream:
|
|
data = stream.read()
|
|
except NoNodeError:
|
|
return None
|
|
|
|
try:
|
|
return json.loads(data)['min_ltimes']
|
|
except Exception:
|
|
return None
|
|
|
|
def setMinLtimes(self, layout_state, min_ltimes):
|
|
data = dict(min_ltimes=min_ltimes)
|
|
encoded_data = json.dumps(data).encode("utf-8")
|
|
|
|
path = f"{self.layout_data_root}/{layout_state.uuid}"
|
|
with sharding.BufferedShardWriter(
|
|
self.kazoo_client, path) as stream:
|
|
stream.write(encoded_data)
|
|
|
|
def cleanup(self, delay=300):
|
|
self.log.debug("Starting layout data cleanup")
|
|
known_layouts = set()
|
|
for tenant in self.kazoo_client.get_children(
|
|
self.layout_root):
|
|
state = self.get(tenant)
|
|
if state:
|
|
known_layouts.add(state.uuid)
|
|
|
|
for layout_uuid in self.kazoo_client.get_children(
|
|
self.layout_data_root):
|
|
if layout_uuid in known_layouts:
|
|
continue
|
|
|
|
path = f"{self.layout_data_root}/{layout_uuid}"
|
|
zstat = self.kazoo_client.exists(path)
|
|
if zstat is None:
|
|
continue
|
|
|
|
now = time.time()
|
|
if now - zstat.created >= delay:
|
|
self.log.debug("Deleting unused layout data for %s",
|
|
layout_uuid)
|
|
with suppress(NoNodeError):
|
|
self.kazoo_client.delete(path, recursive=True)
|
|
self.log.debug("Finished layout data cleanup")
|