cb3942da81
Traditionally a sharded zkobject did not store a zstat because it isn't represented by a single object. But some users or subclasses do use a zstat of a sharded zkobject. They rely on the fact that we always truncate the contents of the object before writing, and truncating means removing the base znode. That means that each time we write, we will create a new base znode, and that is sufficient to act as a zstat for the whole object. This change normalizes this and implements it in the ShardedZKObject class, and removes some now duplicated code. Change-Id: I4b973dcfb1175e882f7e3c33d61867424dd3f654
246 lines
8.3 KiB
Python
246 lines
8.3 KiB
Python
# Copyright 2021 BMW Group
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import contextlib
|
|
import json
|
|
import logging
|
|
import zlib
|
|
|
|
from collections.abc import MutableMapping
|
|
from urllib.parse import quote_plus, unquote_plus
|
|
|
|
from kazoo.exceptions import NoNodeError
|
|
from kazoo.recipe import lock
|
|
|
|
from zuul import model
|
|
from zuul.zk import sharding, ZooKeeperSimpleBase
|
|
|
|
CONFIG_ROOT = "/zuul/config"
|
|
|
|
|
|
def _safe_path(root_path, *keys):
|
|
return "/".join((root_path, *(quote_plus(k) for k in keys)))
|
|
|
|
|
|
class FilesCache(ZooKeeperSimpleBase, MutableMapping):
|
|
"""Cache of raw config files in Zookeeper for a project-branch.
|
|
|
|
Data will be stored in Zookeeper using the following path:
|
|
/zuul/config/<project>/<branch>/<filename>
|
|
|
|
"""
|
|
log = logging.getLogger("zuul.zk.config_cache.FilesCache")
|
|
|
|
def __init__(self, client, root_path):
|
|
super().__init__(client)
|
|
self.root_path = root_path
|
|
self.files_path = f"{root_path}/files"
|
|
|
|
def setValidFor(self, extra_config_files, extra_config_dirs, ltime):
|
|
"""Set the cache valid for the given extra config files/dirs."""
|
|
data = {
|
|
"extra_files_searched": list(extra_config_files),
|
|
"extra_dirs_searched": list(extra_config_dirs),
|
|
"ltime": ltime,
|
|
}
|
|
payload = json.dumps(data, sort_keys=True).encode("utf8")
|
|
try:
|
|
self.kazoo_client.set(self.root_path, payload)
|
|
except NoNodeError:
|
|
self.kazoo_client.create(self.root_path, payload, makepath=True)
|
|
|
|
def isValidFor(self, tpc, min_ltime):
|
|
"""Check if the cache is valid.
|
|
|
|
Check if the cache is valid for the given tenant project config
|
|
(tpc) and that it is up-to-date, relative to the give logical
|
|
timestamp.
|
|
|
|
"""
|
|
try:
|
|
data, _ = self.kazoo_client.get(self.root_path)
|
|
except NoNodeError:
|
|
return False
|
|
|
|
try:
|
|
content = json.loads(data)
|
|
extra_files_searched = set(content["extra_files_searched"])
|
|
extra_dirs_searched = set(content["extra_dirs_searched"])
|
|
ltime = content["ltime"]
|
|
except Exception:
|
|
return False
|
|
|
|
if ltime < min_ltime:
|
|
# Cache is outdated
|
|
return False
|
|
|
|
return (set(tpc.extra_config_files) <= extra_files_searched
|
|
and set(tpc.extra_config_dirs) <= extra_dirs_searched)
|
|
|
|
@property
|
|
def ltime(self):
|
|
try:
|
|
data, _ = self.kazoo_client.get(self.root_path)
|
|
content = json.loads(data)
|
|
return content["ltime"]
|
|
except Exception:
|
|
return -1
|
|
|
|
def _key_path(self, key):
|
|
return _safe_path(self.files_path, key)
|
|
|
|
def __getitem__(self, key):
|
|
try:
|
|
with sharding.BufferedShardReader(
|
|
self.kazoo_client, self._key_path(key)
|
|
) as stream:
|
|
return zlib.decompress(stream.read()).decode("utf8")
|
|
except NoNodeError:
|
|
raise KeyError(key)
|
|
|
|
def __setitem__(self, key, value):
|
|
path = self._key_path(key)
|
|
with sharding.BufferedShardWriter(self.kazoo_client, path) as stream:
|
|
stream.truncate(0)
|
|
stream.write(zlib.compress(value.encode("utf8")))
|
|
stream.flush()
|
|
|
|
def __delitem__(self, key):
|
|
try:
|
|
self.kazoo_client.delete(self._key_path(key), recursive=True)
|
|
except NoNodeError:
|
|
raise KeyError(key)
|
|
|
|
def __iter__(self):
|
|
try:
|
|
children = self.kazoo_client.get_children(self.files_path)
|
|
except NoNodeError:
|
|
children = []
|
|
yield from sorted(unquote_plus(c) for c in children)
|
|
|
|
def __len__(self):
|
|
try:
|
|
children = self.kazoo_client.get_children(self.files_path)
|
|
except NoNodeError:
|
|
children = []
|
|
return len(children)
|
|
|
|
def clear(self):
|
|
with contextlib.suppress(NoNodeError):
|
|
self.kazoo_client.delete(self.root_path, recursive=True)
|
|
|
|
|
|
class UnparsedConfigCache(ZooKeeperSimpleBase):
|
|
"""Zookeeper cache for unparsed config files."""
|
|
|
|
log = logging.getLogger("zuul.zk.config_cache.UnparsedConfigCache")
|
|
|
|
def __init__(self, client, config_root=CONFIG_ROOT):
|
|
super().__init__(client)
|
|
self.cache_path = f"{config_root}/cache"
|
|
self.lock_path = f"{config_root}/lock"
|
|
|
|
def readLock(self, project_cname):
|
|
return lock.ReadLock(
|
|
self.kazoo_client, _safe_path(self.lock_path, project_cname))
|
|
|
|
def writeLock(self, project_cname):
|
|
return lock.WriteLock(
|
|
self.kazoo_client, _safe_path(self.lock_path, project_cname))
|
|
|
|
def getFilesCache(self, project_cname, branch_name):
|
|
path = _safe_path(self.cache_path, project_cname, branch_name)
|
|
return FilesCache(self.client, path)
|
|
|
|
def listCachedProjects(self):
|
|
try:
|
|
children = self.kazoo_client.get_children(self.cache_path)
|
|
except NoNodeError:
|
|
children = []
|
|
yield from sorted(unquote_plus(c) for c in children)
|
|
|
|
def clearCache(self, project_cname, branch_name=None):
|
|
if branch_name is None:
|
|
path = _safe_path(self.cache_path, project_cname)
|
|
else:
|
|
path = _safe_path(self.cache_path, project_cname, branch_name)
|
|
with contextlib.suppress(NoNodeError):
|
|
self.kazoo_client.delete(path, recursive=True)
|
|
|
|
|
|
class SystemConfigCache(ZooKeeperSimpleBase):
|
|
"""Zookeeper cache for Zuul system configuration.
|
|
|
|
The system configuration consists of the unparsed abide config and
|
|
the runtime related settings from the Zuul config file.
|
|
"""
|
|
|
|
SYSTEM_ROOT = "/zuul/system"
|
|
log = logging.getLogger("zuul.zk.SystemConfigCache")
|
|
|
|
def __init__(self, client, callback):
|
|
super().__init__(client)
|
|
self.conf_path = f"{self.SYSTEM_ROOT}/conf"
|
|
self.lock_path = f"{self.SYSTEM_ROOT}/conf-lock"
|
|
self._callback = callback
|
|
self.kazoo_client.DataWatch(self.conf_path, self._configChanged)
|
|
|
|
def _configChanged(self, data, stat, event):
|
|
self._callback()
|
|
|
|
@property
|
|
def ltime(self):
|
|
with lock.ReadLock(self.kazoo_client, self.lock_path):
|
|
zstat = self.kazoo_client.exists(self.conf_path)
|
|
return -1 if zstat is None else zstat.last_modified_transaction_id
|
|
|
|
@property
|
|
def is_valid(self):
|
|
return self.ltime > -1
|
|
|
|
def get(self):
|
|
"""Get the system configuration from Zookeeper.
|
|
|
|
:returns: A tuple (unparsed abide config, system attributes)
|
|
"""
|
|
with lock.ReadLock(self.kazoo_client, self.lock_path):
|
|
try:
|
|
with sharding.BufferedShardReader(
|
|
self.kazoo_client, self.conf_path
|
|
) as stream:
|
|
data = json.loads(zlib.decompress(stream.read()))
|
|
zstat = stream.zstat
|
|
except Exception:
|
|
raise RuntimeError("No valid system config")
|
|
return (model.UnparsedAbideConfig.fromDict(
|
|
data["unparsed_abide"],
|
|
ltime=zstat.last_modified_transaction_id
|
|
), model.SystemAttributes.fromDict(data["system_attributes"]))
|
|
|
|
def set(self, unparsed_abide, system_attributes):
|
|
with lock.WriteLock(self.kazoo_client, self.lock_path):
|
|
data = {
|
|
"unparsed_abide": unparsed_abide.toDict(),
|
|
"system_attributes": system_attributes.toDict(),
|
|
}
|
|
with sharding.BufferedShardWriter(
|
|
self.kazoo_client, self.conf_path
|
|
) as stream:
|
|
stream.truncate(0)
|
|
stream.write(zlib.compress(
|
|
json.dumps(data, sort_keys=True).encode("utf8")))
|
|
stream.flush()
|
|
zstat = stream.zstat
|
|
unparsed_abide.ltime = zstat.last_modified_transaction_id
|