Browse Source
Prepare Zookeeper for scale-out scheduler
Prepare Zookeeper for scale-out scheduler
This change is a common root for other Zookeeper related changed regarding scale-out-scheduler. Zookeeper becoming a central component requires to increase "maxClientCnxns". Since the ZooKeeper class is expected to grow significantly (ZooKeeper is becoming a central part of Zuul) a split of the ZooKeeper class (zk.py) into zk module is done here to avoid the current god-class. Also the zookeeper log is copied to the "zuul_output_dir". Change-Id: I714c06052b5e17269a6964892ad53b48cf65db19 Story: 2007192changes/69/717269/64
Jan Kubovy
9 months ago
committed by
Felix Edel
Felix Edel
16 changed files with 683 additions and 600 deletions
-
2.zuul.yaml
-
2doc/source/examples/zoo.cfg
-
5playbooks/common/post-system-logs.yaml
-
15tests/base.py
-
10tests/nodepool/test_nodepool_integration.py
-
18tests/unit/test_nodepool.py
-
48tests/unit/test_scheduler.py
-
37tests/unit/test_zk.py
-
2tools/test-setup.sh
-
14zuul/cmd/scheduler.py
-
39zuul/nodepool.py
-
29zuul/scheduler.py
-
24zuul/web/__init__.py
-
153zuul/zk/__init__.py
-
27zuul/zk/exceptions.py
-
858zuul/zk/nodepool.py
@ -0,0 +1,5 @@ |
|||
- hosts: all |
|||
tasks: |
|||
|
|||
- name: Collect zookeeper logs |
|||
shell: "cp /var/log/zookeeper/zookeeper.log {{ zuul_output_dir }}/logs/zookeeper.log" |
|||
@ -0,0 +1,153 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
import logging |
|||
import time |
|||
from abc import ABCMeta |
|||
from typing import Optional, List, Callable |
|||
|
|||
from kazoo.client import KazooClient |
|||
from kazoo.handlers.threading import KazooTimeoutError |
|||
from kazoo.protocol.states import KazooState |
|||
|
|||
from zuul.zk.exceptions import NoClientException |
|||
|
|||
|
|||
class ZooKeeperClient(object): |
|||
log = logging.getLogger("zuul.zk.base.ZooKeeperClient") |
|||
|
|||
# Log zookeeper retry every 10 seconds |
|||
retry_log_rate = 10 |
|||
|
|||
def __init__(self): |
|||
""" |
|||
Initialize the ZooKeeper base client object. |
|||
""" |
|||
self.client: Optional[KazooClient] = None |
|||
self._last_retry_log: int = 0 |
|||
self.on_connect_listeners: List[Callable[[], None]] = [] |
|||
self.on_disconnect_listeners: List[Callable[[], None]] = [] |
|||
|
|||
def _connectionListener(self, state): |
|||
""" |
|||
Listener method for Kazoo connection state changes. |
|||
|
|||
.. warning:: This method must not block. |
|||
""" |
|||
if state == KazooState.LOST: |
|||
self.log.debug("ZooKeeper connection: LOST") |
|||
elif state == KazooState.SUSPENDED: |
|||
self.log.debug("ZooKeeper connection: SUSPENDED") |
|||
else: |
|||
self.log.debug("ZooKeeper connection: CONNECTED") |
|||
|
|||
@property |
|||
def connected(self): |
|||
return self.client and self.client.state == KazooState.CONNECTED |
|||
|
|||
@property |
|||
def suspended(self): |
|||
return self.client and self.client.state == KazooState.SUSPENDED |
|||
|
|||
@property |
|||
def lost(self): |
|||
return not self.client or self.client.state == KazooState.LOST |
|||
|
|||
def logConnectionRetryEvent(self): |
|||
now = time.monotonic() |
|||
if now - self._last_retry_log >= self.retry_log_rate: |
|||
self.log.warning("Retrying zookeeper connection") |
|||
self._last_retry_log = now |
|||
|
|||
def connect(self, hosts: str, read_only: bool = False, |
|||
timeout: float = 10.0, tls_cert: Optional[str] = None, |
|||
tls_key: Optional[str] = None, |
|||
tls_ca: Optional[str] = None): |
|||
""" |
|||
Establish a connection with ZooKeeper cluster. |
|||
|
|||
Convenience method if a pre-existing ZooKeeper connection is not |
|||
supplied to the ZooKeeper object at instantiation time. |
|||
|
|||
:param str hosts: Comma-separated list of hosts to connect to (e.g. |
|||
127.0.0.1:2181,127.0.0.1:2182,[::1]:2183). |
|||
:param bool read_only: If True, establishes a read-only connection. |
|||
:param float timeout: The ZooKeeper session timeout, in |
|||
seconds (default: 10.0). |
|||
:param str tls_key: Path to TLS key |
|||
:param str tls_cert: Path to TLS cert |
|||
:param str tls_ca: Path to TLS CA cert |
|||
""" |
|||
if self.client is None: |
|||
args = dict(hosts=hosts, read_only=read_only, timeout=timeout) |
|||
if tls_key: |
|||
args['use_ssl'] = True |
|||
args['keyfile'] = tls_key |
|||
args['certfile'] = tls_cert |
|||
args['ca'] = tls_ca |
|||
self.client = KazooClient(**args) |
|||
self.client.add_listener(self._connectionListener) |
|||
# Manually retry initial connection attempt |
|||
while True: |
|||
try: |
|||
self.client.start(1) |
|||
break |
|||
except KazooTimeoutError: |
|||
self.logConnectionRetryEvent() |
|||
|
|||
for listener in self.on_connect_listeners: |
|||
listener() |
|||
|
|||
def disconnect(self): |
|||
""" |
|||
Close the ZooKeeper cluster connection. |
|||
|
|||
You should call this method if you used connect() to establish a |
|||
cluster connection. |
|||
""" |
|||
for listener in self.on_disconnect_listeners: |
|||
listener() |
|||
|
|||
if self.client is not None and self.client.connected: |
|||
self.client.stop() |
|||
self.client.close() |
|||
self.client = None |
|||
|
|||
def resetHosts(self, hosts): |
|||
""" |
|||
Reset the ZooKeeper cluster connection host list. |
|||
|
|||
:param str hosts: Comma-separated list of hosts to connect to (e.g. |
|||
127.0.0.1:2181,127.0.0.1:2182,[::1]:2183). |
|||
""" |
|||
if self.client is not None: |
|||
self.client.set_hosts(hosts=hosts) |
|||
|
|||
|
|||
class ZooKeeperBase(metaclass=ABCMeta): |
|||
"""Base class for components that need to interact with Zookeeper.""" |
|||
|
|||
def __init__(self, client: ZooKeeperClient): |
|||
self.client = client |
|||
self.client.on_connect_listeners.append(self._onConnect) |
|||
self.client.on_disconnect_listeners.append(self._onDisconnect) |
|||
|
|||
@property |
|||
def kazoo_client(self) -> KazooClient: |
|||
if not self.client.client: |
|||
raise NoClientException() |
|||
return self.client.client |
|||
|
|||
def _onConnect(self): |
|||
pass |
|||
|
|||
def _onDisconnect(self): |
|||
pass |
|||
@ -0,0 +1,27 @@ |
|||
# Licensed under the Apache License, Version 2.0 (the "License"); you may |
|||
# not use this file except in compliance with the License. You may obtain |
|||
# a copy of the License at |
|||
# |
|||
# http://www.apache.org/licenses/LICENSE-2.0 |
|||
# |
|||
# Unless required by applicable law or agreed to in writing, software |
|||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT |
|||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the |
|||
# License for the specific language governing permissions and limitations |
|||
# under the License. |
|||
from kazoo.exceptions import KazooException |
|||
|
|||
|
|||
class ZuulZooKeeperException(KazooException): |
|||
"""Base exception class for all custom ZK exceptions""" |
|||
pass |
|||
|
|||
|
|||
class LockException(ZuulZooKeeperException): |
|||
pass |
|||
|
|||
|
|||
class NoClientException(ZuulZooKeeperException): |
|||
|
|||
def __init__(self): |
|||
super().__init__("No zookeeper client!") |
|||
858
zuul/zk/nodepool.py
File diff suppressed because it is too large
View File
File diff suppressed because it is too large
View File
Write
Preview
Loading…
Cancel
Save
Reference in new issue