Browse Source
Prepare Zookeeper for scale-out scheduler
This change is a common root for other Zookeeper related changed regarding scale-out-scheduler. Zookeeper becoming a central component requires to increase "maxClientCnxns". Since the ZooKeeper class is expected to grow significantly (ZooKeeper is becoming a central part of Zuul) a split of the ZooKeeper class (zk.py) into zk module is done here to avoid the current god-class. Also the zookeeper log is copied to the "zuul_output_dir". Change-Id: I714c06052b5e17269a6964892ad53b48cf65db19 Story: 2007192changes/69/717269/64
Jan Kubovy
4 months ago
committed by
Felix Edel
Felix Edel
16 changed files with 683 additions and 600 deletions
Split View
Diff Options
-
+2 -0.zuul.yaml
-
+1 -1doc/source/examples/zoo.cfg
-
+5 -0playbooks/common/post-system-logs.yaml
-
+8 -7tests/base.py
-
+5 -5tests/nodepool/test_nodepool_integration.py
-
+10 -8tests/unit/test_nodepool.py
-
+28 -20tests/unit/test_scheduler.py
-
+20 -17tests/unit/test_zk.py
-
+2 -0tools/test-setup.sh
-
+7 -7zuul/cmd/scheduler.py
-
+20 -19zuul/nodepool.py
-
+16 -13zuul/scheduler.py
-
+13 -11zuul/web/__init__.py
-
+153 -0zuul/zk/__init__.py
-
+27 -0zuul/zk/exceptions.py
-
+366 -492zuul/zk/nodepool.py
+ 2
- 0
.zuul.yaml
View File
+ 1
- 1
doc/source/examples/zoo.cfg
View File
+ 5
- 0
playbooks/common/post-system-logs.yaml
View File
| @ -0,0 +1,5 @@ | |||
| - hosts: all | |||
| tasks: | |||
| - name: Collect zookeeper logs | |||
| shell: "cp /var/log/zookeeper/zookeeper.log {{ zuul_output_dir }}/logs/zookeeper.log" | |||
+ 8
- 7
tests/base.py
View File
+ 5
- 5
tests/nodepool/test_nodepool_integration.py
View File
+ 10
- 8
tests/unit/test_nodepool.py
View File
+ 28
- 20
tests/unit/test_scheduler.py
View File
+ 20
- 17
tests/unit/test_zk.py
View File
+ 2
- 0
tools/test-setup.sh
View File
+ 7
- 7
zuul/cmd/scheduler.py
View File
+ 20
- 19
zuul/nodepool.py
View File
+ 16
- 13
zuul/scheduler.py
View File
+ 13
- 11
zuul/web/__init__.py
View File
+ 153
- 0
zuul/zk/__init__.py
View File
| @ -0,0 +1,153 @@ | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
| # not use this file except in compliance with the License. You may obtain | |||
| # a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
| # License for the specific language governing permissions and limitations | |||
| # under the License. | |||
| import logging | |||
| import time | |||
| from abc import ABCMeta | |||
| from typing import Optional, List, Callable | |||
| from kazoo.client import KazooClient | |||
| from kazoo.handlers.threading import KazooTimeoutError | |||
| from kazoo.protocol.states import KazooState | |||
| from zuul.zk.exceptions import NoClientException | |||
| class ZooKeeperClient(object): | |||
| log = logging.getLogger("zuul.zk.base.ZooKeeperClient") | |||
| # Log zookeeper retry every 10 seconds | |||
| retry_log_rate = 10 | |||
| def __init__(self): | |||
| """ | |||
| Initialize the ZooKeeper base client object. | |||
| """ | |||
| self.client: Optional[KazooClient] = None | |||
| self._last_retry_log: int = 0 | |||
| self.on_connect_listeners: List[Callable[[], None]] = [] | |||
| self.on_disconnect_listeners: List[Callable[[], None]] = [] | |||
| def _connectionListener(self, state): | |||
| """ | |||
| Listener method for Kazoo connection state changes. | |||
| .. warning:: This method must not block. | |||
| """ | |||
| if state == KazooState.LOST: | |||
| self.log.debug("ZooKeeper connection: LOST") | |||
| elif state == KazooState.SUSPENDED: | |||
| self.log.debug("ZooKeeper connection: SUSPENDED") | |||
| else: | |||
| self.log.debug("ZooKeeper connection: CONNECTED") | |||
| @property | |||
| def connected(self): | |||
| return self.client and self.client.state == KazooState.CONNECTED | |||
| @property | |||
| def suspended(self): | |||
| return self.client and self.client.state == KazooState.SUSPENDED | |||
| @property | |||
| def lost(self): | |||
| return not self.client or self.client.state == KazooState.LOST | |||
| def logConnectionRetryEvent(self): | |||
| now = time.monotonic() | |||
| if now - self._last_retry_log >= self.retry_log_rate: | |||
| self.log.warning("Retrying zookeeper connection") | |||
| self._last_retry_log = now | |||
| def connect(self, hosts: str, read_only: bool = False, | |||
| timeout: float = 10.0, tls_cert: Optional[str] = None, | |||
| tls_key: Optional[str] = None, | |||
| tls_ca: Optional[str] = None): | |||
| """ | |||
| Establish a connection with ZooKeeper cluster. | |||
| Convenience method if a pre-existing ZooKeeper connection is not | |||
| supplied to the ZooKeeper object at instantiation time. | |||
| :param str hosts: Comma-separated list of hosts to connect to (e.g. | |||
| 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183). | |||
| :param bool read_only: If True, establishes a read-only connection. | |||
| :param float timeout: The ZooKeeper session timeout, in | |||
| seconds (default: 10.0). | |||
| :param str tls_key: Path to TLS key | |||
| :param str tls_cert: Path to TLS cert | |||
| :param str tls_ca: Path to TLS CA cert | |||
| """ | |||
| if self.client is None: | |||
| args = dict(hosts=hosts, read_only=read_only, timeout=timeout) | |||
| if tls_key: | |||
| args['use_ssl'] = True | |||
| args['keyfile'] = tls_key | |||
| args['certfile'] = tls_cert | |||
| args['ca'] = tls_ca | |||
| self.client = KazooClient(**args) | |||
| self.client.add_listener(self._connectionListener) | |||
| # Manually retry initial connection attempt | |||
| while True: | |||
| try: | |||
| self.client.start(1) | |||
| break | |||
| except KazooTimeoutError: | |||
| self.logConnectionRetryEvent() | |||
| for listener in self.on_connect_listeners: | |||
| listener() | |||
| def disconnect(self): | |||
| """ | |||
| Close the ZooKeeper cluster connection. | |||
| You should call this method if you used connect() to establish a | |||
| cluster connection. | |||
| """ | |||
| for listener in self.on_disconnect_listeners: | |||
| listener() | |||
| if self.client is not None and self.client.connected: | |||
| self.client.stop() | |||
| self.client.close() | |||
| self.client = None | |||
| def resetHosts(self, hosts): | |||
| """ | |||
| Reset the ZooKeeper cluster connection host list. | |||
| :param str hosts: Comma-separated list of hosts to connect to (e.g. | |||
| 127.0.0.1:2181,127.0.0.1:2182,[::1]:2183). | |||
| """ | |||
| if self.client is not None: | |||
| self.client.set_hosts(hosts=hosts) | |||
| class ZooKeeperBase(metaclass=ABCMeta): | |||
| """Base class for components that need to interact with Zookeeper.""" | |||
| def __init__(self, client: ZooKeeperClient): | |||
| self.client = client | |||
| self.client.on_connect_listeners.append(self._onConnect) | |||
| self.client.on_disconnect_listeners.append(self._onDisconnect) | |||
| @property | |||
| def kazoo_client(self) -> KazooClient: | |||
| if not self.client.client: | |||
| raise NoClientException() | |||
| return self.client.client | |||
| def _onConnect(self): | |||
| pass | |||
| def _onDisconnect(self): | |||
| pass | |||
+ 27
- 0
zuul/zk/exceptions.py
View File
| @ -0,0 +1,27 @@ | |||
| # Licensed under the Apache License, Version 2.0 (the "License"); you may | |||
| # not use this file except in compliance with the License. You may obtain | |||
| # a copy of the License at | |||
| # | |||
| # http://www.apache.org/licenses/LICENSE-2.0 | |||
| # | |||
| # Unless required by applicable law or agreed to in writing, software | |||
| # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT | |||
| # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the | |||
| # License for the specific language governing permissions and limitations | |||
| # under the License. | |||
| from kazoo.exceptions import KazooException | |||
| class ZuulZooKeeperException(KazooException): | |||
| """Base exception class for all custom ZK exceptions""" | |||
| pass | |||
| class LockException(ZuulZooKeeperException): | |||
| pass | |||
| class NoClientException(ZuulZooKeeperException): | |||
| def __init__(self): | |||
| super().__init__("No zookeeper client!") | |||