From 74a9c9de9b52b75b7286b7e4873667517b390446 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 24 Feb 2021 16:57:49 -0800 Subject: [PATCH] Use ZooKeeper TLS in tests This mirrors the configuration in Nodepool for using TLS-enabled ZooKeeper in tests. We use the ensure-zookeeper role in order to get a newer ZooKeeper than is supplied in bionic. Change-Id: I14413fccbc9a6a7a75b6233d667e2a1d2856d894 --- .gitignore | 1 + .zuul.yaml | 92 ++++++++++++++++-------- bindep.txt | 1 - playbooks/common/post-system-logs.yaml | 5 -- playbooks/zuul-tox/post-system-logs.yaml | 5 ++ playbooks/zuul-tox/pre.yaml | 4 ++ tests/base.py | 63 ++++++++++++---- tests/unit/test_nodepool.py | 13 ++-- tests/unit/test_streaming.py | 3 +- tests/unit/test_zk.py | 8 ++- tools/docker-compose.yaml | 5 +- tools/test-setup-docker.sh | 6 ++ tools/test-setup.sh | 10 +-- tools/zoo.cfg | 16 +++++ tox.ini | 8 +++ zuul/zk/__init__.py | 7 +- 16 files changed, 171 insertions(+), 76 deletions(-) delete mode 100644 playbooks/common/post-system-logs.yaml create mode 100644 playbooks/zuul-tox/post-system-logs.yaml create mode 100644 playbooks/zuul-tox/pre.yaml create mode 100644 tools/zoo.cfg diff --git a/.gitignore b/.gitignore index c5b47046e5..c2f47953b5 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ zuul/web/static/* !.keep node_modules yarn-error.log +tools/ca/ diff --git a/.zuul.yaml b/.zuul.yaml index 711cb69c3f..57c69a1a9a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -44,16 +44,70 @@ vars: zuul_ansible_version: 2.9 +- job: + name: zuul-tox + description: | + Zuul unit tests with ZooKeeper running + parent: tox + nodeset: ubuntu-bionic + pre-run: playbooks/zuul-tox/pre.yaml + post-run: playbooks/zuul-tox/post-system-logs.yaml + vars: + tox_environment: + ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem + ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem + ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem + ZUUL_TEST_ROOT: /tmp/zuul-test + YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" + test_setup_environment: + ZUUL_TEST_ROOT: /tmp/zuul-test + YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" + - job: name: zuul-tox-remote parent: tox + timeout: 2700 # 45 minutes + pre-run: playbooks/zuul-tox/pre.yaml + post-run: playbooks/zuul-tox/post-system-logs.yaml vars: tox_envlist: remote tox_environment: + ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem + ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem + ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}" ZUUL_REMOTE_KEEP: "true" +- job: + name: zuul-tox-zuul-client + parent: zuul-tox + description: | + Test that Zuul and zuul-client work together. + required-projects: + - zuul/zuul + - zuul/zuul-client + vars: + zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}" + tox_envlist: zuul_client + nodeset: ubuntu-bionic + +- job: + name: zuul-tox-py36 + parent: zuul-tox + timeout: 4800 # 80 minutes + vars: + tox_envlist: py36 + python_version: 3.6 + +- job: + name: zuul-tox-py38 + parent: zuul-tox + timeout: 4800 # 80 minutes + vars: + tox_envlist: py38 + python_version: 3.8 + - job: name: zuul-build-dashboard parent: build-javascript-deployment @@ -223,22 +277,8 @@ - tox-linters: vars: tox_install_bindep: false - - tox-py36: - nodeset: ubuntu-bionic - timeout: 4800 # 80 minutes - vars: &zuul_tox_vars - test_setup_environment: - ZUUL_TEST_ROOT: /tmp/zuul-test - YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" - tox_environment: - ZUUL_TEST_ROOT: /tmp/zuul-test - YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs" - post-run: playbooks/common/post-system-logs.yaml - - tox-py38: - timeout: 4800 # 80 minutes - nodeset: ubuntu-bionic - vars: *zuul_tox_vars - post-run: playbooks/common/post-system-logs.yaml + - zuul-tox-py36 + - zuul-tox-py38 - zuul-build-dashboard-openstack-whitelabel - zuul-build-dashboard-software-factory - zuul-build-dashboard-opendev @@ -253,14 +293,13 @@ - web/.* - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - - zuul-tox-remote: - timeout: 2700 # 45 minutes + - zuul-tox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-build-image - nodepool-zuul-functional: voting: false - - zuul-client-zuul-functional + - zuul-tox-zuul-client - zuul-build-python-release gate: jobs: @@ -269,14 +308,8 @@ - tox-linters: vars: tox_install_bindep: false - - tox-py36: - nodeset: ubuntu-bionic - timeout: 4800 # 80 minutes - vars: *zuul_tox_vars - - tox-py38: - timeout: 4800 # 80 minutes - nodeset: ubuntu-bionic - vars: *zuul_tox_vars + - zuul-tox-py36 + - zuul-tox-py38 - zuul-build-dashboard - nodejs-run-lint: vars: @@ -289,12 +322,11 @@ - web/.* - zuul-stream-functional-2.8 - zuul-stream-functional-2.9 - - zuul-tox-remote: - timeout: 2700 # 45 minutes + - zuul-tox-remote - zuul-quick-start: requires: nodepool-container-image dependencies: zuul-upload-image - - zuul-client-zuul-functional + - zuul-tox-zuul-client - zuul-build-python-release promote: jobs: diff --git a/bindep.txt b/bindep.txt index 63bc952c7d..d8695dc194 100644 --- a/bindep.txt +++ b/bindep.txt @@ -8,7 +8,6 @@ postgresql [test] libjpeg-dev [test !platform:rpm] libjpeg-turbo-devel [test platform:rpm] openssl [test] -zookeeperd [test platform:dpkg] musl-dev [compile test platform:apk] make [compile test platform:apk] linux-headers [compile test platform:apk] diff --git a/playbooks/common/post-system-logs.yaml b/playbooks/common/post-system-logs.yaml deleted file mode 100644 index 830899c387..0000000000 --- a/playbooks/common/post-system-logs.yaml +++ /dev/null @@ -1,5 +0,0 @@ -- hosts: all - tasks: - - - name: Collect zookeeper logs - shell: "cp /var/log/zookeeper/zookeeper.log {{ zuul_output_dir }}/logs/zookeeper.log" diff --git a/playbooks/zuul-tox/post-system-logs.yaml b/playbooks/zuul-tox/post-system-logs.yaml new file mode 100644 index 0000000000..7a9da4821e --- /dev/null +++ b/playbooks/zuul-tox/post-system-logs.yaml @@ -0,0 +1,5 @@ +- hosts: all + tasks: + + - name: Collect zookeeper logs + shell: "cp /opt/zookeeper/logs/* {{ zuul_output_dir }}/logs/" diff --git a/playbooks/zuul-tox/pre.yaml b/playbooks/zuul-tox/pre.yaml new file mode 100644 index 0000000000..45d213e97f --- /dev/null +++ b/playbooks/zuul-tox/pre.yaml @@ -0,0 +1,4 @@ +- hosts: all + roles: + - role: ensure-zookeeper + zookeeper_use_tls: true diff --git a/tests/base.py b/tests/base.py index 1b0c6791e6..351d04d74b 100644 --- a/tests/base.py +++ b/tests/base.py @@ -3352,11 +3352,20 @@ class FakeNodepool(object): log = logging.getLogger("zuul.test.FakeNodepool") - def __init__(self, host, port, chroot): + def __init__(self, zk_chroot_fixture): self.complete_event = threading.Event() self.host_keys = None + self.client = kazoo.client.KazooClient( - hosts='%s:%s%s' % (host, port, chroot)) + hosts='%s:%s%s' % ( + zk_chroot_fixture.zookeeper_host, + zk_chroot_fixture.zookeeper_port, + zk_chroot_fixture.zookeeper_chroot), + use_ssl=True, + keyfile=zk_chroot_fixture.zookeeper_key, + certfile=zk_chroot_fixture.zookeeper_cert, + ca=zk_chroot_fixture.zookeeper_ca, + ) self.client.start() self.registerLauncher() self._running = True @@ -3614,10 +3623,25 @@ class ChrootedKazooFixture(fixtures.Fixture): host = zk_host port = None + zk_ca = os.environ.get('ZUUL_ZK_CA', None) + if not zk_ca: + zk_ca = os.path.join(os.path.dirname(__file__), + '../tools/ca/certs/cacert.pem') + self.zookeeper_ca = zk_ca + zk_cert = os.environ.get('ZUUL_ZK_CERT', None) + if not zk_cert: + zk_cert = os.path.join(os.path.dirname(__file__), + '../tools/ca/certs/client.pem') + self.zookeeper_cert = zk_cert + zk_key = os.environ.get('ZUUL_ZK_KEY', None) + if not zk_key: + zk_key = os.path.join(os.path.dirname(__file__), + '../tools/ca/keys/clientkey.pem') + self.zookeeper_key = zk_key self.zookeeper_host = host if not port: - self.zookeeper_port = 2181 + self.zookeeper_port = 2281 else: self.zookeeper_port = int(port) @@ -3636,7 +3660,11 @@ class ChrootedKazooFixture(fixtures.Fixture): # Ensure the chroot path exists and clean up any pre-existing znodes. _tmp_client = kazoo.client.KazooClient( - hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10 + hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10, + use_ssl=True, + keyfile=self.zookeeper_key, + certfile=self.zookeeper_cert, + ca=self.zookeeper_ca, ) _tmp_client.start() @@ -3651,7 +3679,12 @@ class ChrootedKazooFixture(fixtures.Fixture): '''Remove the chroot path.''' # Need a non-chroot'ed client to remove the chroot path _tmp_client = kazoo.client.KazooClient( - hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port)) + hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port), + use_ssl=True, + keyfile=self.zookeeper_key, + certfile=self.zookeeper_cert, + ca=self.zookeeper_ca, + ) _tmp_client.start() _tmp_client.delete(self.zookeeper_chroot, recursive=True) _tmp_client.stop() @@ -3722,8 +3755,7 @@ class ZuulWebFixture(fixtures.Fixture): self.info = zuul.model.WebInfo.fromConfig(config) else: self.info = info - self.zk_client = ZooKeeperClient.fromConfig(config, - _require_tls=False) + self.zk_client = ZooKeeperClient.fromConfig(config) self.zk_client.connect() self.test_root = test_root @@ -3995,8 +4027,7 @@ class SchedulerTestApp: self.config, self.sched) merge_client = RecordingMergeClient(self.config, self.sched) nodepool = zuul.nodepool.Nodepool(self.sched) - zk_client = ZooKeeperClient.fromConfig(self.config, - _require_tls=False) + zk_client = ZooKeeperClient.fromConfig(self.config) zk_client.connect() self.sched.setExecutor(executor_client) @@ -4179,10 +4210,7 @@ class ZuulTestCase(BaseTestCase): super(ZuulTestCase, self).setUp() self.setupZK() - self.fake_nodepool = FakeNodepool( - self.zk_chroot_fixture.zookeeper_host, - self.zk_chroot_fixture.zookeeper_port, - self.zk_chroot_fixture.zookeeper_chroot) + self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture) if not KEEP_TEMPDIRS: tmp_root = self.useFixture(fixtures.TempDir( @@ -4270,9 +4298,14 @@ class ZuulTestCase(BaseTestCase): self.zk_chroot_fixture.zookeeper_chroot) self.config.set('zookeeper', 'hosts', zk_hosts) self.config.set('zookeeper', 'session_timeout', '30') + self.config.set('zookeeper', 'tls_cert', + self.zk_chroot_fixture.zookeeper_cert) + self.config.set('zookeeper', 'tls_key', + self.zk_chroot_fixture.zookeeper_key) + self.config.set('zookeeper', 'tls_ca', + self.zk_chroot_fixture.zookeeper_ca) - self.zk_client = ZooKeeperClient.fromConfig(self.config, - _require_tls=False) + self.zk_client = ZooKeeperClient.fromConfig(self.config) self.zk_client.connect() self.rpcclient = zuul.rpcclient.RPCClient( diff --git a/tests/unit/test_nodepool.py b/tests/unit/test_nodepool.py index 5a02e5ab9a..d34233a0a7 100644 --- a/tests/unit/test_nodepool.py +++ b/tests/unit/test_nodepool.py @@ -33,12 +33,16 @@ class TestNodepool(BaseTestCase): self.statsd = None self.zk_chroot_fixture = self.useFixture( ChrootedKazooFixture(self.id())) - self.zk_config = '%s:%s%s' % ( + zk_hosts = '%s:%s%s' % ( self.zk_chroot_fixture.zookeeper_host, self.zk_chroot_fixture.zookeeper_port, self.zk_chroot_fixture.zookeeper_chroot) - self.zk_client = ZooKeeperClient(self.zk_config) + self.zk_client = ZooKeeperClient( + zk_hosts, + tls_cert=self.zk_chroot_fixture.zookeeper_cert, + tls_key=self.zk_chroot_fixture.zookeeper_key, + tls_ca=self.zk_chroot_fixture.zookeeper_ca) self.zk_nodepool = ZooKeeperNodepool(self.zk_client) self.addCleanup(self.zk_client.disconnect) self.zk_client.connect() @@ -49,10 +53,7 @@ class TestNodepool(BaseTestCase): # needs, so we pass 'self' as the scheduler. self.nodepool = zuul.nodepool.Nodepool(self) - self.fake_nodepool = FakeNodepool( - self.zk_chroot_fixture.zookeeper_host, - self.zk_chroot_fixture.zookeeper_port, - self.zk_chroot_fixture.zookeeper_chroot) + self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture) self.addCleanup(self.fake_nodepool.stop) def waitForRequests(self): diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index 67d06e13c9..7ca2a3b734 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -522,8 +522,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): logfile = open(ansible_log, 'r') self.addCleanup(logfile.close) - zk_client = ZooKeeperClient.fromConfig(self.config, - _require_tls=False) + zk_client = ZooKeeperClient.fromConfig(self.config) zk_client.connect() self.addCleanup(zk_client.disconnect) diff --git a/tests/unit/test_zk.py b/tests/unit/test_zk.py index c6e883dcaf..ad12b5c1de 100644 --- a/tests/unit/test_zk.py +++ b/tests/unit/test_zk.py @@ -30,12 +30,16 @@ class TestZK(BaseTestCase): self.zk_chroot_fixture = self.useFixture( ChrootedKazooFixture(self.id())) - self.zk_config = '%s:%s%s' % ( + zk_hosts = '%s:%s%s' % ( self.zk_chroot_fixture.zookeeper_host, self.zk_chroot_fixture.zookeeper_port, self.zk_chroot_fixture.zookeeper_chroot) - self.zk_client = ZooKeeperClient(self.zk_config) + self.zk_client = ZooKeeperClient( + zk_hosts, + tls_cert=self.zk_chroot_fixture.zookeeper_cert, + tls_key=self.zk_chroot_fixture.zookeeper_key, + tls_ca=self.zk_chroot_fixture.zookeeper_ca) self.zk_nodepool = ZooKeeperNodepool(self.zk_client) self.addCleanup(self.zk_client.disconnect) self.zk_client.connect() diff --git a/tools/docker-compose.yaml b/tools/docker-compose.yaml index d7c3ce0cd9..e17750bb22 100644 --- a/tools/docker-compose.yaml +++ b/tools/docker-compose.yaml @@ -29,7 +29,10 @@ services: - ZOO_AUTOPURGE_PURGEINTERVAL=1 - ZOO_LOG4J_PROP=WARN ports: - - "2181:2181" + - "2281:2281" tmpfs: - /data - /datalog + volumes: + - "./ca:/var/certs:z" + - "./zoo.cfg:/conf/zoo.cfg:z" diff --git a/tools/test-setup-docker.sh b/tools/test-setup-docker.sh index c64dccb41c..526f025c76 100755 --- a/tools/test-setup-docker.sh +++ b/tools/test-setup-docker.sh @@ -3,6 +3,7 @@ set -eu cd $(dirname $0) +SCRIPT_DIR="$(pwd)" # Select docker or podman if command -v docker > /dev/null; then @@ -33,6 +34,11 @@ else podman-compose down fi +CA_DIR=$SCRIPT_DIR/ca + +mkdir -p $CA_DIR +$SCRIPT_DIR/zk-ca.sh $CA_DIR zuul-test-zookeeper + ${COMPOSE} up -d echo "Waiting for mysql" diff --git a/tools/test-setup.sh b/tools/test-setup.sh index cb524f9c5f..2377635696 100755 --- a/tools/test-setup.sh +++ b/tools/test-setup.sh @@ -7,23 +7,15 @@ # This setup needs to be run as a user that can run sudo. TOOLSDIR=$(dirname $0) -# Config Zookeeper to run on tmpfs -sudo service zookeeper stop -DATADIR=$(sed -n -e 's/^dataDir=//p' /etc/zookeeper/conf/zoo.cfg) -sudo mount -t tmpfs -o nodev,nosuid,size=500M none $DATADIR -echo "autopurge.purgeInterval=1" | sudo tee -a /etc/zookeeper/conf/zoo.cfg -echo "maxClientCnxns=1000" | sudo tee -a /etc/zookeeper/conf/zoo.cfg - # Prepare a tmpfs for Zuul test root if [[ -n "${ZUUL_TEST_ROOT:-}" ]]; then sudo mkdir -p "$ZUUL_TEST_ROOT" sudo mount -t tmpfs -o noatime,nodev,nosuid,size=64M none "$ZUUL_TEST_ROOT" fi -# Be sure mysql and zookeeper are started. +# Be sure mysql is started. sudo service mysql start sudo service postgresql start -sudo service zookeeper start # The root password for the MySQL database; pass it in via # MYSQL_ROOT_PW. diff --git a/tools/zoo.cfg b/tools/zoo.cfg new file mode 100644 index 0000000000..bf1ac6cf7d --- /dev/null +++ b/tools/zoo.cfg @@ -0,0 +1,16 @@ +# zoo.cfg for use in test-setup.sh +dataDir=/data +dataLogDir=/datalog +tickTime=2000 +initLimit=5 +syncLimit=2 +autopurge.snapRetainCount=3 +autopurge.purgeInterval=0 +maxClientCnxns=1000 +standaloneEnabled=true +admin.enableServer=true +server.1=nodepool-test-zookeeper:2888:3888 +serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory +secureClientPort=2281 +ssl.keyStore.location=/var/certs/keystores/zuul-test-zookeeper.pem +ssl.trustStore.location=/var/certs/certs/cacert.pem diff --git a/tox.ini b/tox.ini index 517eb01f8b..a6f704e826 100644 --- a/tox.ini +++ b/tox.ini @@ -27,6 +27,10 @@ passenv = ZUUL_MYSQL_HOST ZUUL_POSTGRES_HOST ZUUL_TEST_ROOT + ZUUL_ZK_HOST + ZUUL_ZK_CA + ZUUL_ZK_CERT + ZUUL_ZK_KEY usedevelop = True whitelist_externals = bash deps = @@ -106,6 +110,10 @@ passenv = ZUUL_REMOTE_IPV4 ZUUL_SSH_KEY ZUUL_TEST_ROOT + ZUUL_ZK_HOST + ZUUL_ZK_CA + ZUUL_ZK_CERT + ZUUL_ZK_KEY commands = stestr run --test-path ./tests/remote {posargs} diff --git a/zuul/zk/__init__.py b/zuul/zk/__init__.py index f0388b4570..c085d21205 100644 --- a/zuul/zk/__init__.py +++ b/zuul/zk/__init__.py @@ -144,17 +144,14 @@ class ZooKeeperClient(object): self.client.set_hosts(hosts=hosts) @classmethod - def fromConfig(cls, config: ConfigParser, - _require_tls=True) -> "ZooKeeperClient": - # _require_tls is temporary, only used until we move the tests - # to use TLS. + def fromConfig(cls, config: ConfigParser) -> "ZooKeeperClient": hosts = get_default(config, "zookeeper", "hosts") if not hosts: raise Exception("The zookeeper hosts config value is required") tls_key = get_default(config, "zookeeper", "tls_key") tls_cert = get_default(config, "zookeeper", "tls_cert") tls_ca = get_default(config, "zookeeper", "tls_ca") - if _require_tls and not all([tls_key, tls_cert, tls_ca]): + if not all([tls_key, tls_cert, tls_ca]): raise Exception( "A TLS ZooKeeper connection is required; please supply the " "tls_* zookeeper config values."