zuul/zuul
Code Issues Proposed changes

Use ZooKeeper TLS in tests

Browse Source 
This mirrors the configuration in Nodepool for using TLS-enabled
ZooKeeper in tests.  We use the ensure-zookeeper role in order
to get a newer ZooKeeper than is supplied in bionic.

Change-Id: I14413fccbc9a6a7a75b6233d667e2a1d2856d894
This commit is contained in:
James E. Blair 2021-02-24 16:57:49 -08:00
parent fa2175c22a
commit 74a9c9de9b
16 changed files with 171 additions and 76 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -24,3 +24,4 @@ zuul/web/static/*
!.keep
node_modules
yarn-error.log
tools/ca/

92
.zuul.yaml
View File

@ -44,16 +44,70 @@
vars:
zuul_ansible_version: 2.9
- job:
name: zuul-tox
description: |
Zuul unit tests with ZooKeeper running
parent: tox
nodeset: ubuntu-bionic
pre-run: playbooks/zuul-tox/pre.yaml
post-run: playbooks/zuul-tox/post-system-logs.yaml
vars:
tox_environment:
ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
test_setup_environment:
ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
- job:
name: zuul-tox-remote
parent: tox
timeout: 2700 # 45 minutes
pre-run: playbooks/zuul-tox/pre.yaml
post-run: playbooks/zuul-tox/post-system-logs.yaml
vars:
tox_envlist: remote
tox_environment:
ZUUL_ZK_CA: /opt/zookeeper/ca/certs/cacert.pem
ZUUL_ZK_CERT: /opt/zookeeper/ca/certs/client.pem
ZUUL_ZK_KEY: /opt/zookeeper/ca/keys/clientkey.pem
ZUUL_SSH_KEY: /home/zuul/.ssh/id_rsa
ZUUL_REMOTE_IPV4: "{{ nodepool.interface_ip }}"
ZUUL_REMOTE_KEEP: "true"
- job:
name: zuul-tox-zuul-client
parent: zuul-tox
description: |
Test that Zuul and zuul-client work together.
required-projects:
- zuul/zuul
- zuul/zuul-client
vars:
zuul_work_dir: "{{ zuul.projects['opendev.org/zuul/zuul'].src_dir }}"
tox_envlist: zuul_client
nodeset: ubuntu-bionic
- job:
name: zuul-tox-py36
parent: zuul-tox
timeout: 4800 # 80 minutes
vars:
tox_envlist: py36
python_version: 3.6
- job:
name: zuul-tox-py38
parent: zuul-tox
timeout: 4800 # 80 minutes
vars:
tox_envlist: py38
python_version: 3.8
- job:
name: zuul-build-dashboard
parent: build-javascript-deployment
@ -223,22 +277,8 @@
- tox-linters:
vars:
tox_install_bindep: false
- tox-py36:
nodeset: ubuntu-bionic
timeout: 4800 # 80 minutes
vars: &zuul_tox_vars
test_setup_environment:
ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
tox_environment:
ZUUL_TEST_ROOT: /tmp/zuul-test
YARN_REGISTRY: "https://{{ zuul_site_mirror_fqdn }}:4443/registry.npmjs"
post-run: playbooks/common/post-system-logs.yaml
- tox-py38:
timeout: 4800 # 80 minutes
nodeset: ubuntu-bionic
vars: *zuul_tox_vars
post-run: playbooks/common/post-system-logs.yaml
- zuul-tox-py36
- zuul-tox-py38
- zuul-build-dashboard-openstack-whitelabel
- zuul-build-dashboard-software-factory
- zuul-build-dashboard-opendev
@ -253,14 +293,13 @@
- web/.*
- zuul-stream-functional-2.8
- zuul-stream-functional-2.9
- zuul-tox-remote:
timeout: 2700 # 45 minutes
- zuul-tox-remote
- zuul-quick-start:
requires: nodepool-container-image
dependencies: zuul-build-image
- nodepool-zuul-functional:
voting: false
- zuul-client-zuul-functional
- zuul-tox-zuul-client
- zuul-build-python-release
gate:
jobs:
@ -269,14 +308,8 @@
- tox-linters:
vars:
tox_install_bindep: false
- tox-py36:
nodeset: ubuntu-bionic
timeout: 4800 # 80 minutes
vars: *zuul_tox_vars
- tox-py38:
timeout: 4800 # 80 minutes
nodeset: ubuntu-bionic
vars: *zuul_tox_vars
- zuul-tox-py36
- zuul-tox-py38
- zuul-build-dashboard
- nodejs-run-lint:
vars:
@ -289,12 +322,11 @@
- web/.*
- zuul-stream-functional-2.8
- zuul-stream-functional-2.9
- zuul-tox-remote:
timeout: 2700 # 45 minutes
- zuul-tox-remote
- zuul-quick-start:
requires: nodepool-container-image
dependencies: zuul-upload-image
- zuul-client-zuul-functional
- zuul-tox-zuul-client
- zuul-build-python-release
promote:
jobs:

1
bindep.txt
View File

@ -8,7 +8,6 @@ postgresql [test]
libjpeg-dev [test !platform:rpm]
libjpeg-turbo-devel [test platform:rpm]
openssl [test]
zookeeperd [test platform:dpkg]
musl-dev [compile test platform:apk]
make [compile test platform:apk]
linux-headers [compile test platform:apk]

5
playbooks/common/post-system-logs.yaml
View File

@ -1,5 +0,0 @@
- hosts: all
tasks:
- name: Collect zookeeper logs
shell: "cp /var/log/zookeeper/zookeeper.log {{ zuul_output_dir }}/logs/zookeeper.log"

5
playbooks/zuul-tox/post-system-logs.yaml Normal file
View File

@ -0,0 +1,5 @@
- hosts: all
tasks:
- name: Collect zookeeper logs
shell: "cp /opt/zookeeper/logs/* {{ zuul_output_dir }}/logs/"

4
playbooks/zuul-tox/pre.yaml Normal file
View File

@ -0,0 +1,4 @@
- hosts: all
roles:
- role: ensure-zookeeper
zookeeper_use_tls: true

63
tests/base.py
View File

@ -3352,11 +3352,20 @@ class FakeNodepool(object):
log = logging.getLogger("zuul.test.FakeNodepool")
def __init__(self, host, port, chroot):
def __init__(self, zk_chroot_fixture):
self.complete_event = threading.Event()
self.host_keys = None
self.client = kazoo.client.KazooClient(
hosts='%s:%s%s' % (host, port, chroot))
hosts='%s:%s%s' % (
zk_chroot_fixture.zookeeper_host,
zk_chroot_fixture.zookeeper_port,
zk_chroot_fixture.zookeeper_chroot),
use_ssl=True,
keyfile=zk_chroot_fixture.zookeeper_key,
certfile=zk_chroot_fixture.zookeeper_cert,
ca=zk_chroot_fixture.zookeeper_ca,
)
self.client.start()
self.registerLauncher()
self._running = True
@ -3614,10 +3623,25 @@ class ChrootedKazooFixture(fixtures.Fixture):
host = zk_host
port = None
zk_ca = os.environ.get('ZUUL_ZK_CA', None)
if not zk_ca:
zk_ca = os.path.join(os.path.dirname(__file__),
'../tools/ca/certs/cacert.pem')
self.zookeeper_ca = zk_ca
zk_cert = os.environ.get('ZUUL_ZK_CERT', None)
if not zk_cert:
zk_cert = os.path.join(os.path.dirname(__file__),
'../tools/ca/certs/client.pem')
self.zookeeper_cert = zk_cert
zk_key = os.environ.get('ZUUL_ZK_KEY', None)
if not zk_key:
zk_key = os.path.join(os.path.dirname(__file__),
'../tools/ca/keys/clientkey.pem')
self.zookeeper_key = zk_key
self.zookeeper_host = host
if not port:
self.zookeeper_port = 2181
self.zookeeper_port = 2281
else:
self.zookeeper_port = int(port)
@ -3636,7 +3660,11 @@ class ChrootedKazooFixture(fixtures.Fixture):
# Ensure the chroot path exists and clean up any pre-existing znodes.
_tmp_client = kazoo.client.KazooClient(
hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10
hosts=f'{self.zookeeper_host}:{self.zookeeper_port}', timeout=10,
use_ssl=True,
keyfile=self.zookeeper_key,
certfile=self.zookeeper_cert,
ca=self.zookeeper_ca,
)
_tmp_client.start()
@ -3651,7 +3679,12 @@ class ChrootedKazooFixture(fixtures.Fixture):
'''Remove the chroot path.'''
# Need a non-chroot'ed client to remove the chroot path
_tmp_client = kazoo.client.KazooClient(
hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port))
hosts='%s:%s' % (self.zookeeper_host, self.zookeeper_port),
use_ssl=True,
keyfile=self.zookeeper_key,
certfile=self.zookeeper_cert,
ca=self.zookeeper_ca,
)
_tmp_client.start()
_tmp_client.delete(self.zookeeper_chroot, recursive=True)
_tmp_client.stop()
@ -3722,8 +3755,7 @@ class ZuulWebFixture(fixtures.Fixture):
self.info = zuul.model.WebInfo.fromConfig(config)
else:
self.info = info
self.zk_client = ZooKeeperClient.fromConfig(config,
_require_tls=False)
self.zk_client = ZooKeeperClient.fromConfig(config)
self.zk_client.connect()
self.test_root = test_root
@ -3995,8 +4027,7 @@ class SchedulerTestApp:
self.config, self.sched)
merge_client = RecordingMergeClient(self.config, self.sched)
nodepool = zuul.nodepool.Nodepool(self.sched)
zk_client = ZooKeeperClient.fromConfig(self.config,
_require_tls=False)
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
self.sched.setExecutor(executor_client)
@ -4179,10 +4210,7 @@ class ZuulTestCase(BaseTestCase):
super(ZuulTestCase, self).setUp()
self.setupZK()
self.fake_nodepool = FakeNodepool(
self.zk_chroot_fixture.zookeeper_host,
self.zk_chroot_fixture.zookeeper_port,
self.zk_chroot_fixture.zookeeper_chroot)
self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture)
if not KEEP_TEMPDIRS:
tmp_root = self.useFixture(fixtures.TempDir(
@ -4270,9 +4298,14 @@ class ZuulTestCase(BaseTestCase):
self.zk_chroot_fixture.zookeeper_chroot)
self.config.set('zookeeper', 'hosts', zk_hosts)
self.config.set('zookeeper', 'session_timeout', '30')
self.config.set('zookeeper', 'tls_cert',
self.zk_chroot_fixture.zookeeper_cert)
self.config.set('zookeeper', 'tls_key',
self.zk_chroot_fixture.zookeeper_key)
self.config.set('zookeeper', 'tls_ca',
self.zk_chroot_fixture.zookeeper_ca)
self.zk_client = ZooKeeperClient.fromConfig(self.config,
_require_tls=False)
self.zk_client = ZooKeeperClient.fromConfig(self.config)
self.zk_client.connect()
self.rpcclient = zuul.rpcclient.RPCClient(

13
tests/unit/test_nodepool.py
View File

@ -33,12 +33,16 @@ class TestNodepool(BaseTestCase):
self.statsd = None
self.zk_chroot_fixture = self.useFixture(
ChrootedKazooFixture(self.id()))
self.zk_config = '%s:%s%s' % (
zk_hosts = '%s:%s%s' % (
self.zk_chroot_fixture.zookeeper_host,
self.zk_chroot_fixture.zookeeper_port,
self.zk_chroot_fixture.zookeeper_chroot)
self.zk_client = ZooKeeperClient(self.zk_config)
self.zk_client = ZooKeeperClient(
zk_hosts,
tls_cert=self.zk_chroot_fixture.zookeeper_cert,
tls_key=self.zk_chroot_fixture.zookeeper_key,
tls_ca=self.zk_chroot_fixture.zookeeper_ca)
self.zk_nodepool = ZooKeeperNodepool(self.zk_client)
self.addCleanup(self.zk_client.disconnect)
self.zk_client.connect()
@ -49,10 +53,7 @@ class TestNodepool(BaseTestCase):
# needs, so we pass 'self' as the scheduler.
self.nodepool = zuul.nodepool.Nodepool(self)
self.fake_nodepool = FakeNodepool(
self.zk_chroot_fixture.zookeeper_host,
self.zk_chroot_fixture.zookeeper_port,
self.zk_chroot_fixture.zookeeper_chroot)
self.fake_nodepool = FakeNodepool(self.zk_chroot_fixture)
self.addCleanup(self.fake_nodepool.stop)
def waitForRequests(self):

3
tests/unit/test_streaming.py
View File

@ -522,8 +522,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
logfile = open(ansible_log, 'r')
self.addCleanup(logfile.close)
zk_client = ZooKeeperClient.fromConfig(self.config,
_require_tls=False)
zk_client = ZooKeeperClient.fromConfig(self.config)
zk_client.connect()
self.addCleanup(zk_client.disconnect)

8
tests/unit/test_zk.py
View File

@ -30,12 +30,16 @@ class TestZK(BaseTestCase):
self.zk_chroot_fixture = self.useFixture(
ChrootedKazooFixture(self.id()))
self.zk_config = '%s:%s%s' % (
zk_hosts = '%s:%s%s' % (
self.zk_chroot_fixture.zookeeper_host,
self.zk_chroot_fixture.zookeeper_port,
self.zk_chroot_fixture.zookeeper_chroot)
self.zk_client = ZooKeeperClient(self.zk_config)
self.zk_client = ZooKeeperClient(
zk_hosts,
tls_cert=self.zk_chroot_fixture.zookeeper_cert,
tls_key=self.zk_chroot_fixture.zookeeper_key,
tls_ca=self.zk_chroot_fixture.zookeeper_ca)
self.zk_nodepool = ZooKeeperNodepool(self.zk_client)
self.addCleanup(self.zk_client.disconnect)
self.zk_client.connect()

5
tools/docker-compose.yaml
View File

@ -29,7 +29,10 @@ services:
- ZOO_AUTOPURGE_PURGEINTERVAL=1
- ZOO_LOG4J_PROP=WARN
ports:
- "2181:2181"
- "2281:2281"
tmpfs:
- /data
- /datalog
volumes:
- "./ca:/var/certs:z"
- "./zoo.cfg:/conf/zoo.cfg:z"

6
tools/test-setup-docker.sh
View File

@ -3,6 +3,7 @@
set -eu
cd $(dirname $0)
SCRIPT_DIR="$(pwd)"
# Select docker or podman
if command -v docker > /dev/null; then
@ -33,6 +34,11 @@ else
podman-compose down
fi
CA_DIR=$SCRIPT_DIR/ca
mkdir -p $CA_DIR
$SCRIPT_DIR/zk-ca.sh $CA_DIR zuul-test-zookeeper
${COMPOSE} up -d
echo "Waiting for mysql"

10
tools/test-setup.sh
View File

@ -7,23 +7,15 @@
# This setup needs to be run as a user that can run sudo.
TOOLSDIR=$(dirname $0)
# Config Zookeeper to run on tmpfs
sudo service zookeeper stop
DATADIR=$(sed -n -e 's/^dataDir=//p' /etc/zookeeper/conf/zoo.cfg)
sudo mount -t tmpfs -o nodev,nosuid,size=500M none $DATADIR
echo "autopurge.purgeInterval=1" | sudo tee -a /etc/zookeeper/conf/zoo.cfg
echo "maxClientCnxns=1000" | sudo tee -a /etc/zookeeper/conf/zoo.cfg
# Prepare a tmpfs for Zuul test root
if [[ -n "${ZUUL_TEST_ROOT:-}" ]]; then
sudo mkdir -p "$ZUUL_TEST_ROOT"
sudo mount -t tmpfs -o noatime,nodev,nosuid,size=64M none "$ZUUL_TEST_ROOT"
fi
# Be sure mysql and zookeeper are started.
# Be sure mysql is started.
sudo service mysql start
sudo service postgresql start
sudo service zookeeper start
# The root password for the MySQL database; pass it in via
# MYSQL_ROOT_PW.

16
tools/zoo.cfg Normal file
View File

@ -0,0 +1,16 @@
# zoo.cfg for use in test-setup.sh
dataDir=/data
dataLogDir=/datalog
tickTime=2000
initLimit=5
syncLimit=2
autopurge.snapRetainCount=3
autopurge.purgeInterval=0
maxClientCnxns=1000
standaloneEnabled=true
admin.enableServer=true
server.1=nodepool-test-zookeeper:2888:3888
serverCnxnFactory=org.apache.zookeeper.server.NettyServerCnxnFactory
secureClientPort=2281
ssl.keyStore.location=/var/certs/keystores/zuul-test-zookeeper.pem
ssl.trustStore.location=/var/certs/certs/cacert.pem

8
tox.ini
View File

@ -27,6 +27,10 @@ passenv =
ZUUL_MYSQL_HOST
ZUUL_POSTGRES_HOST
ZUUL_TEST_ROOT
ZUUL_ZK_HOST
ZUUL_ZK_CA
ZUUL_ZK_CERT
ZUUL_ZK_KEY
usedevelop = True
whitelist_externals = bash
deps =
@ -106,6 +110,10 @@ passenv =
ZUUL_REMOTE_IPV4
ZUUL_SSH_KEY
ZUUL_TEST_ROOT
ZUUL_ZK_HOST
ZUUL_ZK_CA
ZUUL_ZK_CERT
ZUUL_ZK_KEY
commands =
stestr run --test-path ./tests/remote {posargs}

7
zuul/zk/__init__.py
View File

@ -144,17 +144,14 @@ class ZooKeeperClient(object):
self.client.set_hosts(hosts=hosts)
@classmethod
def fromConfig(cls, config: ConfigParser,
_require_tls=True) -> "ZooKeeperClient":
# _require_tls is temporary, only used until we move the tests
# to use TLS.
def fromConfig(cls, config: ConfigParser) -> "ZooKeeperClient":
hosts = get_default(config, "zookeeper", "hosts")
if not hosts:
raise Exception("The zookeeper hosts config value is required")
tls_key = get_default(config, "zookeeper", "tls_key")
tls_cert = get_default(config, "zookeeper", "tls_cert")
tls_ca = get_default(config, "zookeeper", "tls_ca")
if _require_tls and not all([tls_key, tls_cert, tls_ca]):
if not all([tls_key, tls_cert, tls_ca]):
raise Exception(
"A TLS ZooKeeper connection is required; please supply the "
"tls_* zookeeper config values."