From 1ac445b1d9f7647d3f8c9b055fc52e2bac268cd3 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 21 Apr 2021 16:00:40 +1000 Subject: [PATCH] nodepool-base: prefer ZK IPv6 addresses The current loop here uses the ansible_host value of the ZK servers, which we have set to the IPv4 address in the inventory. nb03 is constantly dropping out of ZK; for the record the logs record: 2021-04-21 05:56:15,151 WARNING kazoo.client: Connection dropped: socket connection error: Connection reset by peer 2021-04-21 05:56:15,151 WARNING kazoo.client: Transition to CONNECTING 2021-04-21 05:56:15,151 INFO kazoo.client: Zookeeper connection lost 2021-04-21 05:56:15,152 INFO kazoo.client: Connecting to 23.253.90.246(23.253.90.246):2281, use_ssl: True 2021-04-21 05:56:15,176 INFO kazoo.client: Zookeeper connection established, state: CONNECTED and this happens every few minutes. This cloud does IPv4 behind a NAT and it seems very likely this is related. So the primary motivation here is to see if using IPv6 clears this up, giving us some datapoints. However I think that our other nodepool hosts should all be fine to use ZK over IPv6. However, I think in the gate we may have cases where hosts don't have IPv6 addresses, so this looks for the v6 address and if not found, falls back to the current ansible_host behaviour. Change-Id: Ifde86ddd632662f36bcbe2a0dc99660f06b01ac3 --- .../nodepool-base/library/make_nodepool_zk_hosts.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py b/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py index 2919c98521..a65fc6832d 100644 --- a/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py +++ b/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py @@ -29,8 +29,15 @@ def main(): zk_hosts = [] try: for host in p['zk_group']: + # Prefer public_v6 if set, otherwise ansible_host + addr = None + if 'public_v6' in p['hostvars'][host]: + addr = p['hostvars'][host]['public_v6'] + if not addr: + addr = p['hostvars'][host]['ansible_host'] + zk_hosts.append(dict( - host=p['hostvars'][host]['ansible_host'], + host=addr, port=2281 )) module.exit_json(hosts=zk_hosts, changed=True)