Run zookeeper cluster in nodepool jobs

Rather than running a local zookeeper, just run a real zookeeper.
Also, get rid of nb01-test and just use nb04 - what could possibly
go wrong?

Dynamically write zookeeper host information to nodepool.yaml

So that we can run an actual zk using the new zk role on hosts in
ansible inventory, we need to write out the ip addresses of the
hosts that we build in zuul. This means having the info baked in
to the file in project-config isn't going to work.

We can do this in prod too, it shouldn't hurt anything.

Increase timeout for run-service-nodepool

We need to fix the playbook, but we'll do that after we get the
puppet gone.

Change-Id: Ib01d461ae2c5cec3c31ec5105a41b1a99ff9d84a

.zuul.yaml

@@ -1224,22 +1224,36 @@
     parent: system-config-run
     description: |
       Run the playbook for nodepool.
+    timeout: 3600
     nodeset:
       nodes:
         - name: bridge.openstack.org
           label: ubuntu-bionic
+        - name: zk01.opendev.org
+          label: ubuntu-bionic
         - name: nl01.openstack.org
           label: ubuntu-xenial
         - name: nb01.openstack.org
           label: ubuntu-xenial
-        - name: nb01-test.opendev.org
+        - name: nb04.opendev.org
           label: ubuntu-bionic
     required-projects:
       - openstack/project-config
       - opendev/system-config
+    host-vars:
+      nl01.openstack.org:
+        host_copy_output:
+          '/etc/nodepool/nodepool.yaml': logs
+      nb01.openstack.org:
+        host_copy_output:
+          '/etc/nodepool/nodepool.yaml': logs
+      nb04.opendev.org:
+        host_copy_output:
+          '/etc/nodepool/nodepool.yaml': logs
     vars:
       run_playbooks:
         - playbooks/service-letsencrypt.yaml
+        - playbooks/service-zookeeper.yaml
         - playbooks/service-nodepool.yaml
         - playbooks/remote_puppet_else.yaml
     files:
@@ -2217,7 +2231,6 @@
       - playbooks/roles/configure-kubectl/
       - playbooks/roles/configure-openstacksdk/
       - playbooks/roles/install-docker/
-      - playbooks/roles/install-zookeeper/
       - playbooks/roles/nodepool-
       - playbooks/templates/clouds/nodepool_
 

inventory/groups.yaml

@@ -102,7 +102,6 @@ groups:
     - nb[0-9]*.opendev.org
   nodepool-launcher:
     - nl[0-9]*.openstack.org
-  nodepool-launcher_opendev:
     - nl[0-8]*.opendev.org
   ns:
     - ns[0-9]*.open*.org

playbooks/roles/install-zookeeper/README.rst

@@ -1,4 +0,0 @@
-An ansible role to install Zookeeper
-
-**Role Variables**
-

playbooks/roles/install-zookeeper/tasks/main.yaml

@@ -1,11 +0,0 @@
-- name: Install zookeeper
-  package:
-    name:
-      - zookeeper
-      - zookeeperd
-    state: present
-
-- name: Start zookeeper service
-  service:
-    name: zookeeper
-    state: started

playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+#
+# Copyright 2020 Red Hat, Inc
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ansible.module_utils.basic import AnsibleModule
+
+
+def main():
+    module = AnsibleModule(
+        argument_spec=dict(
+            hostvars=dict(required=True, type='dict'),
+            zk_group=dict(required=True, type='list'),
+        )
+    )
+
+    p = module.params
+    zk_hosts = []
+    try:
+        for host in p['zk_group']:
+            hv = p['hostvars'][host]
+            if hv.get('ansible_default_ipv6'):
+                address=hv['ansible_default_ipv6']['address']
+            else:
+                if 'ansible_default_ipv4' not in hv:
+                    module.fail_json(
+                        msg="No network information facts found",
+                        changed=False)
+                address=hv['ansible_default_ipv4']['address']
+            zk_hosts.append(dict(
+                host=address,
+                port=2181
+            ))
+        module.exit_json(hosts=zk_hosts, changed=True)
+    except Exception as e:
+        module.fail_json(msg=str(e), changed=True)
+
+if __name__ == '__main__':
+    main()

playbooks/roles/nodepool-base/tasks/main.yaml

@@ -13,11 +13,6 @@
     shell: /bin/bash
     uid: '{{ nodepool_base_nodepool_uid }}'
 
-- name: Install zookeeper
-  include_role:
-    name: install-zookeeper
-  when: nodepool_base_install_zookeeper
-
 - name: Sync project-config
   include_role:
     name: sync-project-config
@@ -35,10 +30,33 @@
     path: /opt/project-config/nodepool/{{ inventory_hostname }}.yaml
   register: host_config_file
 
-- name: Set config file symlink
-  file:
-    state: link
-    src: '{{ host_config_file.stat.exists | ternary(host_config_file.stat.path, "/opt/project-config/nodepool/nodepool.yaml") }}'
+- name: Load host specific config file
+  slurp:
+    path: '{{ host_config_file.stat.exists | ternary(host_config_file.stat.path, "/opt/project-config/nodepool/nodepool.yaml") }}'
+  register: nodepool_config_content
+
+- name: Parse nodepool config
+  set_fact:
+    nodepool_config: "{{ nodepool_config_content.content | b64decode | from_yaml }}"
+
+# Have to run service-zookeeper before service-nodepool
+# because we need top populate the fact cache.
+- name: Get zk config
+  make_nodepool_zk_hosts:
+    hostvars: "{{ hostvars }}"
+    zk_group: "{{ groups['zookeeper'] }}"
+  register: zk_hosts
+
+- name: Overwrite zookeeper-servers
+  vars:
+    new_config:
+      zookeeper-servers: '{{ zk_hosts.hosts }}'
+  set_fact:
+    nodepool_config: "{{ nodepool_config | combine(new_config) }}"
+
+- name: Write nodepool config
+  copy:
+    content: "{{ nodepool_config | to_nice_yaml }}"
     dest: /etc/nodepool/nodepool.yaml
 
 - name: Symlink in elements from project-config repo

playbooks/zuul/run-base.yaml

@@ -72,7 +72,6 @@
         - host_vars/mirror-update01.opendev.org.yaml
         - host_vars/backup-test01.opendev.org.yaml
         - host_vars/backup-test02.opendev.org.yaml
-        - host_vars/nb01-test.opendev.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
 

playbooks/zuul/templates/group_vars/nodepool.yaml.j2

@@ -1,3 +1,33 @@
+zuul_worker_ssh_public_key_contents: |
+  ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC85765qHyZ0QX78FDEOvcnqjR/kzsFLwvSRXLvdKQ4G6798kBKWr418qQmV8pZY/0JAEKBbvjCgiGOt8T1FnEizG09fRFc+ZgZoS9hB7M7FYAQA2nFH3xSnDgJYJl2VlNReBVO0VqJkThERpGVuYIw3gOaVcer7zdfxQYjrQhHq4b0KutwJL3erTy9msBus6DpxhTYtjS1SQhoMlMgCJ4eybtH7iIamyvGS2beYU1J0mLJU9XDasLzQrL+AlvYasUballEshuuQ4OyI4Yu7jGziJpwrgDGYaNVmixycv9cAR+PUo2GBEg+vbU98nXQRPYRZgdMvCg7zIM6A4YjQgQb
+zuul_worker_ssh_private_key_contents: |
+  -----BEGIN OPENSSH PRIVATE KEY-----
+  b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn
+  NhAAAAAwEAAQAAAQEAvOe+uah8mdEF+/BQxDr3J6o0f5M7BS8L0kVy73SkOBuu/fJASlq+
+  NfKkJlfKWWP9CQBCgW74woIhjrfE9RZxIsxtPX0RXPmYGaEvYQezOxWAEANpxR98Upw4CW
+  CZdlZTUXgVTtFaiZE4REaRlbmCMN4DmlXHq+83X8UGI60IR6uG9CrrcCS93q08vZrAbrOg
+  6cYU2LY0tUkIaDJTIAieHsm7R+4iGpsrxktm3mFNSdJiyVPVw2rC80Ky/gJb2GrFG2pZRL
+  IbrkODsiOGLu4xs4iacK4AxmGjVZoscnL/XAEfj1KNhgRIPr21PfJ10ET2EWYHTLwoO8yD
+  OgOGI0IEGwAAA9iqH3otqh96LQAAAAdzc2gtcnNhAAABAQC85765qHyZ0QX78FDEOvcnqj
+  R/kzsFLwvSRXLvdKQ4G6798kBKWr418qQmV8pZY/0JAEKBbvjCgiGOt8T1FnEizG09fRFc
+  +ZgZoS9hB7M7FYAQA2nFH3xSnDgJYJl2VlNReBVO0VqJkThERpGVuYIw3gOaVcer7zdfxQ
+  YjrQhHq4b0KutwJL3erTy9msBus6DpxhTYtjS1SQhoMlMgCJ4eybtH7iIamyvGS2beYU1J
+  0mLJU9XDasLzQrL+AlvYasUballEshuuQ4OyI4Yu7jGziJpwrgDGYaNVmixycv9cAR+PUo
+  2GBEg+vbU98nXQRPYRZgdMvCg7zIM6A4YjQgQbAAAAAwEAAQAAAQBcPA782Y5VECEK3VCi
+  L2REAq/1Zm7X3hu4hF5VGq+gEaxEpAbOBzZ+YsXRTuYm9VI0FeNjDUr6tc0qwQGy5U1CP2
+  xLLIjQy7F/OhtHFhpmC95mczQuomvybgJSWc80X3v4+Ff3DvNU3goKFCa7Y0N1FsHNlnqS
+  sY9o/Gs9+htj2j4G5YJvLKGQgfuKSAQmjcqc550wJKP8n0JjlxI4KBlU9Blt4gOAzzgLOc
+  +DJupAhP4ZcNKmuNvjtq4bOBXj9rCJbMopex3m2P8UxlT83ogFtZnFN4N6pkmBDqPPAA7e
+  8dUSG75wI++5dQ/LS9Upz6MjtO7WAQxFUw1RZJ0pFmLZAAAAgQDgLoJmp5yGjqZ9QON/En
+  GWtpibIcbZeBVpVPmHNo01GqfnT9ohoOTd5ITqcZ1HqUXIzlr0+xHE5O5GGG9XlHh1nsiJ
+  dzLLaWeaEU+B3u+bHqbPCs23qybUDPSz+uVlsIXh0r0svKYUW3h9niMpp2Q1Kys0O8mGef
+  u3nCbBDkO8rQAAAIEA7PJ/o0xbEU7NdWouOuGahO96sxTyWFKQv0qWNftCoac2f0VymXjf
+  miSz0kIPxI8qpsVcRCs2TmUuKCg2AXNkBt0cYbi5ONj5MATb+buXx9keKr7ZbSiu4uQpPo
+  7L8eq7A6tTSj/YmTf4YyPa1HSZTT6Y80TDjnzffDApUn99Gn0AAACBAMwYUECXEBgoAKSs
+  t99hpCTwv5On/VVpS0pE1uCQXb89okWWDTcWLbWpKVjfgEziPxoqvrG4HSM+buYJ7zF9LK
+  kKs0kTPPsrkufb/VkksOGVP6WqcaHIfEbcTqxapjrBgLPhPQ9zDI5JSVziJkh4XGzmGNw6
+  2oaCng9UyII8j8R3AAAAH21vcmRyZWRATWFjQm9vay1BaXIubG9jYWxkb21haW4BAgM=
+  -----END OPENSSH PRIVATE KEY-----
 # Necessary for fake clouds.yaml to be written
 nodepool_rackspace_username: user
 nodepool_rackspace_password: password

playbooks/zuul/templates/host_vars/nb01-test.opendev.org.yaml.j2

@@ -1,5 +0,0 @@
-letsencrypt_certs:
-  nb01-test-main:
-    - nb01-test.opendev.org
-
-nodepool_base_install_zookeeper: True

testinfra/test_nodepool.py

@@ -15,7 +15,7 @@
 import pytest
 
 testinfra_hosts = ['nl01.openstack.org', 'nb01.openstack.org',
-                   'nb01-test.opendev.org']
+                   'nb04.opendev.org']
 
 
 def test_clouds_yaml(host):
@@ -38,20 +38,20 @@ def test_kube_config(host):
     assert b'nodepool_k8s_key' in kubeconfig.content
 
 def test_builder_container_running(host):
-    if host.backend.get_hostname() != 'nb01-test.opendev.org':
+    if host.backend.get_hostname() != 'nb04.opendev.org':
         pytest.skip()
 
     cmd = host.run("docker ps -a --format '{{ .Names }}'")
     assert 'nodepool-builder-compose_nodepool-builder_1' in cmd.stdout
 
 def test_builder_webserver_running(host):
-    if host.backend.get_hostname() != 'nb01-test.opendev.org':
+    if host.backend.get_hostname() != 'nb04.opendev.org':
         pytest.skip()
 
     apache = host.service('apache2')
     assert apache.is_running
 
     cmd = host.run('curl --insecure '
-                   '--resolve nb01-test.opendev.org:443:127.0.0.1 '
-                   'https://nb01-test.opendev.org/')
+                   '--resolve nb04.opendev.org:443:127.0.0.1 '
+                   'https://nb04.opendev.org/')
     assert 'Index of /' in cmd.stdout