From 8d7075b02faa705abb1ff6c91d0086c87227247c Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Wed, 29 Apr 2020 15:26:32 -0500 Subject: [PATCH] Run zookeeper cluster in nodepool jobs Rather than running a local zookeeper, just run a real zookeeper. Also, get rid of nb01-test and just use nb04 - what could possibly go wrong? Dynamically write zookeeper host information to nodepool.yaml So that we can run an actual zk using the new zk role on hosts in ansible inventory, we need to write out the ip addresses of the hosts that we build in zuul. This means having the info baked in to the file in project-config isn't going to work. We can do this in prod too, it shouldn't hurt anything. Increase timeout for run-service-nodepool We need to fix the playbook, but we'll do that after we get the puppet gone. Change-Id: Ib01d461ae2c5cec3c31ec5105a41b1a99ff9d84a --- .zuul.yaml | 17 ++++++- inventory/groups.yaml | 1 - playbooks/roles/install-zookeeper/README.rst | 4 -- .../roles/install-zookeeper/tasks/main.yaml | 11 ---- .../roles/nodepool-base/library/__init__.py | 0 .../library/make_nodepool_zk_hosts.py | 50 +++++++++++++++++++ playbooks/roles/nodepool-base/tasks/main.yaml | 36 +++++++++---- playbooks/zuul/run-base.yaml | 1 - .../templates/group_vars/nodepool.yaml.j2 | 30 +++++++++++ .../host_vars/nb01-test.opendev.org.yaml.j2 | 5 -- testinfra/test_nodepool.py | 10 ++-- 11 files changed, 127 insertions(+), 38 deletions(-) delete mode 100644 playbooks/roles/install-zookeeper/README.rst delete mode 100644 playbooks/roles/install-zookeeper/tasks/main.yaml create mode 100644 playbooks/roles/nodepool-base/library/__init__.py create mode 100644 playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py delete mode 100644 playbooks/zuul/templates/host_vars/nb01-test.opendev.org.yaml.j2 diff --git a/.zuul.yaml b/.zuul.yaml index bcb0e29802..2f7e5aada9 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -1224,22 +1224,36 @@ parent: system-config-run description: | Run the playbook for nodepool. + timeout: 3600 nodeset: nodes: - name: bridge.openstack.org label: ubuntu-bionic + - name: zk01.opendev.org + label: ubuntu-bionic - name: nl01.openstack.org label: ubuntu-xenial - name: nb01.openstack.org label: ubuntu-xenial - - name: nb01-test.opendev.org + - name: nb04.opendev.org label: ubuntu-bionic required-projects: - openstack/project-config - opendev/system-config + host-vars: + nl01.openstack.org: + host_copy_output: + '/etc/nodepool/nodepool.yaml': logs + nb01.openstack.org: + host_copy_output: + '/etc/nodepool/nodepool.yaml': logs + nb04.opendev.org: + host_copy_output: + '/etc/nodepool/nodepool.yaml': logs vars: run_playbooks: - playbooks/service-letsencrypt.yaml + - playbooks/service-zookeeper.yaml - playbooks/service-nodepool.yaml - playbooks/remote_puppet_else.yaml files: @@ -2217,7 +2231,6 @@ - playbooks/roles/configure-kubectl/ - playbooks/roles/configure-openstacksdk/ - playbooks/roles/install-docker/ - - playbooks/roles/install-zookeeper/ - playbooks/roles/nodepool- - playbooks/templates/clouds/nodepool_ diff --git a/inventory/groups.yaml b/inventory/groups.yaml index 5828fda144..76cca8d14c 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -102,7 +102,6 @@ groups: - nb[0-9]*.opendev.org nodepool-launcher: - nl[0-9]*.openstack.org - nodepool-launcher_opendev: - nl[0-8]*.opendev.org ns: - ns[0-9]*.open*.org diff --git a/playbooks/roles/install-zookeeper/README.rst b/playbooks/roles/install-zookeeper/README.rst deleted file mode 100644 index 32679cb981..0000000000 --- a/playbooks/roles/install-zookeeper/README.rst +++ /dev/null @@ -1,4 +0,0 @@ -An ansible role to install Zookeeper - -**Role Variables** - diff --git a/playbooks/roles/install-zookeeper/tasks/main.yaml b/playbooks/roles/install-zookeeper/tasks/main.yaml deleted file mode 100644 index 8373973487..0000000000 --- a/playbooks/roles/install-zookeeper/tasks/main.yaml +++ /dev/null @@ -1,11 +0,0 @@ -- name: Install zookeeper - package: - name: - - zookeeper - - zookeeperd - state: present - -- name: Start zookeeper service - service: - name: zookeeper - state: started \ No newline at end of file diff --git a/playbooks/roles/nodepool-base/library/__init__.py b/playbooks/roles/nodepool-base/library/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py b/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py new file mode 100644 index 0000000000..6e0ff4681e --- /dev/null +++ b/playbooks/roles/nodepool-base/library/make_nodepool_zk_hosts.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# +# Copyright 2020 Red Hat, Inc +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from ansible.module_utils.basic import AnsibleModule + + +def main(): + module = AnsibleModule( + argument_spec=dict( + hostvars=dict(required=True, type='dict'), + zk_group=dict(required=True, type='list'), + ) + ) + + p = module.params + zk_hosts = [] + try: + for host in p['zk_group']: + hv = p['hostvars'][host] + if hv.get('ansible_default_ipv6'): + address=hv['ansible_default_ipv6']['address'] + else: + if 'ansible_default_ipv4' not in hv: + module.fail_json( + msg="No network information facts found", + changed=False) + address=hv['ansible_default_ipv4']['address'] + zk_hosts.append(dict( + host=address, + port=2181 + )) + module.exit_json(hosts=zk_hosts, changed=True) + except Exception as e: + module.fail_json(msg=str(e), changed=True) + +if __name__ == '__main__': + main() diff --git a/playbooks/roles/nodepool-base/tasks/main.yaml b/playbooks/roles/nodepool-base/tasks/main.yaml index 229f409ec9..1db9045a09 100644 --- a/playbooks/roles/nodepool-base/tasks/main.yaml +++ b/playbooks/roles/nodepool-base/tasks/main.yaml @@ -13,11 +13,6 @@ shell: /bin/bash uid: '{{ nodepool_base_nodepool_uid }}' -- name: Install zookeeper - include_role: - name: install-zookeeper - when: nodepool_base_install_zookeeper - - name: Sync project-config include_role: name: sync-project-config @@ -35,10 +30,33 @@ path: /opt/project-config/nodepool/{{ inventory_hostname }}.yaml register: host_config_file -- name: Set config file symlink - file: - state: link - src: '{{ host_config_file.stat.exists | ternary(host_config_file.stat.path, "/opt/project-config/nodepool/nodepool.yaml") }}' +- name: Load host specific config file + slurp: + path: '{{ host_config_file.stat.exists | ternary(host_config_file.stat.path, "/opt/project-config/nodepool/nodepool.yaml") }}' + register: nodepool_config_content + +- name: Parse nodepool config + set_fact: + nodepool_config: "{{ nodepool_config_content.content | b64decode | from_yaml }}" + +# Have to run service-zookeeper before service-nodepool +# because we need top populate the fact cache. +- name: Get zk config + make_nodepool_zk_hosts: + hostvars: "{{ hostvars }}" + zk_group: "{{ groups['zookeeper'] }}" + register: zk_hosts + +- name: Overwrite zookeeper-servers + vars: + new_config: + zookeeper-servers: '{{ zk_hosts.hosts }}' + set_fact: + nodepool_config: "{{ nodepool_config | combine(new_config) }}" + +- name: Write nodepool config + copy: + content: "{{ nodepool_config | to_nice_yaml }}" dest: /etc/nodepool/nodepool.yaml - name: Symlink in elements from project-config repo diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 560835af08..25152374bd 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -72,7 +72,6 @@ - host_vars/mirror-update01.opendev.org.yaml - host_vars/backup-test01.opendev.org.yaml - host_vars/backup-test02.opendev.org.yaml - - host_vars/nb01-test.opendev.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' diff --git a/playbooks/zuul/templates/group_vars/nodepool.yaml.j2 b/playbooks/zuul/templates/group_vars/nodepool.yaml.j2 index 8a5e44997b..2c1f747942 100644 --- a/playbooks/zuul/templates/group_vars/nodepool.yaml.j2 +++ b/playbooks/zuul/templates/group_vars/nodepool.yaml.j2 @@ -1,3 +1,33 @@ +zuul_worker_ssh_public_key_contents: | + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQC85765qHyZ0QX78FDEOvcnqjR/kzsFLwvSRXLvdKQ4G6798kBKWr418qQmV8pZY/0JAEKBbvjCgiGOt8T1FnEizG09fRFc+ZgZoS9hB7M7FYAQA2nFH3xSnDgJYJl2VlNReBVO0VqJkThERpGVuYIw3gOaVcer7zdfxQYjrQhHq4b0KutwJL3erTy9msBus6DpxhTYtjS1SQhoMlMgCJ4eybtH7iIamyvGS2beYU1J0mLJU9XDasLzQrL+AlvYasUballEshuuQ4OyI4Yu7jGziJpwrgDGYaNVmixycv9cAR+PUo2GBEg+vbU98nXQRPYRZgdMvCg7zIM6A4YjQgQb +zuul_worker_ssh_private_key_contents: | + -----BEGIN OPENSSH PRIVATE KEY----- + b3BlbnNzaC1rZXktdjEAAAAABG5vbmUAAAAEbm9uZQAAAAAAAAABAAABFwAAAAdzc2gtcn + NhAAAAAwEAAQAAAQEAvOe+uah8mdEF+/BQxDr3J6o0f5M7BS8L0kVy73SkOBuu/fJASlq+ + NfKkJlfKWWP9CQBCgW74woIhjrfE9RZxIsxtPX0RXPmYGaEvYQezOxWAEANpxR98Upw4CW + CZdlZTUXgVTtFaiZE4REaRlbmCMN4DmlXHq+83X8UGI60IR6uG9CrrcCS93q08vZrAbrOg + 6cYU2LY0tUkIaDJTIAieHsm7R+4iGpsrxktm3mFNSdJiyVPVw2rC80Ky/gJb2GrFG2pZRL + IbrkODsiOGLu4xs4iacK4AxmGjVZoscnL/XAEfj1KNhgRIPr21PfJ10ET2EWYHTLwoO8yD + OgOGI0IEGwAAA9iqH3otqh96LQAAAAdzc2gtcnNhAAABAQC85765qHyZ0QX78FDEOvcnqj + R/kzsFLwvSRXLvdKQ4G6798kBKWr418qQmV8pZY/0JAEKBbvjCgiGOt8T1FnEizG09fRFc + +ZgZoS9hB7M7FYAQA2nFH3xSnDgJYJl2VlNReBVO0VqJkThERpGVuYIw3gOaVcer7zdfxQ + YjrQhHq4b0KutwJL3erTy9msBus6DpxhTYtjS1SQhoMlMgCJ4eybtH7iIamyvGS2beYU1J + 0mLJU9XDasLzQrL+AlvYasUballEshuuQ4OyI4Yu7jGziJpwrgDGYaNVmixycv9cAR+PUo + 2GBEg+vbU98nXQRPYRZgdMvCg7zIM6A4YjQgQbAAAAAwEAAQAAAQBcPA782Y5VECEK3VCi + L2REAq/1Zm7X3hu4hF5VGq+gEaxEpAbOBzZ+YsXRTuYm9VI0FeNjDUr6tc0qwQGy5U1CP2 + xLLIjQy7F/OhtHFhpmC95mczQuomvybgJSWc80X3v4+Ff3DvNU3goKFCa7Y0N1FsHNlnqS + sY9o/Gs9+htj2j4G5YJvLKGQgfuKSAQmjcqc550wJKP8n0JjlxI4KBlU9Blt4gOAzzgLOc + +DJupAhP4ZcNKmuNvjtq4bOBXj9rCJbMopex3m2P8UxlT83ogFtZnFN4N6pkmBDqPPAA7e + 8dUSG75wI++5dQ/LS9Upz6MjtO7WAQxFUw1RZJ0pFmLZAAAAgQDgLoJmp5yGjqZ9QON/En + GWtpibIcbZeBVpVPmHNo01GqfnT9ohoOTd5ITqcZ1HqUXIzlr0+xHE5O5GGG9XlHh1nsiJ + dzLLaWeaEU+B3u+bHqbPCs23qybUDPSz+uVlsIXh0r0svKYUW3h9niMpp2Q1Kys0O8mGef + u3nCbBDkO8rQAAAIEA7PJ/o0xbEU7NdWouOuGahO96sxTyWFKQv0qWNftCoac2f0VymXjf + miSz0kIPxI8qpsVcRCs2TmUuKCg2AXNkBt0cYbi5ONj5MATb+buXx9keKr7ZbSiu4uQpPo + 7L8eq7A6tTSj/YmTf4YyPa1HSZTT6Y80TDjnzffDApUn99Gn0AAACBAMwYUECXEBgoAKSs + t99hpCTwv5On/VVpS0pE1uCQXb89okWWDTcWLbWpKVjfgEziPxoqvrG4HSM+buYJ7zF9LK + kKs0kTPPsrkufb/VkksOGVP6WqcaHIfEbcTqxapjrBgLPhPQ9zDI5JSVziJkh4XGzmGNw6 + 2oaCng9UyII8j8R3AAAAH21vcmRyZWRATWFjQm9vay1BaXIubG9jYWxkb21haW4BAgM= + -----END OPENSSH PRIVATE KEY----- # Necessary for fake clouds.yaml to be written nodepool_rackspace_username: user nodepool_rackspace_password: password diff --git a/playbooks/zuul/templates/host_vars/nb01-test.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/nb01-test.opendev.org.yaml.j2 deleted file mode 100644 index 02614e7724..0000000000 --- a/playbooks/zuul/templates/host_vars/nb01-test.opendev.org.yaml.j2 +++ /dev/null @@ -1,5 +0,0 @@ -letsencrypt_certs: - nb01-test-main: - - nb01-test.opendev.org - -nodepool_base_install_zookeeper: True \ No newline at end of file diff --git a/testinfra/test_nodepool.py b/testinfra/test_nodepool.py index 528eabf606..d07579ffdc 100644 --- a/testinfra/test_nodepool.py +++ b/testinfra/test_nodepool.py @@ -15,7 +15,7 @@ import pytest testinfra_hosts = ['nl01.openstack.org', 'nb01.openstack.org', - 'nb01-test.opendev.org'] + 'nb04.opendev.org'] def test_clouds_yaml(host): @@ -38,20 +38,20 @@ def test_kube_config(host): assert b'nodepool_k8s_key' in kubeconfig.content def test_builder_container_running(host): - if host.backend.get_hostname() != 'nb01-test.opendev.org': + if host.backend.get_hostname() != 'nb04.opendev.org': pytest.skip() cmd = host.run("docker ps -a --format '{{ .Names }}'") assert 'nodepool-builder-compose_nodepool-builder_1' in cmd.stdout def test_builder_webserver_running(host): - if host.backend.get_hostname() != 'nb01-test.opendev.org': + if host.backend.get_hostname() != 'nb04.opendev.org': pytest.skip() apache = host.service('apache2') assert apache.is_running cmd = host.run('curl --insecure ' - '--resolve nb01-test.opendev.org:443:127.0.0.1 ' - 'https://nb01-test.opendev.org/') + '--resolve nb04.opendev.org:443:127.0.0.1 ' + 'https://nb04.opendev.org/') assert 'Index of /' in cmd.stdout