From 2a9553ef25c6d797c71c38da69be5ef823d848c4 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Thu, 10 Feb 2022 09:28:46 -0800 Subject: [PATCH] Add Zuul load balancer This adds a load balancer for zuul-web and fingergw. Change-Id: Id5aa01151f64f3c85e1532ad66999ef9471c5896 --- inventory/service/group_vars/zuul-lb.yaml | 23 ++++++++++++ inventory/service/groups.yaml | 2 ++ playbooks/roles/zuul-lb/README.rst | 9 +++++ playbooks/roles/zuul-lb/tasks/main.yaml | 5 +++ .../zuul-lb/templates/zuul-haproxy.cfg.j2 | 36 +++++++++++++++++++ playbooks/service-zuul-lb.yaml | 6 ++++ playbooks/zuul/run-base.yaml | 1 + .../zuul/templates/group_vars/zuul-lb.yaml.j2 | 19 ++++++++++ testinfra/test_zuul_lb.py | 34 ++++++++++++++++++ zuul.d/infra-prod.yaml | 15 ++++++++ zuul.d/project.yaml | 5 +++ zuul.d/system-config-run.yaml | 15 ++++++-- 12 files changed, 167 insertions(+), 3 deletions(-) create mode 100644 inventory/service/group_vars/zuul-lb.yaml create mode 100644 playbooks/roles/zuul-lb/README.rst create mode 100644 playbooks/roles/zuul-lb/tasks/main.yaml create mode 100644 playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2 create mode 100644 playbooks/service-zuul-lb.yaml create mode 100644 playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 create mode 100644 testinfra/test_zuul_lb.py diff --git a/inventory/service/group_vars/zuul-lb.yaml b/inventory/service/group_vars/zuul-lb.yaml new file mode 100644 index 0000000000..444b6113fa --- /dev/null +++ b/inventory/service/group_vars/zuul-lb.yaml @@ -0,0 +1,23 @@ +zuul_lb_listeners: + - name: balance_zuul_http + bind: + - ':::80' + servers: + - name: 'zuul02.opendev.org' + address: '104.130.246.31:80' + - name: balance_zuul_https + bind: + - ':::443' + servers: + - name: 'zuul02.opendev.org' + address: '104.130.246.31:443' + - name: balance_zuul_finger + bind: + - ':::79' + servers: + - name: 'zuul02.opendev.org' + address: '104.130.246.31:79' +iptables_extra_public_tcp_ports: + - 443 + - 80 + - 79 diff --git a/inventory/service/groups.yaml b/inventory/service/groups.yaml index 0650d3e02a..830359b76e 100644 --- a/inventory/service/groups.yaml +++ b/inventory/service/groups.yaml @@ -211,6 +211,8 @@ groups: - wiki-dev[0-9]*.openstack.org zookeeper: - zk[0-9]*.open*.org + zuul-lb: + - zuul-lb[0-9]*.opendev.org zuul: - ze[0-9]*.opendev.org - zm[0-9]*.opendev.org diff --git a/playbooks/roles/zuul-lb/README.rst b/playbooks/roles/zuul-lb/README.rst new file mode 100644 index 0000000000..6f5cd4bc16 --- /dev/null +++ b/playbooks/roles/zuul-lb/README.rst @@ -0,0 +1,9 @@ +Install the zuul-lb services + +This configures haproxy + +**Role Variables** + +.. zuul:rolevar:: zuul_lb_listeners + + The backends to configure diff --git a/playbooks/roles/zuul-lb/tasks/main.yaml b/playbooks/roles/zuul-lb/tasks/main.yaml new file mode 100644 index 0000000000..9835e05311 --- /dev/null +++ b/playbooks/roles/zuul-lb/tasks/main.yaml @@ -0,0 +1,5 @@ +- name: Install haproxy with zuul config + include_role: + name: haproxy + vars: + haproxy_config_template: zuul-haproxy.cfg.j2 diff --git a/playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2 b/playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2 new file mode 100644 index 0000000000..924ff93198 --- /dev/null +++ b/playbooks/roles/zuul-lb/templates/zuul-haproxy.cfg.j2 @@ -0,0 +1,36 @@ +global + uid 1000 + gid 1000 + log /dev/log local0 + maxconn 4000 + pidfile /var/haproxy/run/haproxy.pid + stats socket /var/haproxy/run/stats uid 1000 gid 1000 mode 0600 level admin + +defaults + log-format "%ci:%cp [%t] %ft [%bi]:%bp %b/%s %Tw/%Tc/%Tt %B %ts %ac/%fc/%bc/%sc/%rc %sq/%bq" + log global + maxconn 8000 + option redispatch + retries 3 + stats enable + timeout http-request 10s + timeout queue 1m + timeout connect 10s + timeout client 2m + timeout server 2m + timeout check 10s + +{% for listener in zuul_lb_listeners %} +listen {{ listener.name }} + {% for bind in listener.bind %} + bind {{ bind }} + {% endfor %} + mode tcp + balance source + option tcp-check + + {% for server in listener.servers %} + server {{ server.name }} {{ server.address }} check + {% endfor %} + +{% endfor %} diff --git a/playbooks/service-zuul-lb.yaml b/playbooks/service-zuul-lb.yaml new file mode 100644 index 0000000000..105d068e28 --- /dev/null +++ b/playbooks/service-zuul-lb.yaml @@ -0,0 +1,6 @@ +- hosts: "zuul-lb:!disabled" + name: "Base: configure zuul load balancer" + roles: + - iptables + - install-docker + - zuul-lb diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index bd6fdfcf49..66a1495baa 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -68,6 +68,7 @@ - group_vars/registry.yaml - group_vars/control-plane-clouds.yaml - group_vars/afs-client.yaml + - group_vars/zuul-lb.yaml - group_vars/zuul.yaml - group_vars/zuul-executor.yaml - group_vars/zuul-merger.yaml diff --git a/playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 b/playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 new file mode 100644 index 0000000000..c06a1c99d8 --- /dev/null +++ b/playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 @@ -0,0 +1,19 @@ +zuul_lb_listeners: + - name: balance_zuul_http + bind: + - ":::80" + servers: + - name: "zuul02.opendev.org" + address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:80" + - name: balance_zuul_https + bind: + - ":::443" + servers: + - name: "zuul02.opendev.org" + address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:443" + - name: balance_zuul_finger + bind: + - ":::79" + servers: + - name: "zuul02.opendev.org" + address: "{{ (hostvars['zuul02.opendev.org'] | default({})).get('nodepool', {}).get('public_ipv4', '') }}:79" diff --git a/testinfra/test_zuul_lb.py b/testinfra/test_zuul_lb.py new file mode 100644 index 0000000000..a2236cce6c --- /dev/null +++ b/testinfra/test_zuul_lb.py @@ -0,0 +1,34 @@ +# Copyright 2018 Red Hat, Inc. +# Copyright 2022 Acme Gating, LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json + + +testinfra_hosts = ['zuul-lb01.opendev.org'] + + +def test_zuul_listening(host): + zuul_https = host.socket("tcp://0.0.0.0:443") + assert zuul_https.is_listening + zuul_http = host.socket("tcp://0.0.0.0:80") + assert zuul_http.is_listening + zuul_finger = host.socket("tcp://0.0.0.0:79") + assert zuul_finger.is_listening + +def test_haproxy_statsd_running(host): + cmd = host.run("docker inspect haproxy-docker_haproxy-statsd_1") + out = json.loads(cmd.stdout) + assert out[0]["State"]["Status"] == "running" + assert out[0]["RestartCount"] == 0 diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index 89a7ec4fcd..50d3cefd85 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -386,6 +386,21 @@ - roles/kerberos-client/ - roles/openafs-client/ +- job: + name: infra-prod-service-zuul-lb + parent: infra-prod-service-base + description: Run service-zuul-lb.yaml playbook. + vars: + playbook_name: service-zuul-lb.yaml + files: + - inventory/base + - playbooks/service-zuul-lb.yaml + - inventory/service/group_vars/zuul-lb.yaml + - playbooks/roles/pip3/ + - playbooks/roles/iptables/ + - playbooks/roles/install-docker/ + - playbooks/roles/haproxy/ + - job: name: infra-prod-service-review parent: infra-prod-service-base diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index fb10a1453c..fec6a63964 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -542,6 +542,10 @@ # should reconfigure after any project updates - name: infra-prod-manage-projects soft: true + - infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb + dependencies: + - name: system-config-promote-image-haproxy-statsd + soft: true - infra-prod-service-zuul-preview: &infra-prod-service-zuul-preview dependencies: - name: infra-prod-letsencrypt @@ -632,6 +636,7 @@ - infra-prod-service-review: *infra-prod-service-review - infra-prod-service-zookeeper: *infra-prod-service-zookeeper - infra-prod-service-zuul: *infra-prod-service-zuul + - infra-prod-service-zuul-lb: *infra-prod-service-zuul-lb - infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview - infra-prod-run-accessbot: *infra-prod-run-accessbot - infra-prod-manage-projects: *infra-prod-manage-projects diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 6cb72c470a..219996f1fc 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -858,6 +858,8 @@ label: ubuntu-focal - name: zuul02.opendev.org label: ubuntu-focal + - name: zuul-lb01.opendev.org + label: ubuntu-focal required-projects: - openstack/project-config - opendev/system-config @@ -866,6 +868,7 @@ - playbooks/letsencrypt.yaml - playbooks/service-zookeeper.yaml - playbooks/service-zuul.yaml + - playbooks/service-zuul-lb.yaml # Test our ad hoc restart playbook works - playbooks/zuul_restart.yaml host-vars: @@ -887,19 +890,25 @@ bridge.openstack.org: host_copy_output: '/etc/hosts': logs + zuul-lb01.opendev.org: + host_copy_output: + '/var/haproxy/etc': logs files: - playbooks/bootstrap-bridge.yaml - playbooks/service-zookeeper.yaml - playbooks/service-zuul.yaml + - playbooks/service-zuul-lb.yaml - inventory/service/group_vars/zuul + - inventory/service/group_vars/zuul-lb.yaml - inventory/service/group_vars/zookeeper.yaml - inventory/service/host_vars/zk\d+ - inventory/service/host_vars/zuul02.opendev.org - playbooks/roles/zookeeper/ - - playbooks/roles/install-apt-repo - - playbooks/roles/zuul - - playbooks/zuul/templates/group_vars/zuul + - playbooks/roles/install-apt-repo/ + - playbooks/roles/zuul.* + - playbooks/zuul/templates/group_vars/zuul.* - playbooks/zuul/templates/group_vars/zookeeper.yaml + - playbooks/zuul/templates/group_vars/zuul-lb.yaml.j2 - playbooks/zuul/templates/host_vars/zk\d+ - playbooks/zuul/templates/host_vars/zuul02.opendev.org - playbooks/zuul_restart.yaml