Add a standalone zuul db server

Change-Id: Ibb260f820dbc1d9d6ca523ff3903134612cb003e

inventory/service/group_vars/zuul-db.yaml

@@ -0,0 +1,4 @@
+mariadb_username: zuul
+mariadb_dbname: zuul
+iptables_extra_allowed_groups:
+  - {'protocol': 'tcp', 'port': '3306', 'group': 'zuul'}

inventory/service/groups.yaml

@@ -177,6 +177,8 @@ groups:
     - translate[0-9]*.open*.org
   zookeeper:
     - zk[0-9]*.open*.org
+  zuul-db:
+    - zuul-db[0-9]*.opendev.org
   zuul-lb:
     - zuul-lb[0-9]*.opendev.org
   zuul:

playbooks/roles/mariadb/README.rst

@@ -0,0 +1,22 @@
+Run MariaDB
+
+This role deploys a standalone MariaDB using docker-compose.
+Variables below configure MariaDB connection details.
+
+**Role Variables**
+
+.. zuul:rolevar:: mariadb_dbname
+
+   The database to create.
+
+.. zuul:rolevar:: mariadb_username
+
+   The MariaDB user to make and connect with.
+
+.. zuul:rolevar:: mariadb_password
+
+   The password to set for ``mariadb_username``
+
+.. zuul:rolevar:: mariadb_root_password
+
+   The password to set for the root mariadb user.

playbooks/roles/mariadb/defaults/main.yaml

@@ -0,0 +1 @@
+mariadb_run_compose_up: false

playbooks/roles/mariadb/files/my.cnf

@@ -0,0 +1,15 @@
+[client]
+# Default is Latin1, if you need UTF-8 set this (also in server section)
+default-character-set = utf8mb4
+
+[mysqld]
+wait_timeout = 28800
+#
+# * Character sets
+#
+# Default is Latin1, if you need UTF-8 set all this (also in client section)
+#
+character-set-server  = utf8mb4
+collation-server      = utf8mb4_bin
+character_set_server   = utf8mb4
+collation_server       = utf8mb4_bin

playbooks/roles/mariadb/tasks/main.yaml

@@ -0,0 +1,29 @@
+- name: Ensure /etc/mariadb-compose directory
+  file:
+    state: directory
+    path: /etc/mariadb-compose
+    mode: 0755
+
+- name: Put docker-compose file in place
+  template:
+    src: docker-compose.yaml.j2
+    dest: /etc/mariadb-compose/docker-compose.yaml
+    mode: 0644
+
+- name: Ensure database volume exists
+  file:
+    state: directory
+    path: /var/mariadb/db
+
+- name: Ensure config directory exists
+  file:
+    state: directory
+    path: /var/mariadb/etc
+
+- name: Install database config settings
+  copy:
+    src: my.cnf
+    dest: /var/mariadb/etc/my.cnf
+
+- name: Start mariadb
+  include_tasks: start.yaml

playbooks/roles/mariadb/tasks/start.yaml

@@ -0,0 +1,5 @@
+- name: Run docker-compose up
+  shell:
+    cmd: docker-compose up -d
+    chdir: /etc/mariadb-compose/
+  when: mariadb_run_compose_up | bool

playbooks/roles/mariadb/templates/docker-compose.yaml.j2

@@ -0,0 +1,18 @@
+version: '2'
+services:
+  mariadb:
+    image: docker.io/library/mariadb:10.11
+    network_mode: host
+    environment:
+      MYSQL_ROOT_PASSWORD: "{{ mariadb_root_password }}"
+      MYSQL_DATABASE: "{{ mariadb_dbname }}"
+      MYSQL_USER: "{{ mariadb_username }}"
+      MYSQL_PASSWORD: "{{ mariadb_password }}"
+      MARIADB_AUTO_UPGRADE: 1
+    volumes:
+      - /var/mariadb/db:/var/lib/mysql
+      - /var/mariadb/etc:/etc/mysql/conf.d
+    logging:
+      driver: syslog
+      options:
+        tag: "docker-mariadb"

playbooks/service-zuul-db.yaml

@@ -0,0 +1,6 @@
+- hosts: "zuul-db:!disabled"
+  name: "Configure zuul db servers"
+  roles:
+    - iptables
+    - install-docker
+    - mariadb

playbooks/zuul/run-base.yaml

@@ -132,6 +132,7 @@
         - group_vars/registry.yaml
         - group_vars/control-plane-clouds.yaml
         - group_vars/afs-client.yaml
+        - group_vars/zuul-db.yaml
         - group_vars/zuul-lb.yaml
         - group_vars/zuul.yaml
         - group_vars/zuul-executor.yaml

playbooks/zuul/templates/group_vars/zuul-db.yaml.j2

@@ -0,0 +1,4 @@
+mariadb_password: testuserpassword
+mariadb_root_password: testrootpassword
+mariadb_run_compose_up: true
+

testinfra/test_zuul_db.py

@@ -0,0 +1,23 @@
+# Copyright 2018 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import urllib.parse
+
+testinfra_hosts = ['zuul-db99.opendev.org']
+
+
+def test_mariadb_logs(host):
+    mariadb_log_file = host.file('/var/log/containers/docker-mariadb.log')
+    assert mariadb_log_file.exists
+    assert mariadb_log_file.contains('mariadbd: ready for connections')

zuul.d/infra-prod.yaml

@@ -394,6 +394,20 @@
       - roles/kerberos-client/
       - roles/openafs-client/
 
+- job:
+    name: infra-prod-service-zuul-db
+    parent: infra-prod-service-base
+    description: Run service-zuul-db.yaml playbook.
+    vars:
+      playbook_name: service-zuul-db.yaml
+    files:
+      - inventory/base
+      - playbooks/service-zuul-db.yaml
+      - inventory/service/group_vars/zuul-db.yaml
+      - playbooks/roles/iptables/
+      - playbooks/roles/install-docker/
+      - playbooks/roles/mariadb/
+
 - job:
     name: infra-prod-service-zuul-lb
     parent: infra-prod-service-base

zuul.d/project.yaml

@@ -557,6 +557,7 @@
               # should reconfigure after any project updates
               - name: infra-prod-manage-projects
                 soft: true
+        - infra-prod-service-zuul-db
         - infra-prod-service-zuul-lb: &infra-prod-service-zuul-lb
             dependencies:
               - name: system-config-promote-image-haproxy-statsd
@@ -653,6 +654,7 @@
         - infra-prod-service-tracing: *infra-prod-service-tracing
         - infra-prod-service-zookeeper: *infra-prod-service-zookeeper
         - infra-prod-service-zuul: *infra-prod-service-zuul
+        - infra-prod-service-zuul-db
         - infra-prod-service-zuul-lb: *infra-prod-service-zuul-lb
         - infra-prod-service-zuul-preview: *infra-prod-service-zuul-preview
         - infra-prod-run-accessbot: *infra-prod-run-accessbot

zuul.d/system-config-run.yaml

@@ -966,6 +966,8 @@
           label: ubuntu-focal
         - name: zuul-lb01.opendev.org
           label: ubuntu-focal
+        - name: zuul-db99.opendev.org
+          label: ubuntu-jammy
       groups:
         - <<: *bastion_group
     required-projects:
@@ -975,6 +977,7 @@
       run_playbooks:
         - playbooks/letsencrypt.yaml
         - playbooks/service-zookeeper.yaml
+        - playbooks/service-zuul-db.yaml
         - playbooks/service-zuul.yaml
         - playbooks/service-zuul-lb.yaml
         # Test our ad hoc restart playbook works
@@ -1006,12 +1009,15 @@
       - playbooks/bootstrap-bridge.yaml
       - playbooks/service-zookeeper.yaml
       - playbooks/service-zuul.yaml
+      - playbooks/service-zuul-db.yaml
       - playbooks/service-zuul-lb.yaml
       - inventory/service/group_vars/zuul
+      - inventory/service/group_vars/zuul-db.yaml
       - inventory/service/group_vars/zuul-lb.yaml
       - inventory/service/group_vars/zookeeper.yaml
       - inventory/service/host_vars/zk\d+
       - inventory/service/host_vars/zuul02.opendev.org
+      - playbooks/roles/mariadb/
       - playbooks/roles/zookeeper/
       - playbooks/roles/install-apt-repo/
       - playbooks/roles/zuul.*
@@ -1024,6 +1030,7 @@
       - testinfra/test_zuul_executor.py
       - testinfra/test_zuul_scheduler.py
       - testinfra/test_zuul_merger.py
+      - testinfra/test_zuul_db.py
       - testinfra/util.py
 
 - job: