Upgrade to Keycloak 23.0

This includes a switch from the "legacy" style Wildfly-based image
to a new setup using Quarkus.

Because Keycloak maintainers consider H2 databases as a test/dev
only option, there are no good migration and upgrade paths short of
export/import data. Go ahead and change our deployment model to rely
on a proper RDBMS, run locally from a container on the same server.

Change-Id: I01f8045563e9f6db6168b92c5a868b8095c0d97b

doc/source/keycloak.rst

@@ -20,13 +20,14 @@ At a Glance
   * :git_file:`playbooks/service-keycloak.yaml`
 :Projects:
   * https://www.keycloak.org/
-  * https://github.com/keycloak/keycloak-containers
+  * https://github.com/keycloak/keycloak
+  * https://github.com/keycloak/keycloak/tree/main/quarkus/container
 :Bugs:
   * https://storyboard.openstack.org/#!/project/748
-  * https://issues.jboss.org/browse/KEYCLOAK
+  * https://github.com/keycloak/keycloak/issues
 
 Overview
 ========
 
-Apache is configured as a reverse proxy and there is an internal H2
-database stored at ``/var/keycloak/data``.
+Apache is configured as a reverse proxy to ``[::1]:8080`` and there is
+also a separate MariaDB database listening on ``[::1]:3306``.

inventory/service/group_vars/keycloak.yaml

@@ -1,6 +1,6 @@
 letsencrypt_certs:
-  keycloak01-opendev-org-main:
+  keycloak-opendev-org-main:
     # List the service name first since that determines the filename
     # and is referenced in the apache config.
     - keycloak.opendev.org
-    - keycloak01.opendev.org
+    - "{{ inventory_hostname }}"

playbooks/roles/keycloak/files/99-bind-address.cnf

@@ -0,0 +1,3 @@
+[mysqld]
+# Only listen on the loopback address, for added safety
+bind-address=::1

playbooks/roles/keycloak/handlers/main.yaml

@@ -1,4 +1,12 @@
-- name: keycloak Reload apache2
+- name: keycloak restart apache2
+  service:
+    name: apache2
+    state: restarted
+
+- name: keycloak reload apache2
   service:
     name: apache2
     state: reloaded
+
+- name: keycloak restart containers
+  include_tasks: roles/keycloak/handlers/restart_keycloak.yaml

playbooks/roles/keycloak/handlers/restart_keycloak.yaml

@@ -0,0 +1,15 @@
+- name: keycloak check for running containers
+  command: pgrep -f quarkus
+  ignore_errors: yes
+  register: quarkus_pids
+
+- name: keycloak restart containers if running
+  # Also makes sure the containers weren't just restarted by an image update
+  when: quarkus_pids.rc == 0 and "is up-to-date" in keycloak_dcup.stderr
+  block:
+    - name: down containers
+      shell:
+        cmd: docker-compose -f /etc/keycloak-compose/docker-compose.yaml down
+    - name: up containers
+      shell:
+        cmd: docker-compose -f /etc/keycloak-compose/docker-compose.yaml up -d

playbooks/roles/keycloak/tasks/main.yaml

@@ -7,22 +7,27 @@
   template:
     src: docker-compose.yaml.j2
     dest: /etc/keycloak-docker/docker-compose.yaml
+    owner: root
+    group: root
+    mode: "0600"
+  notify: keycloak restart containers
 
+# This deliberately does not set owner/group/mode, as the mariadb container
+# chowns this directory to be owned by a container-internal user and drops
+# root privileges.  We don't want to reset this from outside the container.
 - name: Ensure data directory exists
   file:
     state: directory
-    path: /var/keycloak/data
-    owner: "1000"
-    group: "root"
-    mode: "0755"
+    path: /var/lib/keycloak/db
 
-- name: Ensure log directory exists
-  file:
-    state: directory
-    path: /var/log/keycloak
-    owner: "1000"
-    group: "root"
-    mode: "0755"
+- name: Copy our MariaDB config stub overriding bind-address
+  copy:
+    src: 99-bind-address.cnf
+    dest: /var/lib/keycloak/99-bind-address.cnf
+    owner: root
+    group: root
+    mode: "0644"
+  notify: keycloak restart containers
 
 - name: Install apache2
   apt:
@@ -42,6 +47,7 @@
     - ssl
     - headers
     - proxy_wstunnel
+  notify: keycloak restart apache2
 
 - name: Copy apache config
   template:
@@ -50,7 +56,7 @@
     owner: root
     group: root
     mode: 0644
-  notify: keycloak Reload apache2
+  notify: keycloak reload apache2
 
 - name: Run docker-compose pull
   shell:
@@ -61,11 +67,13 @@
   shell:
     cmd: docker-compose up -d
     chdir: /etc/keycloak-docker/
+  register: keycloak_dcup
 
 - name: Wait for keycloak to start
   wait_for:
+    host: "::1"
     port: 8080
-    timeout: 60
+    timeout: 300
 
 - name: Run docker prune to cleanup unneeded images
   shell:

playbooks/roles/keycloak/templates/docker-compose.yaml.j2

@@ -3,18 +3,47 @@
 version: '2'
 
 services:
-  keycloak:
-    image: quay.io/keycloak/keycloak:legacy
+  mariadb:
+    # 10.11 was synonymous with the "lts" tag when we brought up the service
+    image: docker.io/library/mariadb:10.11
     network_mode: host
     restart: always
     environment:
-      - KEYCLOAK_USER=admin
-      - KEYCLOAK_PASSWORD="{{ keycloak_admin_password }}"
-      - DB_VENDOR=h2
-      - PROXY_ADDRESS_FORWARDING=true
-    command:
-      -Djboss.bind.address.private=127.0.0.1
-      -Djboss.bind.address=127.0.0.1
+      MARIADB_ROOT_PASSWORD: "{{ keycloak_root_db_password }}"
+      MARIADB_DATABASE: keycloak
+      MARIADB_USER: keycloak
+      MARIADB_PASSWORD: "{{ keycloak_db_password }}"
     volumes:
-      - /var/keycloak/data:/opt/jboss/keycloak/standalone/data
-      - /var/log/keycloak:/opt/jboss/keycloak/standalone/log
+      - /var/lib/keycloak/db:/var/lib/mysql
+      - /var/lib/keycloak/99-bind-address.cnf:/etc/mysql/conf.d/99-bind-address.cnf:ro
+    logging:
+      driver: syslog
+      options:
+        tag: docker-mariadb
+  keycloak:
+    depends_on:
+      - mariadb
+    image: quay.io/keycloak/keycloak:23.0
+    network_mode: host
+    restart: always
+    environment:
+      KC_DB_PASSWORD: "{{ keycloak_db_password }}"
+      KC_DB_USERNAME: keycloak
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: "{{ keycloak_admin_password }}"
+    command:
+      - 'start'
+      - '--hostname-strict=false'
+      - '--http-enabled=true'
+      - '--http-host=::1'
+      - '--proxy=edge'
+      - '--db=mariadb'
+      # Wrap the DB host address here because it ends up inserted into a
+      # colon-delimited JDBC URL internally.
+      - '--db-url-host=[::1]'
+      - '--db-url-port=3306'
+      - '--db-url-database=keycloak'
+    logging:
+      driver: syslog
+      options:
+        tag: docker-keycloak

playbooks/roles/keycloak/templates/keycloak.vhost.j2

@@ -48,8 +48,8 @@
   # https://localhost:8443/server-status
   RewriteRule ^/server-status$ /server-status [L]
 
-  ProxyPass  / http://localhost:8080/ retry=0
-  ProxyPassReverse / http://localhost:8080/
+  ProxyPass  / http://[::1]:8080/ retry=0
+  ProxyPassReverse / http://[::1]:8080/
   ProxyPreserveHost on
   RequestHeader set "X-Forwarded-Proto" expr=%{REQUEST_SCHEME}
 

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -247,7 +247,7 @@
 - name: letsencrypt updated etherpad-opendev-org-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
-- name: letsencrypt updated keycloak01-opendev-org-main
+- name: letsencrypt updated keycloak-opendev-org-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
 
 - name: letsencrypt updated storyboard01-opendev-org-main

playbooks/zuul/templates/group_vars/keycloak.yaml.j2

@@ -1 +1,3 @@
 keycloak_admin_password: testpassword
+keycloak_root_db_password: testdbrootpass
+keycloak_db_password: testdbuserpass

testinfra/test_keycloak.py

@@ -17,21 +17,39 @@
 import json
 
 
-testinfra_hosts = ['keycloak01.opendev.org']
+testinfra_hosts = ['keycloak99.opendev.org']
 
 
+def test_rdbms_listening(host):
+    keycloak = host.socket("tcp://::1:3306")
+    assert keycloak.is_listening
+
 def test_keycloak_listening(host):
-    keycloak = host.socket("tcp://127.0.0.1:8080")
+    keycloak = host.socket("tcp://::1:8080")
     assert keycloak.is_listening
 
+def test_rdbms_used(host):
+    # This checks that keycloak created tables in the database,
+    # ensuring our intended database backend is actually used.
+
+    # The nested quotes get really ornery, so try to defuse some
+    # of it with a raw string included via string formatting.
+    query = (r'select DESCRIPTION from keycloak.KEYCLOAK_ROLE '
+             'where NAME=\\"default-roles-master\\"')
+    cmd = host.run(
+        """docker-compose -f /etc/keycloak-docker/docker-compose.yaml \
+        exec -T mariadb bash -c '/usr/bin/mysql -B -p$MARIADB_PASSWORD \
+        -ukeycloak -e "%s"'""" % query)
+    assert ("role_default-roles" in cmd.stdout)
+
 def test_keycloak_openid_config(host):
     # This tests the proxy config since the output is determined by
     # the proxy headers and is not hard-coded configuration.
     cmd = host.run('curl --insecure '
-                   '--resolve keycloak.opendev.org:443:127.0.0.1 '
-                   'https://keycloak.opendev.org/auth/realms/master'
+                   '--resolve keycloak.opendev.org:443:[::1] '
+                   'https://keycloak.opendev.org/realms/master'
                    '/.well-known/openid-configuration')
-    assert ('"issuer":"https://keycloak.opendev.org/auth/realms/master"'
+    assert ('"issuer":"https://keycloak.opendev.org/realms/master"'
             in cmd.stdout)
 
 def test_keycloak_admin_api(host):
@@ -39,7 +57,7 @@ def test_keycloak_admin_api(host):
     # acquire an OIDC bearer token and then use it to check the
     # user count.
     cmd = host.run('curl --insecure '
-                   '--resolve keycloak.opendev.org:443:127.0.0.1 '
+                   '--resolve keycloak.opendev.org:443:[::1] '
                    '-X POST '
                    '-H "Content-Type: application/x-www-form-urlencoded" '
                    '-d "username=admin" '
@@ -47,14 +65,13 @@ def test_keycloak_admin_api(host):
                    '-d "grant_type=password" '
                    '-d "client_id=admin-cli" '
                    'https://keycloak.opendev.org'
-                   '/auth/realms/master/protocol/openid-connect/token')
+                   '/realms/master/protocol/openid-connect/token')
     token = json.loads(cmd.stdout)
     assert token["token_type"] == "Bearer"
     cmd = host.run('curl --insecure '
-                   '--resolve keycloak.opendev.org:443:127.0.0.1 '
+                   '--resolve keycloak.opendev.org:443:[::1] '
                    '-H "Authorization: Bearer %s" '
                    '-H "Content-Type: application/json" '
                    'https://keycloak.opendev.org'
-                   '/auth/admin/realms/master/users/count'
-                   % token["access_token"])
+                   '/admin/realms/master/users/count' % token["access_token"])
     assert cmd.stdout == "1"

zuul.d/infra-prod.yaml

@@ -208,8 +208,7 @@
     files:
       - inventory/base
       - playbooks/service-keycloak.yaml
-      - inventory/service/host_vars/keycloak01.opendev.org.yaml
-      - inventory/service/group_vars/keycloak
+      - inventory/service/group_vars/keycloak.yaml
       - playbooks/roles/keycloak/
       - playbooks/roles/install-docker/
       - playbooks/roles/iptables/

zuul.d/system-config-run.yaml

@@ -785,8 +785,8 @@
     nodeset:
       nodes:
         - <<: *bridge_node_x86
-        - name: keycloak01.opendev.org
-          label: ubuntu-focal
+        - name: keycloak99.opendev.org
+          label: ubuntu-jammy
       groups:
         - <<: *bastion_group
     vars:
@@ -794,7 +794,7 @@
         - playbooks/letsencrypt.yaml
         - playbooks/service-keycloak.yaml
     files:
-      - inventory/service/host_vars/keycloak01.opendev.org.yaml
+      - inventory/service/group_vars/keycloak.yaml
       - playbooks/install-ansible.yaml
       - playbooks/letsencrypt.yaml
       - playbooks/service-keycloak.yaml