Incorporate HA PostgreSQL

This change integrates changes related to the new Patroni-enabled
high availability included in the postgresql helm chart.

1. Updates the version of the postgresql chart
2. Adds users/passphrases for the postgresql replication and exporter users
3. Sets the default number of replicas to 3 (except for skiff & sloop)
4. Adds documentation/scripting for upgrading the chart

Co-Authored-By: Evgeniy L <eli@mirantis.com>
Change-Id: I51e63fb2444610cfb144bc587a6a179fa4830809

global/software/charts/ucp/core/postgresql.yaml

@@ -40,6 +40,18 @@ metadata:
         path: .ucp.postgres.admin
       dest:
         path: .values.endpoints.postgresql.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.postgres.replica
+      dest:
+        path: .values.endpoints.postgresql.auth.replica
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.postgres.exporter
+      dest:
+        path: .values.endpoints.postgresql.auth.exporter
 
     # Secrets
     - dest:
@@ -48,17 +60,31 @@ metadata:
         schema: deckhand/Passphrase/v1
         name: ucp_postgres_admin_password
         path: .
+    - dest:
+        path: .values.endpoints.postgresql.auth.replica.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_postgres_replication_password
+        path: .
+    - dest:
+        path: .values.endpoints.postgresql.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_postgres_exporter_password
+        path: .
 data:
   chart_name: ucp-postgresql
   release: ucp-postgresql
   namespace: ucp
   wait:
-    timeout: 600
+    timeout: 1800
     labels:
       release_group: airship-ucp-postgresql
   install:
     no_hooks: false
   upgrade:
+    options:
+      force: true
     no_hooks: false
     pre:
       delete:
@@ -69,6 +95,9 @@ data:
     post:
       create: []
   values:
+    pod:
+      replicas:
+        server: 3
     conf:
       postgresql:
         max_connections: 1000

global/software/config/versions.yaml

@@ -463,7 +463,7 @@ data:
         type: git
       postgresql:
         location: https://opendev.org/openstack/openstack-helm-infra
-        reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
+        reference: 09ae22d8493d5cef34c80cb69117c69dc0f2dc8e
         subpath: postgresql
         type: git
       postgresql-htk:

site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airskiff/software/charts/ucp/core/postgresql.yaml

@@ -0,0 +1,21 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-postgresql
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-postgresql-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        server: 1
+...

site/airskiff/software/config/service_accounts.yaml

@@ -15,6 +15,10 @@ data:
         postgres:
             admin:
                 username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
         oslo_db:
             admin:
                 username: root

site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: airsloop123
+...

site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml

@@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...

site/seaworthy/software/config/service_accounts.yaml

@@ -15,6 +15,10 @@ data:
         postgres:
             admin:
                 username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
         oslo_db:
             admin:
                 username: root

tools/upgrades/postgresql/README.md

@@ -0,0 +1,24 @@
+# PostgreSQL Patroni Upgrade Scripts
+
+Upgrading a live site from the old, unclustered PostgreSQL chart to the newer,
+Patroni-managed version takes a small amount of out-of-band scripting to ensure
+a smooth hands-free upgrade.
+
+## Prior to upgrade
+
+The ``patroni_endpoint_cleaner_unit.sh`` script should be run prior to upgrading
+the postgresql chart.  It installs a systemd unit which in turn will run
+the ``patroni_endpoint_cleaner.sh`` script.  During chart upgrade, the script
+will delete the postgresql endpoints, allowing Patroni to recreate them with the
+appropriate annotations for it to manage them ongoing.
+
+This documentation project outlines a reference architecture for automated
+cloud provisioning and management, leveraging a collection of interoperable
+open-source tools.
+
+## Post upgrade
+
+After the chart upgrade is complete, the ``patroni_endpoint_cleaner_remove.sh``
+script should be run.  This will simply clean up the systemd unit that was
+created previously.
+

tools/upgrades/postgresql/patroni_endpoint_cleaner.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This script should be run as a one-time fix DURING an upgrade of
+# "vanilla" postgres to patroni (in either a single or HA multi-replica
+# configuration).
+#
+# This addresses an issue where the previous version of the chart had a
+# service-managed `endpoints` object, while patroni needs to manage its
+# own kubernetes `endpoints`.  Patroni won't successfully manage
+# (i.e. apply annotation to, etc) the postgresql endpoints until the
+# service-managed endpoints are out of the way; however deletion of the
+# postgresql endpoints must be done with care during an upgrade.
+#
+# This script watches for the right moment and deletes the endpoints.
+
+export KUBECONFIG=${KUBECONFIG:-"/etc/kubernetes/admin.conf"}
+
+while true; do
+  echo "Checking to see if patroni is deployed..."
+  # Wait for the patroni-based chart to get deployed
+  if [ $(kubectl describe pod -n ucp postgresql-0 | grep -c "patroni") -gt 0 ]; then
+    echo 'Detected that patroni is deployed'
+
+    # The port name used by the single-node postgres chart is "db",
+    # while the new port name is "postgres"
+    FIRST_PORT_NAME=$(kubectl get -n ucp endpoints postgresql -o jsonpath='{.subsets[0].ports[0].name}')
+    if [ "x${FIRST_PORT_NAME}" == "xdb" ]; then
+      echo "matched the old endpoints: deleting old postgresql endpoints"
+      kubectl delete endpoints -n ucp postgresql
+      echo "done."
+      exit 0
+    fi
+  fi
+
+  sleep 5
+done

tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+
+echo "Cleaning up the patroni_endpoint_cleaner"
+sudo systemctl stop patroni_endpoint_cleaner
+sudo systemctl disable patroni_endpoint_cleaner
+sudo rm -f /opt/patroni_endpoint_cleaner.sh
+sudo rm -f /lib/systemd/system/patroni_endpoint_cleaner.service
+sudo rm -f /etc/systemd/system/multi-user.target.wants/patroni_endpoint_cleaner.service
+sudo systemctl daemon-reload
+sudo systemctl reset-failed

tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+
+sudo chmod 700 patroni_endpoint_cleaner.sh
+sudo cp patroni_endpoint_cleaner.sh /opt
+
+cat > ./patroni_endpoint_cleaner.service << EOF
+[Unit]
+Description=Helper script for initial upgrade to HA Postgres
+
+[Service]
+ExecStart=/opt/patroni_endpoint_cleaner.sh
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sudo mv patroni_endpoint_cleaner.service /lib/systemd/system/
+
+sudo systemctl restart patroni_endpoint_cleaner
+sudo systemctl enable patroni_endpoint_cleaner
+sudo systemctl daemon-reload

type/sloop/charts/ucp/core/postgresql.yaml

@@ -0,0 +1,23 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-postgresql
+  replacement: true
+  labels:
+    name: ucp-postgresql-type
+  layeringDefinition:
+    abstract: false
+    layer: type
+    parentSelector:
+      name: ucp-postgresql-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        server: 1
+...

type/sloop/config/service_accounts.yaml

@@ -17,6 +17,10 @@ data:
         postgres:
             admin:
                 username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
         oslo_db:
             admin:
                 username: root