Incorporate HA PostgreSQL

This change integrates changes related to the new Patroni-enabled high availability included in the postgresql helm chart. 1. Updates the version of the postgresql chart 2. Adds users/passphrases for the postgresql replication and exporter users 3. Sets the default number of replicas to 3 (except for skiff & sloop) 4. Adds documentation/scripting for upgrading the chart Co-Authored-By: Evgeniy L <eli@mirantis.com> Change-Id: I51e63fb2444610cfb144bc587a6a179fa4830809
2019-05-07 13:56:28 -05:00 · 2019-05-07 13:56:28 -05:00 · 16cdf842d2
parent b15899667f
commit 16cdf842d2
19 changed files with 268 additions and 2 deletions
--- a/global/software/charts/ucp/core/postgresql.yaml
+++ b/global/software/charts/ucp/core/postgresql.yaml
@ -40,6 +40,18 @@ metadata:
        path: .ucp.postgres.admin
      dest:
        path: .values.endpoints.postgresql.auth.admin
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.postgres.replica
+      dest:
+        path: .values.endpoints.postgresql.auth.replica
+    - src:
+        schema: pegleg/AccountCatalogue/v1
+        name: ucp_service_accounts
+        path: .ucp.postgres.exporter
+      dest:
+        path: .values.endpoints.postgresql.auth.exporter

    # Secrets
    - dest:
@ -48,17 +60,31 @@ metadata:
        schema: deckhand/Passphrase/v1
        name: ucp_postgres_admin_password
        path: .
+    - dest:
+        path: .values.endpoints.postgresql.auth.replica.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_postgres_replication_password
+        path: .
+    - dest:
+        path: .values.endpoints.postgresql.auth.exporter.password
+      src:
+        schema: deckhand/Passphrase/v1
+        name: ucp_postgres_exporter_password
+        path: .
 data:
  chart_name: ucp-postgresql
  release: ucp-postgresql
  namespace: ucp
  wait:
-    timeout: 600
+    timeout: 1800
    labels:
      release_group: airship-ucp-postgresql
  install:
    no_hooks: false
  upgrade:
+    options:
+      force: true
    no_hooks: false
    pre:
      delete:
@ -69,6 +95,9 @@ data:
    post:
      create: []
  values:
+    pod:
+      replicas:
+        server: 3
    conf:
      postgresql:
        max_connections: 1000
--- a/global/software/config/versions.yaml
+++ b/global/software/config/versions.yaml
@ -463,7 +463,7 @@ data:
        type: git
      postgresql:
        location: https://opendev.org/openstack/openstack-helm-infra
-        reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
+        reference: 09ae22d8493d5cef34c80cb69117c69dc0f2dc8e
        subpath: postgresql
        type: git
      postgresql-htk:
--- a/site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml
+++ b/site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml
+++ b/site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml
+++ b/site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml
+++ b/site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airskiff/software/charts/ucp/core/postgresql.yaml
+++ b/site/airskiff/software/charts/ucp/core/postgresql.yaml
@ -0,0 +1,21 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-postgresql
+  replacement: true
+  layeringDefinition:
+    abstract: false
+    layer: site
+    parentSelector:
+      name: ucp-postgresql-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        server: 1
+...
--- a/site/airskiff/software/config/service_accounts.yaml
+++ b/site/airskiff/software/config/service_accounts.yaml
@ -15,6 +15,10 @@ data:
        postgres:
            admin:
                username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
        oslo_db:
            admin:
                username: root
--- a/site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml
+++ b/site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml
+++ b/site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: airsloop123
+...
--- a/site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml
+++ b/site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_exporter_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml
+++ b/site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml
@ -0,0 +1,11 @@
+---
+schema: deckhand/Passphrase/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp_postgres_replication_password
+  layeringDefinition:
+    abstract: false
+    layer: site
+  storagePolicy: cleartext
+data: password123
+...
--- a/site/seaworthy/software/config/service_accounts.yaml
+++ b/site/seaworthy/software/config/service_accounts.yaml
@ -15,6 +15,10 @@ data:
        postgres:
            admin:
                username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
        oslo_db:
            admin:
                username: root
--- a/tools/upgrades/postgresql/README.md
+++ b/tools/upgrades/postgresql/README.md
@ -0,0 +1,24 @@
+# PostgreSQL Patroni Upgrade Scripts
+
+Upgrading a live site from the old, unclustered PostgreSQL chart to the newer,
+Patroni-managed version takes a small amount of out-of-band scripting to ensure
+a smooth hands-free upgrade.
+
+## Prior to upgrade
+
+The ``patroni_endpoint_cleaner_unit.sh`` script should be run prior to upgrading
+the postgresql chart.  It installs a systemd unit which in turn will run
+the ``patroni_endpoint_cleaner.sh`` script.  During chart upgrade, the script
+will delete the postgresql endpoints, allowing Patroni to recreate them with the
+appropriate annotations for it to manage them ongoing.
+
+This documentation project outlines a reference architecture for automated
+cloud provisioning and management, leveraging a collection of interoperable
+open-source tools.
+
+## Post upgrade
+
+After the chart upgrade is complete, the ``patroni_endpoint_cleaner_remove.sh``
+script should be run.  This will simply clean up the systemd unit that was
+created previously.
+
--- a/tools/upgrades/postgresql/patroni_endpoint_cleaner.sh
+++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner.sh
@ -0,0 +1,36 @@
+#!/bin/bash
+
+# This script should be run as a one-time fix DURING an upgrade of
+# "vanilla" postgres to patroni (in either a single or HA multi-replica
+# configuration).
+#
+# This addresses an issue where the previous version of the chart had a
+# service-managed `endpoints` object, while patroni needs to manage its
+# own kubernetes `endpoints`.  Patroni won't successfully manage
+# (i.e. apply annotation to, etc) the postgresql endpoints until the
+# service-managed endpoints are out of the way; however deletion of the
+# postgresql endpoints must be done with care during an upgrade.
+#
+# This script watches for the right moment and deletes the endpoints.
+
+export KUBECONFIG=${KUBECONFIG:-"/etc/kubernetes/admin.conf"}
+
+while true; do
+  echo "Checking to see if patroni is deployed..."
+  # Wait for the patroni-based chart to get deployed
+  if [ $(kubectl describe pod -n ucp postgresql-0 | grep -c "patroni") -gt 0 ]; then
+    echo 'Detected that patroni is deployed'
+
+    # The port name used by the single-node postgres chart is "db",
+    # while the new port name is "postgres"
+    FIRST_PORT_NAME=$(kubectl get -n ucp endpoints postgresql -o jsonpath='{.subsets[0].ports[0].name}')
+    if [ "x${FIRST_PORT_NAME}" == "xdb" ]; then
+      echo "matched the old endpoints: deleting old postgresql endpoints"
+      kubectl delete endpoints -n ucp postgresql
+      echo "done."
+      exit 0
+    fi
+  fi
+
+  sleep 5
+done
--- a/tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh
+++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh
@ -0,0 +1,11 @@
+#!/bin/bash
+set -x
+
+echo "Cleaning up the patroni_endpoint_cleaner"
+sudo systemctl stop patroni_endpoint_cleaner
+sudo systemctl disable patroni_endpoint_cleaner
+sudo rm -f /opt/patroni_endpoint_cleaner.sh
+sudo rm -f /lib/systemd/system/patroni_endpoint_cleaner.service
+sudo rm -f /etc/systemd/system/multi-user.target.wants/patroni_endpoint_cleaner.service
+sudo systemctl daemon-reload
+sudo systemctl reset-failed
--- a/tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh
+++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh
@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+
+sudo chmod 700 patroni_endpoint_cleaner.sh
+sudo cp patroni_endpoint_cleaner.sh /opt
+
+cat > ./patroni_endpoint_cleaner.service << EOF
+[Unit]
+Description=Helper script for initial upgrade to HA Postgres
+
+[Service]
+ExecStart=/opt/patroni_endpoint_cleaner.sh
+
+[Install]
+WantedBy=multi-user.target
+EOF
+
+sudo mv patroni_endpoint_cleaner.service /lib/systemd/system/
+
+sudo systemctl restart patroni_endpoint_cleaner
+sudo systemctl enable patroni_endpoint_cleaner
+sudo systemctl daemon-reload
--- a/type/sloop/charts/ucp/core/postgresql.yaml
+++ b/type/sloop/charts/ucp/core/postgresql.yaml
@ -0,0 +1,23 @@
+---
+schema: armada/Chart/v1
+metadata:
+  schema: metadata/Document/v1
+  name: ucp-postgresql
+  replacement: true
+  labels:
+    name: ucp-postgresql-type
+  layeringDefinition:
+    abstract: false
+    layer: type
+    parentSelector:
+      name: ucp-postgresql-global
+    actions:
+      - method: merge
+        path: .
+  storagePolicy: cleartext
+data:
+  values:
+    pod:
+      replicas:
+        server: 1
+...
--- a/type/sloop/config/service_accounts.yaml
+++ b/type/sloop/config/service_accounts.yaml
@ -17,6 +17,10 @@ data:
        postgres:
            admin:
                username: postgres
+            replica:
+                username: standby
+            exporter:
+                username: psql_exporter
        oslo_db:
            admin:
                username: root