diff --git a/global/software/charts/ucp/core/postgresql.yaml b/global/software/charts/ucp/core/postgresql.yaml index 3bd707e28..6a25b6534 100644 --- a/global/software/charts/ucp/core/postgresql.yaml +++ b/global/software/charts/ucp/core/postgresql.yaml @@ -40,6 +40,18 @@ metadata: path: .ucp.postgres.admin dest: path: .values.endpoints.postgresql.auth.admin + - src: + schema: pegleg/AccountCatalogue/v1 + name: ucp_service_accounts + path: .ucp.postgres.replica + dest: + path: .values.endpoints.postgresql.auth.replica + - src: + schema: pegleg/AccountCatalogue/v1 + name: ucp_service_accounts + path: .ucp.postgres.exporter + dest: + path: .values.endpoints.postgresql.auth.exporter # Secrets - dest: @@ -48,17 +60,31 @@ metadata: schema: deckhand/Passphrase/v1 name: ucp_postgres_admin_password path: . + - dest: + path: .values.endpoints.postgresql.auth.replica.password + src: + schema: deckhand/Passphrase/v1 + name: ucp_postgres_replication_password + path: . + - dest: + path: .values.endpoints.postgresql.auth.exporter.password + src: + schema: deckhand/Passphrase/v1 + name: ucp_postgres_exporter_password + path: . data: chart_name: ucp-postgresql release: ucp-postgresql namespace: ucp wait: - timeout: 600 + timeout: 1800 labels: release_group: airship-ucp-postgresql install: no_hooks: false upgrade: + options: + force: true no_hooks: false pre: delete: @@ -69,6 +95,9 @@ data: post: create: [] values: + pod: + replicas: + server: 3 conf: postgresql: max_connections: 1000 diff --git a/global/software/config/versions.yaml b/global/software/config/versions.yaml index 32bd5c61e..22fa6ed30 100644 --- a/global/software/config/versions.yaml +++ b/global/software/config/versions.yaml @@ -463,7 +463,7 @@ data: type: git postgresql: location: https://opendev.org/openstack/openstack-helm-infra - reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e + reference: 09ae22d8493d5cef34c80cb69117c69dc0f2dc8e subpath: postgresql type: git postgresql-htk: diff --git a/site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml b/site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml new file mode 100644 index 000000000..abdaa5bc4 --- /dev/null +++ b/site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_exporter_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml b/site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml new file mode 100644 index 000000000..2176e714f --- /dev/null +++ b/site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_replication_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml b/site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml new file mode 100644 index 000000000..abdaa5bc4 --- /dev/null +++ b/site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_exporter_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml b/site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml new file mode 100644 index 000000000..2176e714f --- /dev/null +++ b/site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_replication_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/airskiff/software/charts/ucp/core/postgresql.yaml b/site/airskiff/software/charts/ucp/core/postgresql.yaml new file mode 100644 index 000000000..08fae7c00 --- /dev/null +++ b/site/airskiff/software/charts/ucp/core/postgresql.yaml @@ -0,0 +1,21 @@ +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ucp-postgresql + replacement: true + layeringDefinition: + abstract: false + layer: site + parentSelector: + name: ucp-postgresql-global + actions: + - method: merge + path: . + storagePolicy: cleartext +data: + values: + pod: + replicas: + server: 1 +... diff --git a/site/airskiff/software/config/service_accounts.yaml b/site/airskiff/software/config/service_accounts.yaml index f15b49218..751f1b1f7 100644 --- a/site/airskiff/software/config/service_accounts.yaml +++ b/site/airskiff/software/config/service_accounts.yaml @@ -15,6 +15,10 @@ data: postgres: admin: username: postgres + replica: + username: standby + exporter: + username: psql_exporter oslo_db: admin: username: root diff --git a/site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml b/site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml new file mode 100644 index 000000000..abdaa5bc4 --- /dev/null +++ b/site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_exporter_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml b/site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml new file mode 100644 index 000000000..452043e9f --- /dev/null +++ b/site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_replication_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: airsloop123 +... diff --git a/site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml b/site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml new file mode 100644 index 000000000..abdaa5bc4 --- /dev/null +++ b/site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_exporter_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml b/site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml new file mode 100644 index 000000000..2176e714f --- /dev/null +++ b/site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml @@ -0,0 +1,11 @@ +--- +schema: deckhand/Passphrase/v1 +metadata: + schema: metadata/Document/v1 + name: ucp_postgres_replication_password + layeringDefinition: + abstract: false + layer: site + storagePolicy: cleartext +data: password123 +... diff --git a/site/seaworthy/software/config/service_accounts.yaml b/site/seaworthy/software/config/service_accounts.yaml index f15b49218..751f1b1f7 100644 --- a/site/seaworthy/software/config/service_accounts.yaml +++ b/site/seaworthy/software/config/service_accounts.yaml @@ -15,6 +15,10 @@ data: postgres: admin: username: postgres + replica: + username: standby + exporter: + username: psql_exporter oslo_db: admin: username: root diff --git a/tools/upgrades/postgresql/README.md b/tools/upgrades/postgresql/README.md new file mode 100644 index 000000000..3c6f2d461 --- /dev/null +++ b/tools/upgrades/postgresql/README.md @@ -0,0 +1,24 @@ +# PostgreSQL Patroni Upgrade Scripts + +Upgrading a live site from the old, unclustered PostgreSQL chart to the newer, +Patroni-managed version takes a small amount of out-of-band scripting to ensure +a smooth hands-free upgrade. + +## Prior to upgrade + +The ``patroni_endpoint_cleaner_unit.sh`` script should be run prior to upgrading +the postgresql chart. It installs a systemd unit which in turn will run +the ``patroni_endpoint_cleaner.sh`` script. During chart upgrade, the script +will delete the postgresql endpoints, allowing Patroni to recreate them with the +appropriate annotations for it to manage them ongoing. + +This documentation project outlines a reference architecture for automated +cloud provisioning and management, leveraging a collection of interoperable +open-source tools. + +## Post upgrade + +After the chart upgrade is complete, the ``patroni_endpoint_cleaner_remove.sh`` +script should be run. This will simply clean up the systemd unit that was +created previously. + diff --git a/tools/upgrades/postgresql/patroni_endpoint_cleaner.sh b/tools/upgrades/postgresql/patroni_endpoint_cleaner.sh new file mode 100755 index 000000000..3e6d23324 --- /dev/null +++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This script should be run as a one-time fix DURING an upgrade of +# "vanilla" postgres to patroni (in either a single or HA multi-replica +# configuration). +# +# This addresses an issue where the previous version of the chart had a +# service-managed `endpoints` object, while patroni needs to manage its +# own kubernetes `endpoints`. Patroni won't successfully manage +# (i.e. apply annotation to, etc) the postgresql endpoints until the +# service-managed endpoints are out of the way; however deletion of the +# postgresql endpoints must be done with care during an upgrade. +# +# This script watches for the right moment and deletes the endpoints. + +export KUBECONFIG=${KUBECONFIG:-"/etc/kubernetes/admin.conf"} + +while true; do + echo "Checking to see if patroni is deployed..." + # Wait for the patroni-based chart to get deployed + if [ $(kubectl describe pod -n ucp postgresql-0 | grep -c "patroni") -gt 0 ]; then + echo 'Detected that patroni is deployed' + + # The port name used by the single-node postgres chart is "db", + # while the new port name is "postgres" + FIRST_PORT_NAME=$(kubectl get -n ucp endpoints postgresql -o jsonpath='{.subsets[0].ports[0].name}') + if [ "x${FIRST_PORT_NAME}" == "xdb" ]; then + echo "matched the old endpoints: deleting old postgresql endpoints" + kubectl delete endpoints -n ucp postgresql + echo "done." + exit 0 + fi + fi + + sleep 5 +done diff --git a/tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh b/tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh new file mode 100755 index 000000000..03343c543 --- /dev/null +++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -x + +echo "Cleaning up the patroni_endpoint_cleaner" +sudo systemctl stop patroni_endpoint_cleaner +sudo systemctl disable patroni_endpoint_cleaner +sudo rm -f /opt/patroni_endpoint_cleaner.sh +sudo rm -f /lib/systemd/system/patroni_endpoint_cleaner.service +sudo rm -f /etc/systemd/system/multi-user.target.wants/patroni_endpoint_cleaner.service +sudo systemctl daemon-reload +sudo systemctl reset-failed diff --git a/tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh b/tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh new file mode 100755 index 000000000..83736fa9e --- /dev/null +++ b/tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh @@ -0,0 +1,22 @@ +#!/bin/bash +set -ex + +sudo chmod 700 patroni_endpoint_cleaner.sh +sudo cp patroni_endpoint_cleaner.sh /opt + +cat > ./patroni_endpoint_cleaner.service << EOF +[Unit] +Description=Helper script for initial upgrade to HA Postgres + +[Service] +ExecStart=/opt/patroni_endpoint_cleaner.sh + +[Install] +WantedBy=multi-user.target +EOF + +sudo mv patroni_endpoint_cleaner.service /lib/systemd/system/ + +sudo systemctl restart patroni_endpoint_cleaner +sudo systemctl enable patroni_endpoint_cleaner +sudo systemctl daemon-reload diff --git a/type/sloop/charts/ucp/core/postgresql.yaml b/type/sloop/charts/ucp/core/postgresql.yaml new file mode 100644 index 000000000..8e7a3e3b1 --- /dev/null +++ b/type/sloop/charts/ucp/core/postgresql.yaml @@ -0,0 +1,23 @@ +--- +schema: armada/Chart/v1 +metadata: + schema: metadata/Document/v1 + name: ucp-postgresql + replacement: true + labels: + name: ucp-postgresql-type + layeringDefinition: + abstract: false + layer: type + parentSelector: + name: ucp-postgresql-global + actions: + - method: merge + path: . + storagePolicy: cleartext +data: + values: + pod: + replicas: + server: 1 +... diff --git a/type/sloop/config/service_accounts.yaml b/type/sloop/config/service_accounts.yaml index c9b76e76f..4541cdc2f 100644 --- a/type/sloop/config/service_accounts.yaml +++ b/type/sloop/config/service_accounts.yaml @@ -17,6 +17,10 @@ data: postgres: admin: username: postgres + replica: + username: standby + exporter: + username: psql_exporter oslo_db: admin: username: root