Browse Source

Incorporate HA PostgreSQL

This change integrates changes related to the new Patroni-enabled
high availability included in the postgresql helm chart.

1. Updates the version of the postgresql chart
2. Adds users/passphrases for the postgresql replication and exporter users
3. Sets the default number of replicas to 3 (except for skiff & sloop)
4. Adds documentation/scripting for upgrading the chart

Co-Authored-By: Evgeniy L <eli@mirantis.com>
Change-Id: I51e63fb2444610cfb144bc587a6a179fa4830809
changes/67/657667/13
Matt McEuen 2 years ago
committed by Kaspars Skels
parent
commit
16cdf842d2
  1. 31
      global/software/charts/ucp/core/postgresql.yaml
  2. 2
      global/software/config/versions.yaml
  3. 11
      site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml
  4. 11
      site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml
  5. 11
      site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml
  6. 11
      site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml
  7. 21
      site/airskiff/software/charts/ucp/core/postgresql.yaml
  8. 4
      site/airskiff/software/config/service_accounts.yaml
  9. 11
      site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml
  10. 11
      site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml
  11. 11
      site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml
  12. 11
      site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml
  13. 4
      site/seaworthy/software/config/service_accounts.yaml
  14. 24
      tools/upgrades/postgresql/README.md
  15. 36
      tools/upgrades/postgresql/patroni_endpoint_cleaner.sh
  16. 11
      tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh
  17. 22
      tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh
  18. 23
      type/sloop/charts/ucp/core/postgresql.yaml
  19. 4
      type/sloop/config/service_accounts.yaml

31
global/software/charts/ucp/core/postgresql.yaml

@ -40,6 +40,18 @@ metadata:
path: .ucp.postgres.admin
dest:
path: .values.endpoints.postgresql.auth.admin
- src:
schema: pegleg/AccountCatalogue/v1
name: ucp_service_accounts
path: .ucp.postgres.replica
dest:
path: .values.endpoints.postgresql.auth.replica
- src:
schema: pegleg/AccountCatalogue/v1
name: ucp_service_accounts
path: .ucp.postgres.exporter
dest:
path: .values.endpoints.postgresql.auth.exporter
# Secrets
- dest:
@ -48,17 +60,31 @@ metadata:
schema: deckhand/Passphrase/v1
name: ucp_postgres_admin_password
path: .
- dest:
path: .values.endpoints.postgresql.auth.replica.password
src:
schema: deckhand/Passphrase/v1
name: ucp_postgres_replication_password
path: .
- dest:
path: .values.endpoints.postgresql.auth.exporter.password
src:
schema: deckhand/Passphrase/v1
name: ucp_postgres_exporter_password
path: .
data:
chart_name: ucp-postgresql
release: ucp-postgresql
namespace: ucp
wait:
timeout: 600
timeout: 1800
labels:
release_group: airship-ucp-postgresql
install:
no_hooks: false
upgrade:
options:
force: true
no_hooks: false
pre:
delete:
@ -69,6 +95,9 @@ data:
post:
create: []
values:
pod:
replicas:
server: 3
conf:
postgresql:
max_connections: 1000

2
global/software/config/versions.yaml

@ -463,7 +463,7 @@ data:
type: git
postgresql:
location: https://opendev.org/openstack/openstack-helm-infra
reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
reference: 09ae22d8493d5cef34c80cb69117c69dc0f2dc8e
subpath: postgresql
type: git
postgresql-htk:

11
site/aiab/secrets/passphrases/ucp_postgres_exporter_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_exporter_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/aiab/secrets/passphrases/ucp_postgres_replication_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_replication_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airskiff/secrets/passphrases/ucp_postgres_exporter_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_exporter_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airskiff/secrets/passphrases/ucp_postgres_replication_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_replication_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

21
site/airskiff/software/charts/ucp/core/postgresql.yaml

@ -0,0 +1,21 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-postgresql
replacement: true
layeringDefinition:
abstract: false
layer: site
parentSelector:
name: ucp-postgresql-global
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
values:
pod:
replicas:
server: 1
...

4
site/airskiff/software/config/service_accounts.yaml

@ -15,6 +15,10 @@ data:
postgres:
admin:
username: postgres
replica:
username: standby
exporter:
username: psql_exporter
oslo_db:
admin:
username: root

11
site/airsloop/secrets/passphrases/ucp_postgres_exporter_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_exporter_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/airsloop/secrets/passphrases/ucp_postgres_replication_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_replication_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: airsloop123
...

11
site/seaworthy/secrets/passphrases/ucp_postgres_exporter_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_exporter_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

11
site/seaworthy/secrets/passphrases/ucp_postgres_replication_password.yaml

@ -0,0 +1,11 @@
---
schema: deckhand/Passphrase/v1
metadata:
schema: metadata/Document/v1
name: ucp_postgres_replication_password
layeringDefinition:
abstract: false
layer: site
storagePolicy: cleartext
data: password123
...

4
site/seaworthy/software/config/service_accounts.yaml

@ -15,6 +15,10 @@ data:
postgres:
admin:
username: postgres
replica:
username: standby
exporter:
username: psql_exporter
oslo_db:
admin:
username: root

24
tools/upgrades/postgresql/README.md

@ -0,0 +1,24 @@
# PostgreSQL Patroni Upgrade Scripts
Upgrading a live site from the old, unclustered PostgreSQL chart to the newer,
Patroni-managed version takes a small amount of out-of-band scripting to ensure
a smooth hands-free upgrade.
## Prior to upgrade
The ``patroni_endpoint_cleaner_unit.sh`` script should be run prior to upgrading
the postgresql chart. It installs a systemd unit which in turn will run
the ``patroni_endpoint_cleaner.sh`` script. During chart upgrade, the script
will delete the postgresql endpoints, allowing Patroni to recreate them with the
appropriate annotations for it to manage them ongoing.
This documentation project outlines a reference architecture for automated
cloud provisioning and management, leveraging a collection of interoperable
open-source tools.
## Post upgrade
After the chart upgrade is complete, the ``patroni_endpoint_cleaner_remove.sh``
script should be run. This will simply clean up the systemd unit that was
created previously.

36
tools/upgrades/postgresql/patroni_endpoint_cleaner.sh

@ -0,0 +1,36 @@
#!/bin/bash
# This script should be run as a one-time fix DURING an upgrade of
# "vanilla" postgres to patroni (in either a single or HA multi-replica
# configuration).
#
# This addresses an issue where the previous version of the chart had a
# service-managed `endpoints` object, while patroni needs to manage its
# own kubernetes `endpoints`. Patroni won't successfully manage
# (i.e. apply annotation to, etc) the postgresql endpoints until the
# service-managed endpoints are out of the way; however deletion of the
# postgresql endpoints must be done with care during an upgrade.
#
# This script watches for the right moment and deletes the endpoints.
export KUBECONFIG=${KUBECONFIG:-"/etc/kubernetes/admin.conf"}
while true; do
echo "Checking to see if patroni is deployed..."
# Wait for the patroni-based chart to get deployed
if [ $(kubectl describe pod -n ucp postgresql-0 | grep -c "patroni") -gt 0 ]; then
echo 'Detected that patroni is deployed'
# The port name used by the single-node postgres chart is "db",
# while the new port name is "postgres"
FIRST_PORT_NAME=$(kubectl get -n ucp endpoints postgresql -o jsonpath='{.subsets[0].ports[0].name}')
if [ "x${FIRST_PORT_NAME}" == "xdb" ]; then
echo "matched the old endpoints: deleting old postgresql endpoints"
kubectl delete endpoints -n ucp postgresql
echo "done."
exit 0
fi
fi
sleep 5
done

11
tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh

@ -0,0 +1,11 @@
#!/bin/bash
set -x
echo "Cleaning up the patroni_endpoint_cleaner"
sudo systemctl stop patroni_endpoint_cleaner
sudo systemctl disable patroni_endpoint_cleaner
sudo rm -f /opt/patroni_endpoint_cleaner.sh
sudo rm -f /lib/systemd/system/patroni_endpoint_cleaner.service
sudo rm -f /etc/systemd/system/multi-user.target.wants/patroni_endpoint_cleaner.service
sudo systemctl daemon-reload
sudo systemctl reset-failed

22
tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh

@ -0,0 +1,22 @@
#!/bin/bash
set -ex
sudo chmod 700 patroni_endpoint_cleaner.sh
sudo cp patroni_endpoint_cleaner.sh /opt
cat > ./patroni_endpoint_cleaner.service << EOF
[Unit]
Description=Helper script for initial upgrade to HA Postgres
[Service]
ExecStart=/opt/patroni_endpoint_cleaner.sh
[Install]
WantedBy=multi-user.target
EOF
sudo mv patroni_endpoint_cleaner.service /lib/systemd/system/
sudo systemctl restart patroni_endpoint_cleaner
sudo systemctl enable patroni_endpoint_cleaner
sudo systemctl daemon-reload

23
type/sloop/charts/ucp/core/postgresql.yaml

@ -0,0 +1,23 @@
---
schema: armada/Chart/v1
metadata:
schema: metadata/Document/v1
name: ucp-postgresql
replacement: true
labels:
name: ucp-postgresql-type
layeringDefinition:
abstract: false
layer: type
parentSelector:
name: ucp-postgresql-global
actions:
- method: merge
path: .
storagePolicy: cleartext
data:
values:
pod:
replicas:
server: 1
...

4
type/sloop/config/service_accounts.yaml

@ -17,6 +17,10 @@ data:
postgres:
admin:
username: postgres
replica:
username: standby
exporter:
username: psql_exporter
oslo_db:
admin:
username: root

Loading…
Cancel
Save