Incorporate HA PostgreSQL
This change integrates changes related to the new Patroni-enabled high availability included in the postgresql helm chart. 1. Updates the version of the postgresql chart 2. Adds users/passphrases for the postgresql replication and exporter users 3. Sets the default number of replicas to 3 (except for skiff & sloop) 4. Adds documentation/scripting for upgrading the chart Co-Authored-By: Evgeniy L <eli@mirantis.com> Change-Id: I51e63fb2444610cfb144bc587a6a179fa4830809
This commit is contained in:
Matt McEuen
Kaspars Skels
parent
b15899667f
commit
16cdf842d2
@ -40,6 +40,18 @@ metadata:
|
||||
path: .ucp.postgres.admin
|
||||
dest:
|
||||
path: .values.endpoints.postgresql.auth.admin
|
||||
- src:
|
||||
schema: pegleg/AccountCatalogue/v1
|
||||
name: ucp_service_accounts
|
||||
path: .ucp.postgres.replica
|
||||
dest:
|
||||
path: .values.endpoints.postgresql.auth.replica
|
||||
- src:
|
||||
schema: pegleg/AccountCatalogue/v1
|
||||
name: ucp_service_accounts
|
||||
path: .ucp.postgres.exporter
|
||||
dest:
|
||||
path: .values.endpoints.postgresql.auth.exporter
|
||||
|
||||
# Secrets
|
||||
- dest:
|
||||
@ -48,17 +60,31 @@ metadata:
|
||||
schema: deckhand/Passphrase/v1
|
||||
name: ucp_postgres_admin_password
|
||||
path: .
|
||||
- dest:
|
||||
path: .values.endpoints.postgresql.auth.replica.password
|
||||
src:
|
||||
schema: deckhand/Passphrase/v1
|
||||
name: ucp_postgres_replication_password
|
||||
path: .
|
||||
- dest:
|
||||
path: .values.endpoints.postgresql.auth.exporter.password
|
||||
src:
|
||||
schema: deckhand/Passphrase/v1
|
||||
name: ucp_postgres_exporter_password
|
||||
path: .
|
||||
data:
|
||||
chart_name: ucp-postgresql
|
||||
release: ucp-postgresql
|
||||
namespace: ucp
|
||||
wait:
|
||||
timeout: 600
|
||||
timeout: 1800
|
||||
labels:
|
||||
release_group: airship-ucp-postgresql
|
||||
install:
|
||||
no_hooks: false
|
||||
upgrade:
|
||||
options:
|
||||
force: true
|
||||
no_hooks: false
|
||||
pre:
|
||||
delete:
|
||||
@ -69,6 +95,9 @@ data:
|
||||
post:
|
||||
create: []
|
||||
values:
|
||||
pod:
|
||||
replicas:
|
||||
server: 3
|
||||
conf:
|
||||
postgresql:
|
||||
max_connections: 1000
|
||||
|
||||
@ -463,7 +463,7 @@ data:
|
||||
type: git
|
||||
postgresql:
|
||||
location: https://opendev.org/openstack/openstack-helm-infra
|
||||
reference: 5e1ecd9840397bf9e8829ce0d98fcb721db1b74e
|
||||
reference: 09ae22d8493d5cef34c80cb69117c69dc0f2dc8e
|
||||
subpath: postgresql
|
||||
type: git
|
||||
postgresql-htk:
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_exporter_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_replication_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_exporter_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_replication_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
21
site/airskiff/software/charts/ucp/core/postgresql.yaml
Normal file
21
site/airskiff/software/charts/ucp/core/postgresql.yaml
Normal file
@ -0,0 +1,21 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp-postgresql
|
||||
replacement: true
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
parentSelector:
|
||||
name: ucp-postgresql-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
storagePolicy: cleartext
|
||||
data:
|
||||
values:
|
||||
pod:
|
||||
replicas:
|
||||
server: 1
|
||||
...
|
||||
@ -15,6 +15,10 @@ data:
|
||||
postgres:
|
||||
admin:
|
||||
username: postgres
|
||||
replica:
|
||||
username: standby
|
||||
exporter:
|
||||
username: psql_exporter
|
||||
oslo_db:
|
||||
admin:
|
||||
username: root
|
||||
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_exporter_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_replication_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: airsloop123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_exporter_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -0,0 +1,11 @@
|
||||
---
|
||||
schema: deckhand/Passphrase/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp_postgres_replication_password
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: site
|
||||
storagePolicy: cleartext
|
||||
data: password123
|
||||
...
|
||||
@ -15,6 +15,10 @@ data:
|
||||
postgres:
|
||||
admin:
|
||||
username: postgres
|
||||
replica:
|
||||
username: standby
|
||||
exporter:
|
||||
username: psql_exporter
|
||||
oslo_db:
|
||||
admin:
|
||||
username: root
|
||||
|
||||
24
tools/upgrades/postgresql/README.md
Normal file
24
tools/upgrades/postgresql/README.md
Normal file
@ -0,0 +1,24 @@
|
||||
# PostgreSQL Patroni Upgrade Scripts
|
||||
|
||||
Upgrading a live site from the old, unclustered PostgreSQL chart to the newer,
|
||||
Patroni-managed version takes a small amount of out-of-band scripting to ensure
|
||||
a smooth hands-free upgrade.
|
||||
|
||||
## Prior to upgrade
|
||||
|
||||
The ``patroni_endpoint_cleaner_unit.sh`` script should be run prior to upgrading
|
||||
the postgresql chart. It installs a systemd unit which in turn will run
|
||||
the ``patroni_endpoint_cleaner.sh`` script. During chart upgrade, the script
|
||||
will delete the postgresql endpoints, allowing Patroni to recreate them with the
|
||||
appropriate annotations for it to manage them ongoing.
|
||||
|
||||
This documentation project outlines a reference architecture for automated
|
||||
cloud provisioning and management, leveraging a collection of interoperable
|
||||
open-source tools.
|
||||
|
||||
## Post upgrade
|
||||
|
||||
After the chart upgrade is complete, the ``patroni_endpoint_cleaner_remove.sh``
|
||||
script should be run. This will simply clean up the systemd unit that was
|
||||
created previously.
|
||||
|
||||
36
tools/upgrades/postgresql/patroni_endpoint_cleaner.sh
Executable file
36
tools/upgrades/postgresql/patroni_endpoint_cleaner.sh
Executable file
@ -0,0 +1,36 @@
|
||||
#!/bin/bash
|
||||
|
||||
# This script should be run as a one-time fix DURING an upgrade of
|
||||
# "vanilla" postgres to patroni (in either a single or HA multi-replica
|
||||
# configuration).
|
||||
#
|
||||
# This addresses an issue where the previous version of the chart had a
|
||||
# service-managed `endpoints` object, while patroni needs to manage its
|
||||
# own kubernetes `endpoints`. Patroni won't successfully manage
|
||||
# (i.e. apply annotation to, etc) the postgresql endpoints until the
|
||||
# service-managed endpoints are out of the way; however deletion of the
|
||||
# postgresql endpoints must be done with care during an upgrade.
|
||||
#
|
||||
# This script watches for the right moment and deletes the endpoints.
|
||||
|
||||
export KUBECONFIG=${KUBECONFIG:-"/etc/kubernetes/admin.conf"}
|
||||
|
||||
while true; do
|
||||
echo "Checking to see if patroni is deployed..."
|
||||
# Wait for the patroni-based chart to get deployed
|
||||
if [ $(kubectl describe pod -n ucp postgresql-0 | grep -c "patroni") -gt 0 ]; then
|
||||
echo 'Detected that patroni is deployed'
|
||||
|
||||
# The port name used by the single-node postgres chart is "db",
|
||||
# while the new port name is "postgres"
|
||||
FIRST_PORT_NAME=$(kubectl get -n ucp endpoints postgresql -o jsonpath='{.subsets[0].ports[0].name}')
|
||||
if [ "x${FIRST_PORT_NAME}" == "xdb" ]; then
|
||||
echo "matched the old endpoints: deleting old postgresql endpoints"
|
||||
kubectl delete endpoints -n ucp postgresql
|
||||
echo "done."
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
|
||||
sleep 5
|
||||
done
|
||||
11
tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh
Executable file
11
tools/upgrades/postgresql/patroni_endpoint_cleaner_remove.sh
Executable file
@ -0,0 +1,11 @@
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
echo "Cleaning up the patroni_endpoint_cleaner"
|
||||
sudo systemctl stop patroni_endpoint_cleaner
|
||||
sudo systemctl disable patroni_endpoint_cleaner
|
||||
sudo rm -f /opt/patroni_endpoint_cleaner.sh
|
||||
sudo rm -f /lib/systemd/system/patroni_endpoint_cleaner.service
|
||||
sudo rm -f /etc/systemd/system/multi-user.target.wants/patroni_endpoint_cleaner.service
|
||||
sudo systemctl daemon-reload
|
||||
sudo systemctl reset-failed
|
||||
22
tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh
Executable file
22
tools/upgrades/postgresql/patroni_endpoint_cleaner_unit.sh
Executable file
@ -0,0 +1,22 @@
|
||||
#!/bin/bash
|
||||
set -ex
|
||||
|
||||
sudo chmod 700 patroni_endpoint_cleaner.sh
|
||||
sudo cp patroni_endpoint_cleaner.sh /opt
|
||||
|
||||
cat > ./patroni_endpoint_cleaner.service << EOF
|
||||
[Unit]
|
||||
Description=Helper script for initial upgrade to HA Postgres
|
||||
|
||||
[Service]
|
||||
ExecStart=/opt/patroni_endpoint_cleaner.sh
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
EOF
|
||||
|
||||
sudo mv patroni_endpoint_cleaner.service /lib/systemd/system/
|
||||
|
||||
sudo systemctl restart patroni_endpoint_cleaner
|
||||
sudo systemctl enable patroni_endpoint_cleaner
|
||||
sudo systemctl daemon-reload
|
||||
23
type/sloop/charts/ucp/core/postgresql.yaml
Normal file
23
type/sloop/charts/ucp/core/postgresql.yaml
Normal file
@ -0,0 +1,23 @@
|
||||
---
|
||||
schema: armada/Chart/v1
|
||||
metadata:
|
||||
schema: metadata/Document/v1
|
||||
name: ucp-postgresql
|
||||
replacement: true
|
||||
labels:
|
||||
name: ucp-postgresql-type
|
||||
layeringDefinition:
|
||||
abstract: false
|
||||
layer: type
|
||||
parentSelector:
|
||||
name: ucp-postgresql-global
|
||||
actions:
|
||||
- method: merge
|
||||
path: .
|
||||
storagePolicy: cleartext
|
||||
data:
|
||||
values:
|
||||
pod:
|
||||
replicas:
|
||||
server: 1
|
||||
...
|
||||
@ -17,6 +17,10 @@ data:
|
||||
postgres:
|
||||
admin:
|
||||
username: postgres
|
||||
replica:
|
||||
username: standby
|
||||
exporter:
|
||||
username: psql_exporter
|
||||
oslo_db:
|
||||
admin:
|
||||
username: root
|
||||
|
||||
Loading…
Reference in New Issue
Block a user