diff --git a/mcapi_vexxhost/playbooks/files/openstack_deploy/group_vars/k8s_all/main.yml b/mcapi_vexxhost/playbooks/files/openstack_deploy/group_vars/k8s_all/main.yml index aa50a19d..32fb0c0e 100644 --- a/mcapi_vexxhost/playbooks/files/openstack_deploy/group_vars/k8s_all/main.yml +++ b/mcapi_vexxhost/playbooks/files/openstack_deploy/group_vars/k8s_all/main.yml @@ -33,3 +33,7 @@ openstack_host_nf_conntrack_max: 1572864 # OSA containers dont run ssh by default so cannot use synchronize upload_helm_chart_method: copy + +# Enable periodic cluster API state collection (note: this is not a guaranteed functional backup) +# See https://cluster-api.sigs.k8s.io/clusterctl/commands/move +cluster_api_backups_enabled: False diff --git a/mcapi_vexxhost/playbooks/files/scripts/export-cluster-api-state.sh b/mcapi_vexxhost/playbooks/files/scripts/export-cluster-api-state.sh new file mode 100644 index 00000000..ca9a030d --- /dev/null +++ b/mcapi_vexxhost/playbooks/files/scripts/export-cluster-api-state.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +MKTEMP=$(mktemp -d) +TEMPDEST=$MKTEMP/cluster-api-backup +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +# ensure data is not readable inside working directory +mkdir --mode 0600 $TEMPDEST +umask 0037 + +# Try several times if necessary to export objects as this may fail if any cluster is changing state (e.g. being provisioned) +for retry in {1..20}; do + clusterctl move --to-directory $TEMPDEST --namespace magnum-system && break + sleep 30 +done + +# Output list of workload cluster names, such as "kube-abc12" +CLUSTER_NAMES=$(kubectl get clusters -n magnum-system --no-headers -o custom-columns=:metadata.name) +echo "$CLUSTER_NAMES" > $TEMPDEST/cluster-names.txt + +# Export cloud-config secrets from each cluster in turn +for cluster in $CLUSTER_NAMES; do + echo Exporting ${cluster}-cloud-config + kubectl get secret -n magnum-system ${cluster}-cloud-config -o jsonpath='{.data.cacert}' | base64 --decode > $TEMPDEST/cacert-${cluster}.txt + kubectl get secret -n magnum-system ${cluster}-cloud-config -o jsonpath='{.data.clouds\.yaml}' | base64 --decode > $TEMPDEST/clouds-yaml-${cluster}.txt +done + +tar -czvf $SCRIPT_DIR/cluster-api-objects.tar.gz -C $MKTEMP cluster-api-backup +chmod g+r $SCRIPT_DIR/cluster-api-objects.tar.gz + +echo Cluster API objects saved in $SCRIPT_DIR + +# Make sure the temp directory gets removed on script exit. +trap "exit 1" HUP INT PIPE QUIT TERM +trap 'rm -rf "$MKTEMP"' EXIT diff --git a/mcapi_vexxhost/playbooks/files/scripts/import-cluster-api-state.sh b/mcapi_vexxhost/playbooks/files/scripts/import-cluster-api-state.sh new file mode 100644 index 00000000..aac73ea5 --- /dev/null +++ b/mcapi_vexxhost/playbooks/files/scripts/import-cluster-api-state.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +echo Unpacking tarball cluster-api-objects.tar.gz +tar -xvf cluster-api-objects.tar.gz -C $SCRIPT_DIR + +# Separate out the secrets from the yaml files +mkdir $SCRIPT_DIR/secrets +mv $SCRIPT_DIR/cluster-api-backup/*.txt $SCRIPT_DIR/secrets/ + +echo Restoring Cluster API objects from yaml files in the backup +clusterctl move --from-directory $SCRIPT_DIR/cluster-api-backup/ + +# Iterate over .txt files for each cluster and recreate k8s secrets +while read clustername; do + echo Creating secret ${clustername}-cloud-config + kubectl create secret -n magnum-system generic ${clustername}-cloud-config --from-file=cacert=$SCRIPT_DIR/secrets/cacert-${clustername}.txt --from-file=clouds\.yaml=$SCRIPT_DIR/secrets/clouds-yaml-${clustername}.txt +done <$SCRIPT_DIR/secrets/cluster-names.txt + +# Delete the secrets files +rm -rf $SCRIPT_DIR/secrets diff --git a/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml b/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml index 701444d8..abf36a90 100644 --- a/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml +++ b/mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml @@ -141,3 +141,69 @@ when: _k8s_is_first_play_host tags: - cluster-api + +- name: Set up Cluster API backups + hosts: k8s_all + gather_facts: true + user: root + vars: + cluster_api_backups_path: "/var/backup/cluster-api-backup" + cluster_api_backups_group_name: "backups" + # cluster_api_backups_group_gid: "" + cluster_api_backups_init_overrides: {} + cluster_api_backups_on_calendar: "*-*-* 00:00:00" + cluster_api_backups_randomized_delay_sec: 0 + cluster_api_backups_nodes: "{{ [(groups['k8s_all'] | select('in', ansible_play_hosts)) | last] }}" + tasks: + - name: Set up Cluster API backups + block: + - name: Ensure group backups exists + ansible.builtin.group: + name: "{{ cluster_api_backups_group_name }}" + state: present + gid: "{{ cluster_api_backups_group_gid | default(omit) }}" + + - name: Create Cluster API backup directory + ansible.builtin.file: + path: "{{ cluster_api_backups_path }}" + state: "directory" + group: "{{ cluster_api_backups_group_name }}" + mode: "0750" + + - name: Copy Cluster API backup scripts + ansible.builtin.copy: + src: "scripts/{{ item }}" + dest: "{{ cluster_api_backups_path }}/{{ item }}" + mode: "0755" + with_items: + - "export-cluster-api-state.sh" + - "import-cluster-api-state.sh" + + - name: Create service and timer for backups + ansible.builtin.import_role: + name: systemd_service + vars: + systemd_service_enabled: true + systemd_service_restart_changed: false + systemd_overrides: "{{ cluster_api_backups_init_overrides }}" + systemd_group_name: "{{ cluster_api_backups_group_name }}" + systemd_services: + - service_name: "cluster-api-backup" + execstarts: + - "{{ cluster_api_backups_path }}/export-cluster-api-state.sh" + environment: + UMASK: "0640" + UMASK_DIR: "0750" + timer: + state: "started" + options: + OnCalendar: "{{ cluster_api_backups_on_calendar }}" + RandomizedDelaySec: "{{ cluster_api_backups_randomized_delay_sec }}" + Persistent: true + Unit: "cluster-api-backup.service" + + when: + - cluster_api_backups_enabled is defined and cluster_api_backups_enabled | bool + - inventory_hostname in cluster_api_backups_nodes + tags: + - backups