Merge "Add backup capability to Cluster API setup"

mcapi_vexxhost/playbooks/files/openstack_deploy/group_vars/k8s_all/main.yml

@@ -33,3 +33,7 @@ openstack_host_nf_conntrack_max: 1572864
 
 # OSA containers dont run ssh by default so cannot use synchronize
 upload_helm_chart_method: copy
+
+# Enable periodic cluster API state collection (note: this is not a guaranteed functional backup)
+# See https://cluster-api.sigs.k8s.io/clusterctl/commands/move
+cluster_api_backups_enabled: False

mcapi_vexxhost/playbooks/files/scripts/export-cluster-api-state.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+
+MKTEMP=$(mktemp -d)
+TEMPDEST=$MKTEMP/cluster-api-backup
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+# ensure data is not readable inside working directory
+mkdir --mode 0600 $TEMPDEST
+umask 0037
+
+# Try several times if necessary to export objects as this may fail if any cluster is changing state (e.g. being provisioned)
+for retry in {1..20}; do
+  clusterctl move --to-directory $TEMPDEST --namespace magnum-system && break
+  sleep 30
+done
+
+# Output list of workload cluster names, such as "kube-abc12"
+CLUSTER_NAMES=$(kubectl get clusters -n magnum-system --no-headers -o custom-columns=:metadata.name)
+echo "$CLUSTER_NAMES" > $TEMPDEST/cluster-names.txt
+
+# Export cloud-config secrets from each cluster in turn
+for cluster in $CLUSTER_NAMES; do
+  echo Exporting ${cluster}-cloud-config
+  kubectl get secret -n magnum-system ${cluster}-cloud-config  -o jsonpath='{.data.cacert}' | base64 --decode > $TEMPDEST/cacert-${cluster}.txt
+  kubectl get secret -n magnum-system ${cluster}-cloud-config  -o jsonpath='{.data.clouds\.yaml}' | base64 --decode > $TEMPDEST/clouds-yaml-${cluster}.txt
+done
+
+tar -czvf $SCRIPT_DIR/cluster-api-objects.tar.gz -C $MKTEMP cluster-api-backup
+chmod g+r $SCRIPT_DIR/cluster-api-objects.tar.gz
+
+echo Cluster API objects saved in $SCRIPT_DIR
+
+# Make sure the temp directory gets removed on script exit.
+trap "exit 1"           HUP INT PIPE QUIT TERM
+trap 'rm -rf "$MKTEMP"' EXIT

mcapi_vexxhost/playbooks/files/scripts/import-cluster-api-state.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+
+SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+echo Unpacking tarball cluster-api-objects.tar.gz
+tar -xvf cluster-api-objects.tar.gz -C $SCRIPT_DIR
+
+# Separate out the secrets from the yaml files
+mkdir $SCRIPT_DIR/secrets
+mv $SCRIPT_DIR/cluster-api-backup/*.txt $SCRIPT_DIR/secrets/
+
+echo Restoring Cluster API objects from yaml files in the backup
+clusterctl move --from-directory $SCRIPT_DIR/cluster-api-backup/
+
+# Iterate over .txt files for each cluster and recreate k8s secrets
+while read clustername; do
+  echo Creating secret ${clustername}-cloud-config
+  kubectl create secret -n magnum-system generic ${clustername}-cloud-config --from-file=cacert=$SCRIPT_DIR/secrets/cacert-${clustername}.txt --from-file=clouds\.yaml=$SCRIPT_DIR/secrets/clouds-yaml-${clustername}.txt
+done <$SCRIPT_DIR/secrets/cluster-names.txt
+
+# Delete the secrets files
+rm -rf $SCRIPT_DIR/secrets

mcapi_vexxhost/playbooks/mcapi_control_plane_k8s.yml

@@ -141,3 +141,69 @@
       when: _k8s_is_first_play_host
   tags:
     - cluster-api
+
+- name: Set up Cluster API backups
+  hosts: k8s_all
+  gather_facts: true
+  user: root
+  vars:
+    cluster_api_backups_path: "/var/backup/cluster-api-backup"
+    cluster_api_backups_group_name: "backups"
+    # cluster_api_backups_group_gid: ""
+    cluster_api_backups_init_overrides: {}
+    cluster_api_backups_on_calendar: "*-*-* 00:00:00"
+    cluster_api_backups_randomized_delay_sec: 0
+    cluster_api_backups_nodes: "{{ [(groups['k8s_all'] | select('in', ansible_play_hosts)) | last] }}"
+  tasks:
+    - name: Set up Cluster API backups
+      block:
+        - name: Ensure group backups exists
+          ansible.builtin.group:
+            name: "{{ cluster_api_backups_group_name }}"
+            state: present
+            gid: "{{ cluster_api_backups_group_gid | default(omit) }}"
+
+        - name: Create Cluster API backup directory
+          ansible.builtin.file:
+            path: "{{ cluster_api_backups_path }}"
+            state: "directory"
+            group: "{{ cluster_api_backups_group_name }}"
+            mode: "0750"
+
+        - name: Copy Cluster API backup scripts
+          ansible.builtin.copy:
+            src: "scripts/{{ item }}"
+            dest: "{{ cluster_api_backups_path }}/{{ item }}"
+            mode: "0755"
+          with_items:
+            - "export-cluster-api-state.sh"
+            - "import-cluster-api-state.sh"
+
+        - name: Create service and timer for backups
+          ansible.builtin.import_role:
+            name: systemd_service
+          vars:
+            systemd_service_enabled: true
+            systemd_service_restart_changed: false
+            systemd_overrides: "{{ cluster_api_backups_init_overrides }}"
+            systemd_group_name: "{{ cluster_api_backups_group_name }}"
+            systemd_services:
+              - service_name: "cluster-api-backup"
+                execstarts:
+                  - "{{ cluster_api_backups_path }}/export-cluster-api-state.sh"
+                environment:
+                  UMASK: "0640"
+                  UMASK_DIR: "0750"
+                timer:
+                  state: "started"
+                  options:
+                    OnCalendar: "{{ cluster_api_backups_on_calendar }}"
+                    RandomizedDelaySec: "{{ cluster_api_backups_randomized_delay_sec }}"
+                    Persistent: true
+                    Unit: "cluster-api-backup.service"
+
+      when:
+        - cluster_api_backups_enabled is defined and cluster_api_backups_enabled | bool
+        - inventory_hostname in cluster_api_backups_nodes
+  tags:
+    - backups