diff --git a/README-backup-ops.md b/README-backup-ops.md index 3c3b3a9..cdfc812 100644 --- a/README-backup-ops.md +++ b/README-backup-ops.md @@ -3,7 +3,7 @@ The `openstack-operations` role includes some foundational backup and restore Ansible tasks to help with automatically backing up and restoring OpenStack services. The current services available to backup and restore include: * MySQL on a galera cluster -* More coming soon... +* Redis Scenarios tested: @@ -34,6 +34,10 @@ MySQL/Galera Filesystem * It has no special requirements, only the `tar` command is going to be used. +Redis +* Target Hosts needs access to the `redis` package. Tasks in the backup and restore files will attempt to install it. +* When restoring Redis, the Control Host requires the `pacemaker_resource` module. You can obtain this module from the `ansible-pacemaker` RPM. If your operating system does not have access to this package, you can clone the [ansible-pacemaker git repo](https://github.com/redhat-openstack/ansible-pacemaker). When running a restore playbook, include the `ansible-pacemaker` module using the `-M` option (e.g. `ansible-playbook -M /usr/share/ansible-modules ...`) + ## Task Files ## The following is a list of the task files used in the backup and restore process. @@ -47,12 +51,15 @@ Initialization Tasks: Backup Tasks: * `backup_mysql.yml` - Performs a backup of the OpenStack MySQL data and grants, archives them, and sends them to the desired backup host. * `backup_filesystem.yml` - Creates a tar file of a list of files/directories given and sends then to a desired backup host. +* `backup_redis.yml` - Performs a backup of Redis data from one node, archives them, and sends them to the desired backup host. Restore Tasks: * `restore_galera.yml` - Performs a restore of the OpenStack MySQL data and grants on a containerized galera cluster. This involves shutting down the current galera cluster, creating a brand new MySQL database, then importing the data and grants from the archive. In addition, the playbook saves a copy of the old data in case the restore process fails. +* `restore_redis.yml` - Performs a restore of Redis data from one node to all nodes and resets the permissions using a redis container. Validation Tasks: * `validate_galera.yml` - Performs the equivalent of `clustercheck` i.e. checks the `wsrep_local_state` is 4 ("Synced"). +* `validate_galera.yml` - Performs a Redis check with `redis-cli ping`. ## Variables ## @@ -77,6 +84,11 @@ Filesystem backup variables: * `baclup_exclude` - List of the files that where not included on the backup. * `backup_file` - The end of the backup file name. +Redis backup and restore variables: +* `redis_vip` - The VIP address of the Redis cluster. If unsent, it checks the Puppet hieradata for the VIP. +* `redis_matherauth_password` - The master password for the Redis cluster. If unsent, it checks the Puppet hieradata for the password. +* `redis_container_image` - The image to use for the temporary container that restores the permissions to the Redis data directory. If unset, it tries to determine the image from the existing redis container. + ## Inventory and Playbooks ## You ultimately define how to use the tasks with your own playbooks and inventory. The inventory should include the host groups and users to access each host type. For example: @@ -105,7 +117,7 @@ The process for your playbook depends largely on whether you want to backup or r The following examples show how to use the backup and restore tasks. -### Backup and restore galera to a remote backup server ### +### Backup and restore galera and redis to a remote backup server ### This example shows how to backup data to the `root` user on a remote backup server, and then restore it. The inventory file for both functions are the same: @@ -118,6 +130,11 @@ This example shows how to backup data to the `root` user on a remote backup serv 192.0.2.102 ansible_user=heat-admin 192.0.2.103 ansible_user=heat-admin +[redis] +192.0.2.101 ansible_user=heat-admin +192.0.2.102 ansible_user=heat-admin +192.0.2.103 ansible_user=heat-admin + [all:vars] backup_directory="/root/backup-test/" ~~~~ @@ -146,6 +163,21 @@ Backup Playbook: - import_role: name: ansible-role-openstack-operations tasks_from: disable_ssh + +- name: Backup Redis database + hosts: "{{ target_hosts | default('redis') }}[0]" + vars: + backup_server_hostgroup: "{{ backup_hosts | default('backup') }}" + tasks: + - import_role: + name: ansible-role-openstack-operations + tasks_from: enable_ssh + - import_role: + name: ansible-role-openstack-operations + tasks_from: backup_redis + - import_role: + name: ansible-role-openstack-operations + tasks_from: disable_ssh ~~~~ We do not need to include the bootstrap tasks with the backup since all tasks are performed by one of the Target Hosts. @@ -177,11 +209,29 @@ Restore Playbook: - import_role: name: ansible-role-openstack-operations tasks_from: disable_ssh + +- name: Restore Redis data + hosts: "{{ target_hosts | default('redis') }}" + vars: + backup_server_hostgroup: "{{ backup_hosts | default('backup') }}" + tasks: + - import_role: + name: ansible-role-openstack-operations + tasks_from: set_bootstrap + - import_role: + name: ansible-role-openstack-operations + tasks_from: enable_ssh + - import_role: + name: ansible-role-openstack-operations + tasks_from: restore_redis + - import_role: + name: ansible-role-openstack-operations + tasks_from: disable_ssh ~~~~ We include the bootstrap tasks with the backup since all Target Hosts are required for the restore but only certain operations are performed on one of the hosts. -### Backup and restore galera to a combined control/backup host ### +### Backup and restore galera and redis to a combined control/backup host ### This example shows how to back to a directory on the Control Host using the same user. In this case, we use the `stack` user for both Ansible and rsync operations. We also use the `heat-admin` user to access the OpenStack nodes. Both the backup and restore operations use the same inventory file: @@ -194,6 +244,11 @@ localhost ansible_user=stack 192.0.2.102 ansible_user=heat-admin 192.0.2.103 ansible_user=heat-admin +[redis] +192.0.2.101 ansible_user=heat-admin +192.0.2.102 ansible_user=heat-admin +192.0.2.103 ansible_user=heat-admin + [all:vars] backup_directory="/home/stack/backup-test/" ~~~~ @@ -219,6 +274,15 @@ Backup Playbook: - import_role: name: ansible-role-openstack-operations tasks_from: backup_mysql + +- name: Backup Redis database + hosts: "{{ target_hosts | default('redis') }}[0]" + vars: + backup_server_hostgroup: "{{ backup_hosts | default('backup') }}" + tasks: + - import_role: + name: ansible-role-openstack-operations + tasks_from: backup_redis ~~~~ Restore Playbook: @@ -245,6 +309,21 @@ Restore Playbook: - import_role: name: ansible-role-openstack-operations tasks_from: restore_galera + +- name: Restore MySQL database on galera cluster + hosts: "{{ target_hosts | default('redis') }}" + vars: + backup_server_hostgroup: "{{ backup_hosts | default('backup') }}" + tasks: + - import_role: + name: ansible-role-openstack-operations + tasks_from: set_bootstrap + - import_role: + name: ansible-role-openstack-operations + tasks_from: enable_ssh + - import_role: + name: ansible-role-openstack-operations + tasks_from: restore_redis ~~~~ In This situation, we do not include the `disable_ssh` tasks since this would disable access from the Control Host to the OpenStack nodes for future Ansible operations. diff --git a/tasks/backup_redis.yml b/tasks/backup_redis.yml new file mode 100644 index 0000000..445e7ec --- /dev/null +++ b/tasks/backup_redis.yml @@ -0,0 +1,75 @@ +# Tasks for dumping a Redis backup from each host and pulling it to the +# Backup Server. + +- name: Make sure Redis client is installed on the Target Hosts + yum: + name: redis + state: installed + +- name: Remove any existing Redis backup directory + file: + path: "{{ backup_tmp_dir }}/redis" + state: absent + +- name: Create a new Redis backup directory + file: + path: "{{ backup_tmp_dir }}/redis" + state: directory + +- name: Get the Redis masterauth password + shell: | + /bin/hiera -c /etc/puppet/hiera.yaml redis::masterauth + when: redis_masterauth_password is undefined + register: redis_masterauth_password_cmd_output + become: true + no_log: true + +- name: Convert the Redis masterauth password if unknown + set_fact: + redis_masterauth_password: "{{ redis_masterauth_password_cmd_output.stdout_lines[0] }}" + when: redis_masterauth_password is undefined + no_log: true + +- name: Get the Redis VIP + shell: | + /bin/hiera -c /etc/puppet/hiera.yaml redis_vip + when: redis_vip is undefined + register: redis_vip_cmd_output + become: true + +- name: Convert the Redis VIP if unknown + set_fact: + redis_vip: "{{ redis_vip_cmd_output.stdout_lines[0] }}" + when: redis_vip is undefined + +- name: Run the redis backup command + command: /bin/redis-cli -h {{ redis_vip }} -a {{ redis_masterauth_password }} save + no_log: true + +- name: Copy the Redis dump + copy: + src: /var/lib/redis/dump.rdb + dest: "{{ backup_tmp_dir }}/redis/dump.rdb" + remote_src: yes + become: true + +# The archive module is pretty limited. Using a shell instead. +- name: Archive the OpenStack Redis dump + shell: | + /bin/tar --ignore-failed-read --xattrs \ + -zcf {{ backup_tmp_dir }}/redis/openstack-backup-redis.tar \ + {{ backup_tmp_dir }}/redis/dump.rdb + +- name: Copy the archive to the backup server + synchronize: + mode: pull + src: "{{ backup_tmp_dir }}/redis/openstack-backup-redis.tar" + dest: "{{ backup_directory }}" + set_remote_user: false + ssh_args: "{{ backup_host_ssh_args }}" + delegate_to: "{{ backup_host }}" + +- name: Remove the Redis backup directory + file: + path: "{{ backup_tmp_dir }}/redis" + state: absent diff --git a/tasks/restore_redis.yml b/tasks/restore_redis.yml new file mode 100644 index 0000000..0365679 --- /dev/null +++ b/tasks/restore_redis.yml @@ -0,0 +1,100 @@ +# Tasks for restoring Redis backups on a cluster + +- name: Make sure Redis client is installed on the Target Hosts + yum: + name: redis + state: installed + +- name: Get the Redis container image if not user-defined + command: "/bin/bash docker ps --filter name=.*redis.* --format='{{ '{{' }} .Image {{ '}}' }}'" + when: redis_container_image is undefined + register: redis_container_image_cmd_output + become: true + +- name: Convert the Redis container image variable if unknown + set_fact: + redis_container_image: "{{ redis_container_image_cmd_output.stdout_lines[0] }}" + when: redis_container_image is undefined + +- name: Get the Redis VIP + shell: | + /bin/hiera -c /etc/puppet/hiera.yaml redis_vip + when: redis_vip is undefined + register: redis_vip_cmd_output + become: true + +- name: Convert the Redis VIP if unknown + set_fact: + redis_vip: "{{ redis_vip_cmd_output.stdout_lines[0] }}" + when: redis_vip is undefined + +- name: Remove any existing Redis backup directory + file: + path: "{{ backup_tmp_dir }}/redis" + state: absent + +- name: Create a new Redis backup directory + file: + path: "{{ backup_tmp_dir }}/redis" + state: directory + +- name: Copy Redis backup archive from the backup server + synchronize: + mode: push + src: "{{ backup_directory }}/openstack-backup-redis.tar" + dest: "{{ backup_tmp_dir }}/redis/" + set_remote_user: false + ssh_args: "{{ backup_host_ssh_args }}" + delegate_to: "{{ backup_host }}" + +- name: Unarchive the database archive + shell: | + /bin/tar --xattrs \ + -zxf {{ backup_tmp_dir }}/redis/openstack-backup-redis.tar \ + -C / + +- name: Disable redis-bundle + pacemaker_resource: + resource: redis-bundle + state: disable + wait_for_resource: true + become: true + when: bootstrap_node | bool + +- name: Delete the old Redis dump + file: + path: /var/lib/redis/dump.rdb + state: absent + become: true + +- name: Copy the new Redis dump + copy: + src: "{{ backup_tmp_dir }}/redis/dump.rdb" + dest: /var/lib/redis/dump.rdb + remote_src: yes + become: true + +- name: Create a redis_restore container to restore container-based permissions + docker_container: + name: redis_restore + user: root + detach: false + command: "/usr/bin/chown -R redis: /var/lib/redis" + image: "{{ redis_container_image }}" + volumes: + - /var/lib/redis:/var/lib/redis:rw + become: true + +- name: Remove redis_restore container + docker_container: + name: redis_restore + state: absent + become: true + +- name: Enable redis + pacemaker_resource: + resource: redis-bundle + state: enable + wait_for_resource: true + become: true + when: bootstrap_node | bool diff --git a/tasks/validate_redis.yaml b/tasks/validate_redis.yaml new file mode 100644 index 0000000..b183f7d --- /dev/null +++ b/tasks/validate_redis.yaml @@ -0,0 +1,44 @@ +- name: Get the Redis masterauth password + shell: | + /bin/hiera -c /etc/puppet/hiera.yaml redis::masterauth + when: redis_masterauth_password is undefined + register: redis_masterauth_password_cmd_output + become: true + no_log: true + +- name: Convert the Redis masterauth password if unknown + set_fact: + redis_masterauth_password: "{{ redis_masterauth_password_cmd_output.stdout_lines[0] }}" + when: redis_masterauth_password is undefined + no_log: true + +- name: Get the Redis VIP + shell: | + /bin/hiera -c /etc/puppet/hiera.yaml redis_vip + when: redis_vip is undefined + register: redis_vip_cmd_output + become: true + +- name: Convert the Redis VIP if unknown + set_fact: + redis_vip: "{{ redis_vip_cmd_output.stdout_lines[0] }}" + when: redis_vip is undefined + +- name: Perform a Redis check + command: /bin/redis-cli -h {{ redis_vip }} -a {{ redis_masterauth_password }} ping + register: redis_status_check_output + no_log: true + +- name: Convert the Redis status + set_fact: + redis_status_check: "{{ redis_status_check_output.stdout_lines[0] }}" + +- name: Fail if Redis is not running on the node + fail: + msg: "Redis not running on node: {{ inventory_hostname }}. Check the service is running on the node and try again." + when: redis_status_check != "PONG" + +- name: Report Redis success + debug: + msg: "Redis running on node: {{ inventory_hostname }}" + when: redis_status_check == "PONG" \ No newline at end of file