add encryption to backup/restore playbooks

Add new option to encrypt the backup tarballs.  Initially disabled by
default with the backup_encryption_enabled boolean value.  When enabled
then backup_encryption_passphrase is required.

A third variable backup_encyption_include is a list of string
identifiers indicating which tarballs are included in the feature. The
backup_encryption_enabled and backup_encyption_include options allow the
feature to be rolled out in phases.

The existing backup tarballs include: platform, openstack, user_images,
dc_vault, registry, and most recently: hc_vault.  Of these, the image
registry backup files are unlikely to require encryption.  The hc_vault
and platform tarballs are selected initially to support encryption for
the snapshot of HC (Hashicorp) Vault application.

Test Plan:
PASS  AIO-SX backup and restore, remotely
PASS  AIO-SX backup and restore, remotely, with hc_vault
PASS  AIO-SX restore, local (on_box_data=true), with hc_vault
PASS  ansible-lint, yamllint
PASS  unit test
PASS  AIO-SX (optimized)
PASS  duplex backup playbook
n/a   duplex restore playbook
      PASS-ish: duplex restore up to known bug "DX B&R failed in restore
      ansible by 'platform service in region not found'"
PASS  remote playbook execution
PASS  local playbook execution
PASS  with/without vault applied
PASS  with/without encryption

Story: 2011073
Task: 49842

Change-Id: If98300d3b75102c11f8ad08698b55442688cf442
Signed-off-by: Michel Thebeau <michel.thebeau@windriver.com>

playbookconfig/src/playbooks/host_vars/backup-restore/default.yml

@@ -61,6 +61,26 @@ backup_dir: /opt/backups
 # regardless of where the restore playbook is executed (locally vs remotely)
 target_backup_dir: /opt/platform-backup
 
+# Enable encryption of backup files.  The default is false until this
+# feature is activated.
+backup_encryption_enabled: false
+
+# A passphrase with which to encrypt.  Required when
+# backup_encryption_enabled is true
+backup_encryption_passphrase: ""
+
+# A list of identifiers indicating which backup files to encrypt:
+#   [platform, openstack, user_images, dc_vault, registry, hc_vault]
+backup_encyption_include:
+  - platform
+  - hc_vault
+
+# Internal boolean variables for encryption to simplify logic.  These
+# will be adjusted later when the overriden parameters above are
+# considered.
+platform_tarball_encrypted: false
+hc_vault_tarball_encrypted: false
+
 # The platform backup tarball will be named in this format:
 # <platform_backup_filename_prefix>_<timestamp>.tgz
 #

playbookconfig/src/playbooks/roles/backup-restore/stage-backup-archives/tasks/main.yml

@@ -8,20 +8,73 @@
 #   This role stages the backup archives for later usage.
 
 - name: Transfer backup tarballs to target if the file is off-box
+  block:
+  - name: Transfer backup tarballs to target
     include_role:
       name: backup-restore/transfer-file
+
+  - name: Record the new location of encrypted file
+    set_fact:
+      encrypted_backup_filepath: "{{ target_backup_dir }}/{{ encrypted_backup_filename }}"
+    when: platform_tarball_encrypted|bool
+
   when: on_box_data|bool == false
 
+- name: Decrypt the file
+  block:
+  # we already know the file can be decrypted when this is running
+  - name: Ensure the decrypted file is absent
+    file:
+      path: "{{ decrypted_backup_filepath }}"
+      state: absent
+
+  - name: Set the transferred encrypted file's location
+    # when off_box_data or initial_backup_dir == target_backup_dir
+    set_fact:
+      encrypted_file_location: "{{ target_backup_dir }}"
+
+  - name: Set the original encrypted file's location
+    set_fact:
+      encrypted_file_location: "{{ initial_backup_dir }}"
+    when:
+    - on_box_data|bool
+    - initial_backup_dir != target_backup_dir
+
+  - name: decrypt the file
+    import_role:
+      name: decrypt
+    vars:
+      decrypt_file: "{{ encrypted_file_location }}/{{ backup_filename }}"
+      decrypt_output_file: "{{ decrypted_backup_filepath }}"
+      decrypt_passphrase: "{{ backup_encryption_passphrase }}"
+
+  - name: change ownership of the tarball
+    file:
+      path: "{{ decrypted_backup_filepath }}"
+      owner: root
+      group: root
+      mode: 0644
+
+  - name: Set image platform backup fqpn
+    set_fact:
+      platform_backup_fqpn: "{{ decrypted_backup_filepath }}"
+  when: platform_tarball_encrypted|bool
+
 - name: Link the backup tarballs to {{ target_backup_dir }} if the file is already on-box
   block:
-    - file:
+    - name: Link the platform tarball if it was not decrypted
+      block:
+      - name: Ensure the link file doesn't exist
+        file:
           path: "{{ target_backup_dir }}/{{ backup_filename }}"
           state: absent
 
-    - file:
+      - name: Link the platform file
+        file:
           src: "{{ initial_backup_dir }}/{{ backup_filename }}"
           dest: "{{ target_backup_dir }}/{{ backup_filename }}"
           state: link
+      when: not platform_tarball_encrypted|bool
 
     - file:
         path: "{{ target_backup_dir }}/{{ registry_backup_filename }}"
@@ -39,6 +92,7 @@
 - name: Set image platform backup fqpn
   set_fact:
     platform_backup_fqpn: "{{ target_backup_dir }}/{{ backup_filename }}"
+  when: not platform_tarball_encrypted|bool
 
 - name: Set image registry backup fqpn
   set_fact:

playbookconfig/src/playbooks/roles/backup/backup-system/tasks/main.yml

@@ -19,6 +19,33 @@
     include_hc_vault: "{{ backup_hc_vault | bool }}"
     omit_hc_vault: false
 
+# The feature is enabled by backup_encryption_enabled variable.
+# The backup tarballs are included in the feature by
+# backup_encyption_include variable.
+# The user's decision to encrypt is registered with the supplied
+# passphrase, backup_encryption_passphrase.
+#
+# The defaults were set in host_vars/backup-restore/default.yml
+- name: Interpret variables for enabling encryption
+  block:
+  - name: Fail when a passphrase is omitted
+    fail:
+      msg: >
+        A passphrase is required for encryption; set variable override
+        backup_encryption_passphrase.  To disable encryption set
+        override backup_encryption_enabled=false
+    when: backup_encryption_passphrase | length == 0
+
+  - name: Set platform tar encryption enabled
+    set_fact:
+      platform_tarball_encrypted: true
+    when: '"platform" in backup_encyption_include'
+
+  - name: Set HC Vault tar encryption enabled
+    set_fact:
+      hc_vault_tarball_encrypted: true
+    when: '"hc_vault" in backup_encyption_include'
+  when: backup_encryption_enabled|bool
 
 - name: Do StarlingX backup
   block:
@@ -581,8 +608,8 @@
           name: vault/vault_backup
         vars:
           vault_backup_dir: "{{ hc_vault_dir.path }}"
-          vault_encrypt: false
+          vault_encrypt: "{{ hc_vault_tarball_encrypted|bool }}"
-          encrypt_hc_vault_secret: ""
+          encrypt_hc_vault_secret: "{{ backup_encryption_passphrase }}"
           op_mode: "platform"
 
       - name: Find result files
@@ -814,16 +841,41 @@
       failed_when: tar_cmd.rc >= 2 or tar_cmd.rc < 0
       when: include_hc_vault | bool
 
-    - name: Notify the user backup tar file(s) are available
+    - name: Set default backup files absolute path
+      set_fact:
+        platform_backup_file_path_final: "{{ platform_backup_file_path }}"
+
+    - name: Encrypt the platform tar file
+      block:
+      - name: Set platform backup file final path with gpg extension
+        set_fact:
+          platform_backup_file_path_final: "{{ platform_backup_file_path }}.gpg"
+
+      - name: Import encrypt for platform
+        import_role:
+          name: encrypt
+        vars:
+          encrypt_file: "{{ platform_backup_file_path }}"
+          encrypt_output_file: "{{ platform_backup_file_path_final }}"
+          encrypt_passphrase: "{{ backup_encryption_passphrase }}"
+          encrypt_shred: true
+      when: platform_tarball_encrypted|bool
+
+    # The sensitive data of HC vault is encrypted by the included
+    # vault/vault_backup role. This is done such that the sensitive data
+    # is encrypted before reaching the disk. For this reason we omit
+    # encrypting the tarball at hc_vault_backup_file_path
+
+    - name: Notify the user backup file(s) are available
       debug:
         msg: >-
-          Backup tar file(s) are now available in {{ backup_dir }} on the active controller.
+          Backup file(s) are now available in {{ backup_dir }} on the active controller.
 
     - block:
-      - name: Transfer platform backup tar file to the local machine
+      - name: Transfer platform backup file to the local machine
         synchronize:
           mode: pull
-          src: "{{ platform_backup_file_path }}"
+          src: "{{ platform_backup_file_path_final }}"
           dest: "{{ host_backup_dir }}/"
         register: backup_transfer
         until: backup_transfer.rc == 0
@@ -852,19 +904,19 @@
       # Maybe use synchronize module after upgrading ansible, backup-restore/transfer-file
       # role shows a github issue regarding why it can't be used now.
 
-      - name: Notify the user where the backup tar file(s) can be found
+      - name: Notify the user where the backup file(s) can be found
         debug:
           msg: >-
-            Backup tar file(s) have been transferred to {{ host_backup_dir }} on Ansible control host.
+            Backup file(s) have been transferred to {{ host_backup_dir }} on Ansible control host.
       when: inventory_hostname != 'localhost'
 
-    - name: Find the backup tar files
+    - name: Find the backup files
       find:
         paths: "{{ backup_dir }}"
-        patterns: '*backup_*.tgz'
+        patterns: '*backup_*.tgz,*backup_*.tgz.gpg'
       register: backup_files_perm_change
 
-    - name: change permission of backup tar files
+    - name: change permission of backup files
       file:
         path: "{{ item.path }}"
         owner: root
@@ -873,10 +925,11 @@
       with_items: "{{ backup_files_perm_change.files }}"
 
   always:
-    - name: Remove the temp dir
+    - name: Shred and remove the temp dir
-      file:
+      import_role:
-        path: "{{ tempdir.path }}"
+        name: shred
-        state: absent
+      vars:
+        shred_path: "{{ tempdir.path }}"
       when: tempdir is defined and tempdir.path is defined
 
     - name: Remove the backup in progress flag file

playbookconfig/src/playbooks/roles/optimized-restore/complete-restore/tasks/main.yml

@@ -21,6 +21,18 @@
       - "{{ registry_backup_fqpn|default('') }}"
   when: initial_backup_dir != target_backup_dir
 
+- block:
+  - set_fact:
+      _encrypted_remove_items:
+        - "{{ encrypted_backup_filepath }}"
+    when: not on_box_data|bool
+
+  - set_fact:
+      _decrypted_remove_items:
+        - "{{ decrypted_backup_filepath }}"
+    when: initial_backup_dir == target_backup_dir
+  when: platform_tarball_encrypted|bool
+
 - set_fact:
     _upgrade_remove_items:
       - "{{ temp_upgrade_platform_dir }}"
@@ -34,7 +46,10 @@
     state: absent
   loop: >
     {{
-      _remove_items + _restore_remove_items|default([]) + _upgrade_remove_items|default([])
+      _remove_items + _restore_remove_items|default([])
+      + _upgrade_remove_items|default([])
+      + _encrypted_remove_items|default([])
+      + _decrypted_remove_items|default([])
       | reject('equalto', '')
     }}
 

playbookconfig/src/playbooks/roles/optimized-restore/prepare-env/tasks/main.yml

@@ -47,14 +47,23 @@
 
 - name: Set image backup filename
   block:
+    - name: Copy backup_filename
+      set_fact:
+        tmp_filename: "{{ backup_filename }}"
+
+    - name: Strip gpg extension when the backup_filename is encrypted
+      set_fact:
+        tmp_filename: "{{ tmp_filename | regex_replace('[.]gpg$') }}"
+      when: backup_encryption_enabled|bool
+
     - name: Set image backup filename for user images method
       set_fact:
-        registry_backup_filename: "{{ backup_filename.replace(default_backup_mark, default_user_images_backup_mark) }}"
+        registry_backup_filename: "{{ tmp_filename.replace(default_backup_mark, default_user_images_backup_mark) }}"
       when: restore_user_images
 
     - name: Set image backup filename for registry backup method
       set_fact:
-        registry_backup_filename: "{{ backup_filename.replace(default_backup_mark, default_registry_backup_mark) }}"
+        registry_backup_filename: "{{ tmp_filename.replace(default_backup_mark, default_registry_backup_mark) }}"
       when: restore_registry_filesystem
 
   when: not registry_backup_filename|default(none)

playbookconfig/src/playbooks/roles/restore-platform/prepare-env/tasks/main.yml

@@ -15,28 +15,105 @@
 #     5. Create restore_in_progress flag.
 #
 
-- block:
+- name: Set default local file path
+  set_fact:
+    local_backup_file_path: "{{ initial_backup_dir }}/{{ backup_filename }}"
+
+- name: Set default local file extraction command
+  set_fact:
+    extract_command_method: "cat '{{ local_backup_file_path }}'"
+
+- name: Set default filename for restore procedure
+  set_fact:
+    decrypted_backup_filename: "{{ backup_filename }}"
+    decrypted_backup_filepath: "{{ target_backup_dir }}/{{ backup_filename }}"
+
+- name: Encryption variables
+  block:
+  - name: retrieve backup file type
+    command: "file {{ local_backup_file_path }}"
+    register: local_backup_file_type
+
+  - name: Assert that backup_filename is an encrypted file
+    assert:
+      that:
+      - '"GPG symmetrically encrypted data" in local_backup_file_type.stdout'
+      fail_msg: >
+        Platform backup file {{ local_backup_file_path }} is not encrypted
+
+  - name: Set platform tarball is encrypted boolean
+    set_fact:
+      platform_tarball_encrypted: true
+
+  - name: Copy name of encrypted platform file
+    set_fact:
+      encrypted_backup_filename: "{{ backup_filename }}"
+      encrypted_backup_filepath: "{{ local_backup_file_path }}"
+
+  - name: adjust filename for restore procedure
+    set_fact:
+      decrypted_backup_filename: "{{ backup_filename }}.tar.gz"
+      decrypted_backup_filepath: "{{ target_backup_dir }}/{{ backup_filename }}.tar.gz"
+
+  - name: Set encrypted file extraction command
+    set_fact:
+      extract_command_method: >-
+          gpg --no-symkey-cache -q
+          --passphrase-fd 0 --batch --pinentry-mode loopback
+          --decrypt '{{ local_backup_file_path }}'
+  when:
+    - backup_encryption_enabled|bool
+    - '"platform" in backup_encyption_include'
+
+- name: Inspect the platform backup file locally
+  block:
+  - name: Get the file stat of backup_filename
+    stat:
+      path: "{{ local_backup_file_path }}"
+    register: local_backup_file_stat
+
+  - name: Assert that backup_filename exists
+    assert:
+      that: local_backup_file_stat.stat.exists
+      fail_msg: >
+        Platform backup_filename {{ local_backup_file_path }} does not exist
+
   - name: Look for override backup file in the backup tarball
-    shell: "tar --use-compress-program=pigz -tf {{ initial_backup_dir }}/{{ backup_filename }} | grep '_override_backup.yml'"
+    shell:
+      cmd: >-
+          {{ extract_command_method }}
+          | tar --use-compress-program=pigz -t
+          | grep '_override_backup.yml'
+      stdin: "{{ backup_encryption_passphrase }}"
     args:
       warn: false
     failed_when: false
     register: search_result
 
   - name: Fail if override file is missing
+    block:
+    - name: Fail with default message
       fail:
         msg: >
-        Cannot find {{ initial_backup_dir }}/{{ backup_filename }}
+          The override file is missing in the backup tarball
-        or the override file is missing in the backup tarball!
+          {{ local_backup_file_path }}!
-    when: search_result.rc != 0
+      when: not platform_tarball_encrypted|bool
-  delegate_to: "{{ inspection_target }}"
+
+    - name: Fail with encryption message
+      fail:
+        msg: >
+          Cannot decrypt {{ local_backup_file_path }}
+          or the override file is missing in the backup tarball!
+      when: platform_tarball_encrypted|bool
+    when: search_result.rc != 0
 
-- block:
   - name: Extract kubeadm version from the backup tarball
     # Match kube_cmd_versions VALUES from sysinv database table.
-    command: >-
+    shell:
-        zgrep -aE '^INSERT INTO .*kube_cmd_versions VALUES'
+      cmd: >-
-        {{ initial_backup_dir }}/{{ backup_filename }}
+        {{ extract_command_method }}
+        | zgrep -aE '^INSERT INTO .*kube_cmd_versions VALUES'
+      stdin: "{{ backup_encryption_passphrase }}"
     args:
       warn: false
     failed_when: false
@@ -45,8 +122,8 @@
   - name: Fail if kube_cmd_versions table is missing
     fail:
       msg: >
-        Cannot find {{ initial_backup_dir }}/{{ backup_filename }}
+        The kube_cmd_versions table is missing in the backup tarball
-        or the kube_cmd_versions table is missing in the backup tarball!
+        {{ local_backup_file_path }}!
     when: kube_cmd_search.rc != 0
 
   - name: Set restore kubernetes version for bootstrap
@@ -85,7 +162,7 @@
 
   - name: Set restore file parameter
     set_fact:
-      restore_data_file: "{{ target_backup_dir }}/{{ backup_filename }}"
+      restore_data_file: "{{ decrypted_backup_filepath }}"
 
   - name: Stage backup tarballs
     include_role:
@@ -94,7 +171,7 @@
   - name: Extract override file from backup tarball
     command: >
       tar --use-compress-program=pigz -C {{ target_backup_dir }}
-      -xf {{ target_backup_dir }}/{{ backup_filename }} --transform='s,.*/,,'
+      -xf {{ decrypted_backup_filepath }} --transform='s,.*/,,'
       {{ search_result.stdout_lines[0] }}
     register: extract_result
     failed_when: false

playbookconfig/src/playbooks/roles/vault/prepare_env/tasks/main.yml

@@ -49,10 +49,15 @@
     state: directory
     mode: 0755
 
-- name: Check if encrypt is enabled
+- name: Fail if passphrase is omitted
-  set_fact:
+  fail:
-    vault_encrypt: true
+    msg: >
-  when: encrypt_hc_vault_secret | length > 0
+      A passphrase is required for encryption; set variable override
+      backup_encryption_passphrase.  To disable encryption set
+      override backup_encryption_enabled=false
+  when:
+  - vault_encrypt|bool
+  - encrypt_hc_vault_secret | length == 0
 
 - name: Check vault apply for backup
   block:

playbookconfig/src/playbooks/vault_backup.yml

@@ -11,14 +11,16 @@
 - hosts: all
   gather_facts: no
 
+  # Specify defaults including:
+  #   backup_encryption_enabled
+  #   backup_encryption_passphrase
   vars_files:
     - host_vars/backup-restore/default.yml
 
   vars:
     password_change: false
-    vault_encrypt: false
+    vault_encrypt: "{{ backup_encryption_enabled|bool }}"
-    # override encrypt_hc_vault_secret when calling the playbook to enable extra encryption
+    encrypt_hc_vault_secret: "{{ backup_encryption_passphrase | default('') }}"
-    encrypt_hc_vault_secret: ""
     vault_mode: "backup"
     op_mode: "standalone"
 

playbookconfig/src/playbooks/vault_restore.yml

@@ -12,14 +12,16 @@
 - hosts: all
   gather_facts: no
 
+  # Specify defaults including:
+  #   backup_encryption_enabled
+  #   backup_encryption_passphrase
   vars_files:
     - host_vars/backup-restore/default.yml
 
   vars:
     password_change: false
-    vault_encrypt: false
+    vault_encrypt: "{{ backup_encryption_enabled|bool }}"
-    # override encrypt_hc_vault_secret when calling the playbook to enable extra encryption
+    encrypt_hc_vault_secret: "{{ backup_encryption_passphrase | default('') }}"
-    encrypt_hc_vault_secret: ""
     vault_mode: "restore"
     op_mode: "standalone"