tripleo-transfer: ability to customize files transfer options

There are use cases where it's useful to be able to explicitly
exclude files from a transfer request. For example when syncing galera
database, we don't want to copy coredumps, local galera state file...

Add the ability to explicitly include or exclude files by specifying
a list of files or wildcard-based patterns.

Also make the default transfer options configurable, to best suit
transfer use cases.

Related-Bug: #1925260

Change-Id: I1e88907db0bb26d52c92cb996eb56767e06b3874
(cherry picked from commit 6f75fd1723)
(cherry picked from commit 23531b28c4)

tripleo_ansible/roles/tripleo_transfer/defaults/main.yml

@@ -33,3 +33,18 @@ tripleo_transfer_key_location: "~/transfer_key"
 # tripleo_transfer_cleanup_keys: clean up the keypair from the source host
 # and remove public key from destination host when true.
 tripleo_transfer_cleanup_keys: true
+# tripleo_transfer_exclude: a list of patterns to selectively exclude
+# some files from the transfer to the destination host.
+tripleo_transfer_exclude: []
+# tripleo_transfer_include: a list of patterns to selectively include
+# some files from the transfer to the destination host. Files in this list
+# are not filtered out with tripleo_transfer_exclude.
+tripleo_transfer_include: []
+# tripleo_transfer_sync_options: override default transfer options
+tripleo_transfer_sync_options:
+  --delay-updates
+  -F
+  --ignore-times
+  --compress
+  --archive
+  --delete

tripleo_ansible/roles/tripleo_transfer/molecule/default/cleanup.yml

@@ -27,3 +27,5 @@
       loop:
         - "{{ test_src_dir }}"
         - "{{ test_dst_dir }}"
+        - "{{ test2_src_dir }}"
+        - "{{ test2_dst_dir }}"

tripleo_ansible/roles/tripleo_transfer/molecule/default/converge.yml

@@ -41,6 +41,8 @@
       loop:
         - "{{ test_src_dir }}"
         - "{{ test_dst_dir }}"
+        - "{{ test2_src_dir }}"
+        - "{{ test2_dst_dir }}"
 
     - name: Create 1GB src test files with random content
       shell: >-
@@ -63,3 +65,19 @@
         tripleo_transfer_src_dir: "{{ test_src_dir }}"
         tripleo_transfer_dest_host: controller1
         tripleo_transfer_dest_dir: "{{ test_dst_dir }}"
+
+    - name: Create empty src test files
+      file:
+        state: touch
+        path: "{{ test2_src_dir }}/{{ item }}"
+      loop: "{{ test2_src_files + test2_exclude_src_files }}"
+
+    - include_role:
+        name: "tripleo_transfer"
+      vars:
+        tripleo_transfer_src_host: controller2
+        tripleo_transfer_src_dir: "{{ test2_src_dir }}"
+        tripleo_transfer_dest_host: controller1
+        tripleo_transfer_dest_dir: "{{ test2_dst_dir }}"
+        tripleo_transfer_exclude: "{{ test2_exclude_pattern }}"
+        tripleo_transfer_include: "{{ test2_include_pattern }}"

tripleo_ansible/roles/tripleo_transfer/molecule/default/molecule.yml

@@ -47,6 +47,20 @@ provisioner:
             - testfile4
             - testfile5
             - testfile6
+          test2_src_dir: "/tmp/src_files_2"
+          test2_dst_dir: "/tmp/dst_files_2"
+          test2_src_files:
+            - transferred
+          test2_exclude_src_files:
+            - skip1
+            - skip2
+            - donotcopy
+          test2_include_pattern:
+            - transferred
+          test2_exclude_pattern:
+            - 'skip*'
+            - donotcopy
+
   log: true
   env:
     ANSIBLE_STDOUT_CALLBACK: yaml

tripleo_ansible/roles/tripleo_transfer/molecule/default/verify.yml

@@ -24,6 +24,8 @@
         paths:
           - "{{ test_src_dir }}"
           - "{{ test_dst_dir }}"
+          - "{{ test2_src_dir }}"
+          - "{{ test2_dst_dir }}"
         get_checksum: true
         recurse: true
       register: _fs_data
@@ -40,8 +42,23 @@
     - set_fact:
         result_file_list: "{{ _fs_data | json_query('files[*].path') }}"
         test_dst_files: "{{ test_dst_files | map('regex_replace', '(.*)', test_dst_dir ~ '/\\1') }}"
+        test2_excluded_files: "{{ test2_excluded_files | map('regex_replace', '(.*)', test2_dst_dir ~ '/\\1') }}"
 
     - name: Verify that the dst file are not there any more
       assert:
         that:
           - test_dst_files is not subset(result_file_list)
+
+    - name: Verify that files excluded from a transfer are not copied
+      assert:
+        that:
+          - test2_excluded_files is not subset(result_file_list)
+
+    - name: Verify that the transfer with exclude patterns did work
+      assert:
+        that:
+          - _fs_data | json_query(srcfilequery) == _fs_data | json_query(dstfilequery)
+      vars:
+        srcfilequery: files[?path =='{{ test2_src_dir }}/{{ item }}'].checksum
+        dstfilequery: files[?path =='{{ test2_dst_dir }}/{{ item }}'].checksum
+      loop: "{{ test2_src_files }}"

tripleo_ansible/roles/tripleo_transfer/tasks/main.yml

@@ -67,15 +67,20 @@
           {{ hostvars[tripleo_transfer_dest_host].ansible_user |
           default(hostvars[tripleo_transfer_dest_host].ansible_ssh_user |
           default(hostvars[tripleo_transfer_dest_host].ansible_user_id)) }}
+        tripleo_transfer_include_parameters: >-
+          {{ tripleo_transfer_include is string |
+          ternary([tripleo_transfer_include], tripleo_transfer_include) |
+          map('regex_replace', '^(.*)$', "--include='\1'") | join(' ') }}
+        tripleo_transfer_exclude_parameters: >-
+          {{ tripleo_transfer_exclude is string |
+          ternary([tripleo_transfer_exclude], tripleo_transfer_exclude) |
+          map('regex_replace', '^(.*)$', "--exclude='\1'") | join(' ') }}
       shell: >-
         /usr/bin/rsync
         -v
-        --delay-updates
-        -F
-        --ignore-times
-        --compress
-        --archive
-        --delete
+        {{ tripleo_transfer_sync_options }}
+        {{ tripleo_transfer_include_parameters }}
+        {{ tripleo_transfer_exclude_parameters }}
         --rsync-path='sudo rsync'
         --rsh='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {{ tripleo_transfer_key_location }}'
         {{ tripleo_transfer_src_dir_safe }}