Merge "[tripleo_transfer] Do transfers directly from src to dst"

tripleo_ansible/roles/tripleo_transfer/defaults/main.yml

@@ -24,9 +24,12 @@
 #   * `tripleo_transfer_dest_host` -- the inventory name of the destination host
 #   * `tripleo_transfer_dest_dir` -- directory on the destination host to transfer to
 tripleo_transfer_debug: "{{ ((ansible_verbosity | int) >= 2) | bool }}"
-tripleo_transfer_storage_root_dir: /var/lib/mistral/tripleo-transfer
-tripleo_transfer_storage_root_become: false
 tripleo_transfer_src_become: true
 tripleo_transfer_dest_become: true
-tripleo_transfer_dest_wipe: true
 tripleo_transfer_flag_file: ~
+# tripleo_transfer_key_location: location of the private key used to connect
+# from src host to dest host.
+tripleo_transfer_key_location: "~/transfer_key"
+# tripleo_transfer_cleanup_keys: clean up the keypair from the source host
+# and remove public key from destination host when true.
+tripleo_transfer_cleanup_keys: true

tripleo_ansible/roles/tripleo_transfer/molecule/default/converge.yml

@@ -14,6 +14,14 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+- name: Collect facts
+  hosts: all
+  gather_facts: false
+  any_errors_fatal: true
+  tasks:
+    - name: Gather a minimal set of facts
+      setup:
+        gather_subset: '!all,min'
 
 - name: Converge
   hosts: localhost
@@ -25,4 +33,3 @@
       tripleo_transfer_src_dir: /etc
       tripleo_transfer_dest_host: controller1
       tripleo_transfer_dest_dir: /opt/etc-target
-      tripleo_transfer_storage_root_dir: /tmp/transfer-staging

tripleo_ansible/roles/tripleo_transfer/tasks/flag.yml

@@ -19,8 +19,6 @@
   stat:
     path: "{{ tripleo_transfer_flag_file }}"
   register: tripleo_transfer_flag_stat
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
 
 - name: fail if flag file exists
   fail:
@@ -32,18 +30,13 @@
       and re-run the data transfer.
   when:
     - tripleo_transfer_flag_stat.stat.exists
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
 
 - name: ensure directory for flag file exists
   file:
     path: "{{ tripleo_transfer_flag_file|dirname }}"
     state: directory
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
 
 - name: create the flag file
   file:
     path: "{{ tripleo_transfer_flag_file }}"
     state: touch
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"

tripleo_ansible/roles/tripleo_transfer/tasks/main.yml

@@ -15,89 +15,87 @@
 # under the License.
 
 
-- name: make sure we don't have a trailing forward slash in the src
+# Note:
-  set_fact:
+# This role is used in a playbook that typically targets the undercloud
-    tripleo_transfer_src_dir_safe: "{{ tripleo_transfer_src_dir | regex_replace('\\/$', '') }}"
+# and may target other hosts, so to ensure that it executes against the
-    cacheable: false
+# right src and dest it uses delegation throughout.
 
-- name: make sure we don't have a trailing forward slash in the dst
+- name: tripleo_transfer tasks
-  set_fact:
+  run_once: true
-    tripleo_transfer_dest_dir_safe: "{{ tripleo_transfer_dest_dir | regex_replace('\\/$', '') }}"
+  block:
-    cacheable: false
+    - name: install requirements in src and dest hosts
+      become: true
+      package:
+        name:
+          - rsync
+          - openssh-clients
+        state: present
+      delegate_to: "{{ item }}"
+      loop:
+        - "{{ tripleo_transfer_src_host }}"
+        - "{{ tripleo_transfer_dest_host }}"
 
-- name: ensure local storage directory exists and has correct permissions
+    - name: generate ssh key-pair in source host
-  file:
+      openssh_keypair:
-    path: "{{ tripleo_transfer_storage_root_dir }}"
+        path: "{{ tripleo_transfer_key_location }}"
-    # Attempting to set an owner fails with "chown failed: failed to
+      delegate_to: "{{ tripleo_transfer_src_host }}"
-    # look up user" so we at least ensure the permissions.
+      become: "{{ tripleo_transfer_src_become }}"
-    mode: 0700
+      register: keypair_gen
-    state: directory
-  delegate_to: localhost
-  become: "{{ tripleo_transfer_storage_root_become }}"
 
-- name: create tempfile for the archive
+    - name: set authorized-keys in destination host
-  tempfile:
+      authorized_key:
-    prefix: ansible.tripleo-transfer.
+        comment: "Added by tripleo-transfer"
-  register: tripleo_transfer_tempfile
+        user: "{{ ansible_user|default(ansible_ssh_user|default(hostvars[tripleo_transfer_dest_host].ansible_user_id)) }}"
-  become: "{{ tripleo_transfer_src_become }}"
+        state: present
-  delegate_to: "{{ tripleo_transfer_src_host }}"
+        key: "{{ keypair_gen.public_key }}"
+      delegate_to: "{{ tripleo_transfer_dest_host }}"
+      when: keypair_gen is succeeded
 
-# Using the "archive" module lists lists all tarred files in module
+    - import_tasks: flag.yml
-# output, if there's too many files, it can crash ansible even with
+      when:
-# "no_log: "{{ not tripleo_transfer_debug | bool }}"".
+        - tripleo_transfer_flag_file != None
-- name: create the archive
+        - tripleo_transfer_flag_file|length > 0
-  shell: |-
+      become: "{{ tripleo_transfer_dest_become }}"
-    set -euo pipefail
+      delegate_to: "{{ tripleo_transfer_dest_host }}"
-    tar --transform "s|^{{ tripleo_transfer_src_dir_safe | basename }}|{{ tripleo_transfer_dest_dir_safe | basename }}|" \
-        -czf "{{ tripleo_transfer_tempfile.path }}" \
-        -C "{{ tripleo_transfer_src_dir_safe | dirname }}" \
-        "{{ tripleo_transfer_src_dir_safe | basename }}"
-  become: "{{ tripleo_transfer_src_become }}"
-  delegate_to: "{{ tripleo_transfer_src_host }}"
 
-- name: fetch the archive
+    - name: synchronize both directories
-  fetch:
+      vars:
-    src: "{{ tripleo_transfer_tempfile.path }}"
+        hostvars_dest_host_ip: >-
-    dest: "{{ tripleo_transfer_storage_root_dir }}/{{ tripleo_transfer_dest_host }}{{ tripleo_transfer_dest_dir_safe }}.tar.gz"
+          {{ hostvars[tripleo_transfer_dest_host].ansible_host |
-    flat: true
+          default(hostvars[tripleo_transfer_dest_host].inventory_hostname) }}
-  become: "{{ tripleo_transfer_src_become }}"
+        tripleo_transfer_dest_user: >-
-  delegate_to: "{{ tripleo_transfer_src_host }}"
+          {{ hostvars[tripleo_transfer_dest_host].ansible_user |
+          default(hostvars[tripleo_transfer_dest_host].ansible_ssh_user |
+          default(hostvars[tripleo_transfer_dest_host].ansible_user_id)) }}
+      shell: >-
+        /usr/bin/rsync
+        -v
+        --delay-updates
+        -F
+        --compress
+        --archive
+        --delete
+        --rsync-path='sudo rsync'
+        --rsh='ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -i {{ tripleo_transfer_key_location }}'
+        {{ tripleo_transfer_src_dir_safe }}
+        {{ tripleo_transfer_dest_user }}@{{ hostvars_dest_host_ip }}:{{ tripleo_transfer_dest_dir_safe }}
+      become: "{{ tripleo_transfer_src_become }}"
+      delegate_to: "{{ tripleo_transfer_src_host }}"
+  always:
+    - name: clean up keys in source host
+      file:
+        path: "{{ item }}"
+        state: absent
+      delegate_to: "{{ tripleo_transfer_src_host }}"
+      become: "{{ tripleo_transfer_src_become }}"
+      loop:
+        - "{{ tripleo_transfer_key_location }}"
+        - "{{ tripleo_transfer_key_location }}.pub"
 
-- name: remove tempfile
+    - name: remove public key from authorized keys in destination host
-  file:
+      lineinfile:
-    name: "{{ tripleo_transfer_tempfile.path }}"
+        path: "~/.ssh/authorized_keys"
-    state: absent
+        state: absent
-  become: "{{ tripleo_transfer_src_become }}"
+        regexp: '.*Added by tripleo-transfer.*$'
-  delegate_to: "{{ tripleo_transfer_src_host }}"
+      delegate_to: "{{ tripleo_transfer_dest_host }}"
-
+  when: tripleo_transfer_cleanup_keys | bool
-- include_tasks: flag.yml
-  when:
-    - tripleo_transfer_flag_file != None
-    - tripleo_transfer_flag_file|length > 0
-
-- name: wipe the destination directory
-  file:
-    path: "{{ tripleo_transfer_dest_dir_safe }}"
-    state: absent
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
-  when: tripleo_transfer_dest_wipe|bool
-
-- name: make sure the destination parent directory is present
-  file:
-    path: "{{ tripleo_transfer_dest_dir_safe|dirname }}"
-    state: directory
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
-
-- name: push and extract the archive
-  unarchive:
-    src: "{{ tripleo_transfer_storage_root_dir }}/{{ tripleo_transfer_dest_host }}{{ tripleo_transfer_dest_dir_safe }}.tar.gz"
-    dest: "{{ tripleo_transfer_dest_dir_safe|dirname }}"
-  become: "{{ tripleo_transfer_dest_become }}"
-  delegate_to: "{{ tripleo_transfer_dest_host }}"
-
-- name: remove the local archive
-  file:
-    path: "{{ tripleo_transfer_storage_root_dir }}/{{ tripleo_transfer_dest_host }}{{ tripleo_transfer_dest_dir_safe }}.tar.gz"
-    state: absent

tripleo_ansible/roles/tripleo_transfer/vars/main.yml

@@ -15,9 +15,11 @@
 # under the License.
 
 
-- name: ensure tripleo_transfer storage directory is removed
+# All variables intended for internal use should be placed in this file.
-  file:
+
-    path: "{{ tripleo_transfer_storage_root_dir }}"
+
-    state: absent
+# make sure we have a trailing forward slash in the src, otherwise rsync creates extra dir
-  delegate_to: localhost
+tripleo_transfer_src_dir_safe: "{{ tripleo_transfer_src_dir }}/"
-  become: "{{ tripleo_transfer_storage_root_become }}"
+
+# make sure we do not have a trailing forward slash in the dest
+tripleo_transfer_dest_dir_safe: "{{ tripleo_transfer_dest_dir | regex_replace('\\/$', '') }}"