collect-logs: add timeout to log uploads

Uploading the logs can take too much time, resulting ssh connection
timeout during the ansible task execution. Log collection usually runs
in post build tasks, making it immune to jenkins job timeouts.

This change makes the upload tasks asyncronous and adds a timeout.

Change-Id: I65cf017717775ac85b953fe554f84c79e4f808b5

roles/collect-logs/README.md

@@ -77,6 +77,8 @@ artcl_create_docs_payload:
   to the target specified by `artcl_rsync_url`. Uses `BUILD_URL`, `BUILD_TAG`
   vars from the environment (set during a Jenkins job run) and requires the
   next to variables to be set.
+* `artcl_publish_timeout`: the maximum seconds the role can spend uploading the
+  logs, the default is 1800 (30 minutes)
 * `artcl_use_rsync`: false/true -- use rsync to upload the logs
 * `artcl_rsync_use_daemon`: false/true -- use rsync daemon instead of ssh to connect
 * `artcl_rsync_url` -- rsync target for uploading the logs. The localhost

roles/collect-logs/defaults/main.yml

@@ -1,5 +1,7 @@
 ---
 
+## collection related vars
+artcl_collect: true
 artcl_collect_list:
   - /var/log/
   - /var/lib/mysql
@@ -32,8 +34,11 @@ artcl_collect_list:
   - /tmp/tripleoclient*
 artcl_collect_dir: "{{ lookup('env', 'PWD') }}/collected_files"
 artcl_gzip_only: true
-artcl_collect: true
+
+## publishing related vars
 artcl_publish: false
+# give up log upload after 30 minutes
+artcl_publish_timeout: 1800
 artcl_full_artifact_url: "{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}/"
 artcl_use_rsync: false
 artcl_rsync_use_daemon: false

roles/collect-logs/tasks/publish.yml

@@ -11,6 +11,8 @@
 
 - name: upload to the artifact server using pubkey auth
   command: rsync -av --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
+  async: "{{ artcl_publish_timeout }}"
+  poll: 15
   retries: 5
   delay: 60
   when: artcl_use_rsync|bool and not artcl_rsync_use_daemon|bool
@@ -19,6 +21,8 @@
   environment:
     RSYNC_PASSWORD: "{{ lookup('env', 'RSYNC_PASSWORD') }}"
   command: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
+  async: "{{ artcl_publish_timeout }}"
+  poll: 15
   retries: 5
   delay: 60
   when: artcl_use_rsync|bool and artcl_rsync_use_daemon|bool
@@ -33,10 +37,14 @@
     OS_USERNAME: "{{ artcl_swift_username }}"
     OS_PASSWORD: "{{ artcl_swift_password }}"
     OS_TENANT_NAME: "{{ artcl_swift_tenant_name }}"
+  async: "{{ artcl_publish_timeout }}"
+  poll: 15
   when: artcl_use_swift|bool
 
 - name: use zuul_swift_upload.py to publish the files
   command: "{{ artcl_zuul_swift_upload_path }}/zuul_swift_upload.py --name {{ artcl_swift_container }} --delete-after {{ artcl_swift_delete_after }} {{ artcl_collect_dir }}"
+  async: "{{ artcl_publish_timeout }}"
+  poll: 15
   when: artcl_use_zuul_swift_upload|bool
 
 - name: create the artifact location redirect file