Add swift as a publisher and publish fixes

* push logs to a swift storage with an automatic expiration
* call the publish part only once
* make rsync --quiet for a more compact playbook output
* make rsync upload optional
* fix full_logs template to include a slash at the end of the URL,
  needed by the os-loganalyze server
* document new settings

Change-Id: If3dc99cce2b4386e9ed0ed1c47c7f4098ec43e5a

README.md

@@ -14,11 +14,15 @@ TripleO-Quickstart during deployment, into rST files. These rST files are
 combined with static rST files and fed into Sphinx to create user friendly
 post-build-documentation specific to an original deployment.
 
-Finally, the role optionally handles uploading these logs to a rsync server.
+Finally, the role optionally handles uploading these logs to a rsync server or
+to an OpenStack Swift object storage. Logs from Swift can be exposed with
+[os-loganalyze](https://github.com/openstack-infra/os-loganalyze).
 
 Role Variables
 --------------
 
+### Collection related
+
 * `artcl_collect_list` -- A list of files and directories to gather from
   the target. Directories are collected recursively. Can include joker
   characters that bash understands. Should be specified as a YaML list,
@@ -36,6 +40,9 @@ artcl_collect_list:
 * `artcl_gzip_only`: false/true  -- When true, gathered files are gzipped one
   by one in `artcl_collect_dir`, when false, a tar.gz file will contain all the
   logs.
+
+### Documentation generation related
+
 * `artcl_gen_docs`: true/false -- If true, the role will use build artifacts
   and Sphinx and produce user friendly documentation.
 * `artcl_docs_source_dir` -- a local directory that serves as the Sphinx source
@@ -64,13 +71,26 @@ artcl_create_docs_payload:
     - undercloud-post-install
 ```
 
+### Publishing related
+
 * `artcl_publish`: true/false -- If true, the role will attempt to rsync logs
   to the target specified by `artcl_rsync_url`. Uses `BUILD_URL`, `BUILD_TAG`
   vars from the environment (set during a Jenkins job run) and requires the
   next to variables to be set.
+* `artcl_use_rsync`: false/true -- use rsync to upload the logs
+* `artcl_rsync_use_daemon`: false/true -- use rsync daemon instead of ssh to connect
 * `artcl_rsync_url` -- rsync target for uploading the logs. The localhost
   needs to have passwordless authentication to the target or the
   `PROVISIONER_KEY` Var specificed in the environment.
+* `artcl_use_swift`: false/true -- use swift object storage to publish the logs
+* `artcl_swift_auth_url` -- the OpenStack auth URL for Swift
+* `artcl_swift_username` -- OpenStack username for Swift
+* `artcl_swift_password` -- password for the Swift user
+* `artcl_swift_tenant_name` -- OpenStack tenant name for Swift
+* `artcl_swift_container` -- the name of the Swift container to use,
+  default is `logs`
+* `artcl_swift_delete_after` -- The number of seconds after which Swift will
+  remove the uploaded objects, the default is 2678400 seconds = 31 days.
 * `artcl_artifact_url` -- a HTTP URL at which the uploaded logs will be
   accessible after upload.
 

configs/centosci-logs.yml

@@ -1,5 +1,6 @@
 artcl_gzip_only: true
 artcl_artifact_url: https://ci.centos.org/artifacts/rdo
-artcl_rsync_path: rdo@artifacts.ci.centos.org::rdo
-artcl_rsync_use_daemon: true
 artcl_publish: true
+artcl_use_rsync: true
+artcl_rsync_use_daemon: true
+artcl_rsync_path: rdo@artifacts.ci.centos.org::rdo

defaults/main.yml

@@ -30,6 +30,13 @@ artcl_collect_dir: "{{ lookup('env', 'PWD') }}/collected_files"
 artcl_gzip_only: true
 artcl_collect: true
 artcl_publish: false
+artcl_use_rsync: false
+artcl_rsync_use_daemon: false
+
+artcl_use_swift: false
+# clean up the logs after 31 days
+artcl_swift_delete_after: 2678400
+artcl_swift_container: logs
 
 ## Doc generation specific vars
 artcl_gen_docs: false

tasks/main.yml

@@ -1,11 +1,14 @@
 ---
 # tasks file for ansible-role-tripleo-collect-logs
 
-- include: collect.yml
+- name: Collect logs
+  include: collect.yml
   when: artcl_collect|bool
 
-- include: create-docs.yml
+- name: Generate docs
+  include: create-docs.yml
   when: artcl_gen_docs|bool
 
-- include: publish.yml
+- name: Publish logs
+  include: publish.yml
   when: artcl_publish|bool

tasks/publish.yml

@@ -1,26 +1,35 @@
 ---
 
-- block:
-    #internal-ci
+- run_once: true
+  delegate_to: localhost
+  block:
     - name: upload to the artifact server using pubkey auth
-      shell: >
-        rsync -av -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/$BUILD_TAG
+      command: rsync -av --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
       retries: 5
       delay: 60
-      when: artcl_rsync_use_daemon == false
+      when: artcl_use_rsync|bool and not artcl_rsync_use_daemon|bool
 
-    #centos-ci
     - name: upload to the artifact server using password auth
       environment:
         RSYNC_PASSWORD: "{{ lookup('env', 'RSYNC_PASSWORD') }}"
-      shell: >
-        rsync -av {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/$BUILD_TAG
+      command: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
       retries: 5
       delay: 60
-      when: artcl_rsync_use_daemon == true
+      when: artcl_use_rsync|bool and artcl_rsync_use_daemon|bool
+
+    - name: upload to swift based artifact server
+      shell: swift upload --quiet --header "X-Delete-After:{{ artcl_swift_delete_after }}" {{ artcl_swift_container }}/{{ lookup('env', 'BUILD_TAG') }} *
+      args:
+        chdir: "{{ artcl_collect_dir }}"
+      changed_when: true
+      environment:
+        OS_AUTH_URL: "{{ artcl_swift_auth_url }}"
+        OS_USERNAME: "{{ artcl_swift_username }}"
+        OS_PASSWORD: "{{ artcl_swift_password }}"
+        OS_TENANT_NAME: "{{ artcl_swift_tenant_name }}"
+      when: artcl_use_swift|bool
 
     - name: create the artifact location redirect file
       template:
         src: full_logs.html.j2
         dest: "{{ artcl_collect_dir }}/full_logs.html"
-  delegate_to: localhost

templates/full_logs.html.j2

@@ -2,13 +2,13 @@
 <html lang="en-US">
     <head>
         <meta charset="UTF-8">
-        <meta http-equiv="refresh" content="1; url={{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}">
+        <meta http-equiv="refresh" content="1; url={{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}/">
         <script type="text/javascript">
-            window.location.href = "{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}"
+            window.location.href = "{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}/"
         </script>
         <title>Redirection to logs</title>
     </head>
     <body>
-        If you are not redirected automatically, follow the <a href='{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}'>link to the logs</a>.
+        If you are not redirected automatically, follow the <a href='{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}/'>link to the logs</a>.
     </body>
 </html>