From 5f8251baf299f46634a47384e3ea2d0942e75ebc Mon Sep 17 00:00:00 2001 From: Attila Darazs Date: Wed, 21 Sep 2016 17:43:17 +0200 Subject: [PATCH] Add swift as a publisher and publish fixes * push logs to a swift storage with an automatic expiration * call the publish part only once * make rsync --quiet for a more compact playbook output * make rsync upload optional * fix full_logs template to include a slash at the end of the URL, needed by the os-loganalyze server * document new settings Change-Id: If3dc99cce2b4386e9ed0ed1c47c7f4098ec43e5a --- README.md | 22 +++++++++++++++++++++- configs/centosci-logs.yml | 5 +++-- defaults/main.yml | 7 +++++++ tasks/main.yml | 9 ++++++--- tasks/publish.yml | 29 +++++++++++++++++++---------- templates/full_logs.html.j2 | 6 +++--- 6 files changed, 59 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 15a1f67..38aa1ed 100644 --- a/README.md +++ b/README.md @@ -14,11 +14,15 @@ TripleO-Quickstart during deployment, into rST files. These rST files are combined with static rST files and fed into Sphinx to create user friendly post-build-documentation specific to an original deployment. -Finally, the role optionally handles uploading these logs to a rsync server. +Finally, the role optionally handles uploading these logs to a rsync server or +to an OpenStack Swift object storage. Logs from Swift can be exposed with +[os-loganalyze](https://github.com/openstack-infra/os-loganalyze). Role Variables -------------- +### Collection related + * `artcl_collect_list` -- A list of files and directories to gather from the target. Directories are collected recursively. Can include joker characters that bash understands. Should be specified as a YaML list, @@ -36,6 +40,9 @@ artcl_collect_list: * `artcl_gzip_only`: false/true -- When true, gathered files are gzipped one by one in `artcl_collect_dir`, when false, a tar.gz file will contain all the logs. + +### Documentation generation related + * `artcl_gen_docs`: true/false -- If true, the role will use build artifacts and Sphinx and produce user friendly documentation. * `artcl_docs_source_dir` -- a local directory that serves as the Sphinx source @@ -64,13 +71,26 @@ artcl_create_docs_payload: - undercloud-post-install ``` +### Publishing related + * `artcl_publish`: true/false -- If true, the role will attempt to rsync logs to the target specified by `artcl_rsync_url`. Uses `BUILD_URL`, `BUILD_TAG` vars from the environment (set during a Jenkins job run) and requires the next to variables to be set. +* `artcl_use_rsync`: false/true -- use rsync to upload the logs +* `artcl_rsync_use_daemon`: false/true -- use rsync daemon instead of ssh to connect * `artcl_rsync_url` -- rsync target for uploading the logs. The localhost needs to have passwordless authentication to the target or the `PROVISIONER_KEY` Var specificed in the environment. +* `artcl_use_swift`: false/true -- use swift object storage to publish the logs +* `artcl_swift_auth_url` -- the OpenStack auth URL for Swift +* `artcl_swift_username` -- OpenStack username for Swift +* `artcl_swift_password` -- password for the Swift user +* `artcl_swift_tenant_name` -- OpenStack tenant name for Swift +* `artcl_swift_container` -- the name of the Swift container to use, + default is `logs` +* `artcl_swift_delete_after` -- The number of seconds after which Swift will + remove the uploaded objects, the default is 2678400 seconds = 31 days. * `artcl_artifact_url` -- a HTTP URL at which the uploaded logs will be accessible after upload. diff --git a/configs/centosci-logs.yml b/configs/centosci-logs.yml index e990527..bd3df42 100644 --- a/configs/centosci-logs.yml +++ b/configs/centosci-logs.yml @@ -1,5 +1,6 @@ artcl_gzip_only: true artcl_artifact_url: https://ci.centos.org/artifacts/rdo -artcl_rsync_path: rdo@artifacts.ci.centos.org::rdo -artcl_rsync_use_daemon: true artcl_publish: true +artcl_use_rsync: true +artcl_rsync_use_daemon: true +artcl_rsync_path: rdo@artifacts.ci.centos.org::rdo diff --git a/defaults/main.yml b/defaults/main.yml index 39494fa..2f6177a 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -30,6 +30,13 @@ artcl_collect_dir: "{{ lookup('env', 'PWD') }}/collected_files" artcl_gzip_only: true artcl_collect: true artcl_publish: false +artcl_use_rsync: false +artcl_rsync_use_daemon: false + +artcl_use_swift: false +# clean up the logs after 31 days +artcl_swift_delete_after: 2678400 +artcl_swift_container: logs ## Doc generation specific vars artcl_gen_docs: false diff --git a/tasks/main.yml b/tasks/main.yml index 603434c..b443d06 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -1,11 +1,14 @@ --- # tasks file for ansible-role-tripleo-collect-logs -- include: collect.yml +- name: Collect logs + include: collect.yml when: artcl_collect|bool -- include: create-docs.yml +- name: Generate docs + include: create-docs.yml when: artcl_gen_docs|bool -- include: publish.yml +- name: Publish logs + include: publish.yml when: artcl_publish|bool diff --git a/tasks/publish.yml b/tasks/publish.yml index 92f2737..683ec68 100644 --- a/tasks/publish.yml +++ b/tasks/publish.yml @@ -1,26 +1,35 @@ --- -- block: - #internal-ci +- run_once: true + delegate_to: localhost + block: - name: upload to the artifact server using pubkey auth - shell: > - rsync -av -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/$BUILD_TAG + command: rsync -av --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null" {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }} retries: 5 delay: 60 - when: artcl_rsync_use_daemon == false + when: artcl_use_rsync|bool and not artcl_rsync_use_daemon|bool - #centos-ci - name: upload to the artifact server using password auth environment: RSYNC_PASSWORD: "{{ lookup('env', 'RSYNC_PASSWORD') }}" - shell: > - rsync -av {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/$BUILD_TAG + command: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }} retries: 5 delay: 60 - when: artcl_rsync_use_daemon == true + when: artcl_use_rsync|bool and artcl_rsync_use_daemon|bool + + - name: upload to swift based artifact server + shell: swift upload --quiet --header "X-Delete-After:{{ artcl_swift_delete_after }}" {{ artcl_swift_container }}/{{ lookup('env', 'BUILD_TAG') }} * + args: + chdir: "{{ artcl_collect_dir }}" + changed_when: true + environment: + OS_AUTH_URL: "{{ artcl_swift_auth_url }}" + OS_USERNAME: "{{ artcl_swift_username }}" + OS_PASSWORD: "{{ artcl_swift_password }}" + OS_TENANT_NAME: "{{ artcl_swift_tenant_name }}" + when: artcl_use_swift|bool - name: create the artifact location redirect file template: src: full_logs.html.j2 dest: "{{ artcl_collect_dir }}/full_logs.html" - delegate_to: localhost diff --git a/templates/full_logs.html.j2 b/templates/full_logs.html.j2 index 8bdcf1e..c18116a 100644 --- a/templates/full_logs.html.j2 +++ b/templates/full_logs.html.j2 @@ -2,13 +2,13 @@ - + Redirection to logs - If you are not redirected automatically, follow the link to the logs. + If you are not redirected automatically, follow the link to the logs.