From 786c48f051f9ecdcbc032e37a5d533a4dcbdae38 Mon Sep 17 00:00:00 2001
From: "Andrea Frittoli (andreaf)" <andrea.frittoli@gmail.com>
Date: Tue, 3 Oct 2017 17:07:35 +0100
Subject: [PATCH] Add a generic stage-output role

This role does roughly what save_dir and save_file do in d-g
functions, only it can be used for a list of files and/or
folders. Jobs can use this role to rename artifacts and place
them in a staging folder which is then used for mass sync to
the executor.

This is typically used in post - the zuul user won't necessarily
have write access to the folders with the artifacts - so it's
easier to stage everything to a folder owned by the zuul user.

Change-Id: Id45a50821539985710da7172a9185fd91e803603
---
 roles/stage-output/README.rst         | 49 +++++++++++++++++++++++++++
 roles/stage-output/defaults/main.yaml |  2 ++
 roles/stage-output/tasks/main.yaml    | 47 +++++++++++++++++++++++++
 3 files changed, 98 insertions(+)
 create mode 100644 roles/stage-output/README.rst
 create mode 100644 roles/stage-output/defaults/main.yaml
 create mode 100644 roles/stage-output/tasks/main.yaml

diff --git a/roles/stage-output/README.rst b/roles/stage-output/README.rst
new file mode 100644
index 000000000..c503758e2
--- /dev/null
+++ b/roles/stage-output/README.rst
@@ -0,0 +1,49 @@
+Stage job output on the remote node
+
+Takes as input a dictionary of files/folders named 'zuul_copy_output'.
+Copies contents into {{ zuul_output_dir }} on the remote node and is
+intended to be used before output fetching in a base job's post-playbook.
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_copy_output
+   :default: None
+
+   Dictionary of files and folders to be staged.
+
+   The input is a dictionary so that it can accumulated via zuul variable
+   merging. Keys of the dictionary will be things to copy. Valid values
+   describe the type of output to copy:
+
+   * logs
+   * artifacts
+   * docs
+   * null   # ansible null, not the string null
+
+   null overrides the will of a parent job to copy something instructing
+   not to copy.
+
+.. zuul:rolevar:: stage_dir
+   :default: {{ ansible_user_dir }}
+
+   The stage directory on the remote node.
+
+.. zuul:rolevar:: extensions_to_txt
+   :default: null
+
+   A list of file extensions to be replaced with .txt when staging.
+   This can be useful to ensure that text files with an extension not
+   registered in the web server may be viewed via browser when uploaded
+   to a file server.
+
+   Note that this is only used for files listed directly in
+   `zuul_copy_output` and it won't be applied to files contained in
+   folders listed in `zuul_copy_output`.
+
+   Example:
+
+     extensions_to_txt:
+     - conf
+     - log
+
+     zuul.conf --(staged as)--> zuul_conf.txt
diff --git a/roles/stage-output/defaults/main.yaml b/roles/stage-output/defaults/main.yaml
new file mode 100644
index 000000000..daece5503
--- /dev/null
+++ b/roles/stage-output/defaults/main.yaml
@@ -0,0 +1,2 @@
+stage_dir: "{{ ansible_user_dir }}"
+extensions_to_txt:
diff --git a/roles/stage-output/tasks/main.yaml b/roles/stage-output/tasks/main.yaml
new file mode 100644
index 000000000..58b0a1248
--- /dev/null
+++ b/roles/stage-output/tasks/main.yaml
@@ -0,0 +1,47 @@
+- name: Register sources
+  stat:
+    path: "{{ item.key }}"
+  with_dict: "{{ zuul_copy_output }}"
+  register: sources
+
+- name: Build the replace regex
+  set_fact:
+    extensions_regex: "{{ extensions_to_txt | join('|') | default('__do_not_replace__') }}"
+
+- debug:
+    var: extensions_regex
+
+# TODO(andreaf) We might want to enforce that item.value is a valid value
+# in docs, artifacts, logs. Null case already handled.
+# We don't check if the item is a file before renaming, but it is not likely
+# to have directories with log, yaml or conf extension.
+- name: Set source and destination for files and folders
+  set_fact:
+    source: "{{ item.stat.path }}"
+    dest: "{{ item.item.value }}/{{ item.stat.path|basename|regex_replace('\\.(' + extensions_regex + ')$', '_\\1.txt') }}"
+  with_items: "{{ sources.results }}"
+  when:
+    - item.stat.exists
+    - item.item.value
+  register: results
+
+- name: Build a list of source, dest dictionaries for text files
+  set_fact:
+    all_sources: "{{ results.results | selectattr('ansible_facts', 'defined') | map(attribute='ansible_facts') | list }}"
+
+- name: ensure target folders exist
+  become: true
+  file:
+    path: "{{ stage_dir }}/{{ item }}"
+    state: directory
+    owner: "{{ ansible_user }}"
+  with_items:
+    - docs
+    - artifacts
+    - logs
+
+- name: Copy text files to staging folder
+  # remote_src copy does not work recursively, synchronise is restricted by
+  # zuul, using command
+  command: cp -pR {{ item.source}} {{ stage_dir }}/{{ item.dest }}
+  with_items: "{{ all_sources }}"