Allow artcl to be run by Zuul

- Avoid implicit use of lookup env which breaks under Zuul CI
- Add Zuul job that tests direct role usage and avoids regression.
  Follows layout used by zuul-jobs repository.

Change-Id: I1cdd31bfc28131a77a1596b87d081957c9089550
Story: https://tree.taiga.io/project/tripleo-ci-board/us/1458

defaults/main.yml

@@ -1,11 +1,15 @@
 ---
 # formally in tq common and tqe extras-common
-local_working_dir: "{{ lookup('env', 'HOME') }}/.quickstart"
+# zuul does not allow use of lookup env plugin (security), so we cannot use them
+# defaults but we can load non-zuul values from vars.
+local_working_dir: "{{ zuul_work_dir | default('~') }}/.quickstart"
 artcl_collect_dir: "{{ local_working_dir }}/collected_files"
 working_dir: "/home/{{ undercloud_user }}"
 undercloud_user: stack
 
 
+artcl_build_tag: "{{ zuul.build | default('') }}"  # overriden by vars/unsecure.yml
+
 artcl_collect: true
 artcl_collect_list:
   - /var/lib/heat-config/
@@ -148,7 +152,7 @@ artcl_txt_rename: false
 # give up log upload after 30 minutes
 artcl_publish_timeout: 1800
 artcl_artifact_url: "file://{{ local_working_dir }}"
-artcl_full_artifact_url: "{{ artcl_artifact_url }}/{{ lookup('env', 'BUILD_TAG') }}/"
+artcl_full_artifact_url: "{{ artcl_artifact_url }}/{{ artcl_build_tag }}/"
 artcl_use_rsync: false
 artcl_rsync_use_daemon: false
 artcl_container_collect_timeout: 1800  # 30 mins
@@ -349,7 +353,6 @@ artcl_logstash_files:
 ara_enabled: true
 ara_overcloud_db_path: "/var/lib/mistral/overcloud/ara_overcloud.sqlite"
 ara_generate_html: true
-ara_graphite_prefix: "tripleo.{{ lookup('env', 'STABLE_RELEASE')|default('master', true) }}.{{ lookup('env', 'TOCI_JOBTYPE') }}."
 ara_only_successful_tasks: true
 ara_tasks_map:
   "overcloud-deploy : Deploy the overcloud": overcloud.deploy.seconds
@@ -374,7 +377,7 @@ influxdb_port: 8086
 influxdb_user:
 influxdb_password:
 influxdb_dbname: testdb
-influxdb_data_file_path: "{{ lookup('env', 'LOCAL_WORKING_DIR')|default('/tmp', true) }}/influxdb_data"
+influxdb_data_file_path: "{{ local_working_dir }}/influxdb_data"
 influxdb_create_data_file: true
 odl_extra_log_dir: /var/log/extra/odl
 odl_extra_info_log: "{{ odl_extra_log_dir }}/odl_info.log"

tasks/main.yml

@@ -4,6 +4,10 @@
     gather_subset: "!min,pkg_mgr,python"
   when: "['pkg_mgr', 'python'] | difference(ansible_facts.keys()|list)"
 
+- name: Load unsecure.yml defaults when outside zuul
+  when: zuul is not defined
+  include_vars: unsecure.yml
+
 - name: Load operating system specific variables
   include_vars: "{{ item }}"
   failed_when: false

tasks/publish.yml

@@ -12,11 +12,13 @@
 - name: Fetch console log
   shell: >
     set -o pipefail &&
-    curl -k "{{ lookup('env', 'BUILD_URL') }}/timestamps/?time=yyyy-MM-dd%20HH:mm:ss.SSS%20|&appendLog&locale=en_GB"
+    curl -k "{{ artcl_build_url }}/timestamps/?time=yyyy-MM-dd%20HH:mm:ss.SSS%20|&appendLog&locale=en_GB"
     > {{ artcl_collect_dir }}/console.log
   args:
     executable: /bin/bash
-  when: lookup('env', 'BUILD_URL')
+  when:
+    - artcl_build_url is defined
+    - artcl_build_url|length > 0
 
 - include: sova.yml
   ignore_errors: true
@@ -100,7 +102,7 @@
   shell: >
     rsync -av
     --quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {{ artcl_report_server_key|default('') }}"
-    {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
+    {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }}
   async: "{{ artcl_publish_timeout }}"
   poll: 15
   retries: 5
@@ -110,8 +112,8 @@
 
 - name: upload to the artifact server using password auth
   environment:
-    RSYNC_PASSWORD: "{{ lookup('env', 'RSYNC_PASSWORD') }}"
-  shell: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}  # noqa 305
+    RSYNC_PASSWORD: "{{ artcl_rsync_password | default(omit) }}"
+  shell: rsync -av --quiet {{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ artcl_build_tag }}  # noqa 305
   async: "{{ artcl_publish_timeout }}"
   poll: 15
   retries: 5
@@ -120,7 +122,7 @@
   tags: notest
 
 - name: upload to swift based artifact server
-  shell: swift upload --quiet --header "X-Delete-After:{{ artcl_swift_delete_after }}" {{ artcl_swift_container }}/{{ lookup('env', 'BUILD_TAG') }} *
+  shell: swift upload --quiet --header "X-Delete-After:{{ artcl_swift_delete_after }}" {{ artcl_swift_container }}/{{ artcl_build_tag }} *
   args:
     chdir: "{{ artcl_collect_dir }}"
   changed_when: true

tasks/publish_ara_graphite.yml

@@ -10,5 +10,5 @@
     graphite_host: "{{ ara_graphite_server }}"
     ara_mapping: "{{ ara_tasks_map }}"
     ara_data: "{{ ara_data.stdout|to_json }}"
-    graphite_prefix: "{{ ara_graphite_prefix }}"
+    graphite_prefix: "{{ ara_graphite_prefix | default('') }}"
     only_successful_tasks: "{{ ara_only_successful_tasks }}"

test-playbooks/zuul-ansible-role-collect-logs.yaml

@@ -0,0 +1,6 @@
+---
+- hosts: all
+  tasks:
+    - name: include ansible-role-collect-logs role
+      include_role:
+        name: collect-logs

vars/unsecure.yml

@@ -0,0 +1,10 @@
+---
+# these vars are defaults overrides which are loaded only when the role is
+# running in an unsecured environment (outside zuul).
+# * any call env lookup plugin under zuul would generate a runtime exception
+ara_graphite_prefix: "tripleo.{{ lookup('env', 'STABLE_RELEASE')|default('master', true) }}.{{ lookup('env', 'TOCI_JOBTYPE') }}."
+artcl_build_url: "{{ lookup('env', 'BUILD_URL') }}"
+artcl_build_tag: "{{ lookup('env', 'BUILD_TAG') }}"
+artcl_rsync_password: "{{ lookup('env', 'RSYNC_PASSWORD') }}"
+influxdb_data_file_path: "{{ lookup('env', 'LOCAL_WORKING_DIR')|default('/tmp', true) }}/influxdb_data"
+local_working_dir: "{{ zuul_work_dir | default(lookup('env', 'HOME')) }}/.quickstart"

zuul.d/layout.yaml

@@ -1,4 +1,13 @@
 ---
+- job:
+    name: zuul-ansible-role-collect-logs
+    description: Validate that zuul can use that role.
+    parent: base
+    run: test-playbooks/zuul-ansible-role-collect-logs.yaml
+    roles:
+      - zuul: opendev.org/openstack/ansible-role-collect-logs
+        name: collect-logs
+
 - project:
     templates:
       - tripleo-undercloud-jobs
@@ -15,6 +24,7 @@
         - tripleo-ci-centos-8-scenario002-standalone
         - tripleo-ci-centos-8-scenario010-ovn-provider-standalone
         - tripleo-ci-centos-8-containers-multinode
+        - zuul-ansible-role-collect-logs
     gate:
       jobs: *jobs
       queue: tripleo