Add option for object store friendly log paths

For logs stored in object storage services we want to be able to shard
into different containers. This improves scalability of the object
stores. To do this consistently we prefix all log paths with the first 3
characters of the build uuid. This way we get container names like
log_123, log_abc, and so on for a total of 4096 containers.

This behavior is off by default because users of filesystems stores
without a zuul dashboard may still want periodic/ to be filed into a top
level dir for browseability.

Change-Id: I8e72a6e587edcbdf89b793cad2d7c96c535601e7

roles/emit-job-header/README.rst

@@ -5,3 +5,13 @@ Log a few lines about the job.
 .. zuul:rolevar:: zuul_log_url
 
    Base URL where logs are to be found.
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+   :default: False
+
+   This var is consumed by set-zuul-log-path-fact which emit-job-header
+   calls into. If you set this you will get log paths prefixed with the
+   first three characters of the build uuid. This will improve log file
+   sharding.
+
+   More details can be found at :zuul:rolevar:`set-zuul-log-path-fact.zuul_log_path_shard_build`

roles/set-zuul-log-path-fact/README.rst

@@ -1 +1,12 @@
 Sets a fact named ``zuul_log_path`` from zuul variables
+
+**Role Variables**
+
+.. zuul:rolevar:: zuul_log_path_shard_build
+   :type: bool
+   :default: False
+
+   Flag to specify whether or not paths that include a three character
+   prefix based on the build uuid should prefix the log path. This is
+   particularly useful for object storage systems where we want to
+   spread out the number of files per container.

roles/set-zuul-log-path-fact/defaults/main.yaml

@@ -0,0 +1 @@
+zuul_log_path_shard_build: false

roles/set-zuul-log-path-fact/tasks/main.yaml

@@ -1,14 +1,35 @@
-- name: Set log path for a change
+- name: Fileserver friendly log path specifications
-  when: zuul.change is defined
+  when: not zuul_log_path_shard_build
-  set_fact:
+  block:
-    zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+    - name: Set log path for a change
+      when: zuul.change is defined
+      set_fact:
+        zuul_log_path: "{{ zuul.change[-2:] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
 
-- name: Set log path for a ref update
+    - name: Set log path for a ref update
-  when: zuul.newrev is defined
+      when: zuul.newrev is defined
-  set_fact:
+      set_fact:
-    zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+        zuul_log_path: "{{ zuul.newrev[:2] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
 
-- name: Set log path for a periodic job
+    - name: Set log path for a periodic job
-  when: zuul.change is not defined and zuul.newrev is not defined
+      when: zuul.change is not defined and zuul.newrev is not defined
-  set_fact:
+      set_fact:
-    zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+        zuul_log_path: "{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+- name: object store friendly log path specifications
+  when: zuul_log_path_shard_build
+  block:
+    - name: Set log path for a change
+      when: zuul.change is defined
+      set_fact:
+        zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.change }}/{{ zuul.patchset }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+    - name: Set log path for a ref update
+      when: zuul.newrev is defined
+      set_fact:
+        zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.newrev }}/{{ zuul.pipeline }}/{{ zuul.job }}/{{ zuul.build[:7] }}"
+
+    - name: Set log path for a periodic job
+      when: zuul.change is not defined and zuul.newrev is not defined
+      set_fact:
+        zuul_log_path: "{{ zuul.build[:3] }}/{{ zuul.pipeline }}/{{ zuul.project.canonical_name }}/{{ zuul.branch }}/{{ zuul.job }}/{{ zuul.build[:7] }}"

test-playbooks/base-roles/emit-job-header.yaml

@@ -1,4 +1,29 @@
-- name: Test the emit-job-header role
+- name: Test the emit-job-header role with swift
+  hosts: all
+  roles:
+    - role: emit-job-header
+      zuul_log_url: "http://logs.openstack.org"
+      zuul_log_path_shard_build: true
+  post_tasks:
+    # All emit-job-header does is a debug statement so the worst that would
+    # happen would be that the debug task would fail outright and we'd prevent
+    # something breaking that debug statement from merging just by running it.
+    # However, the emit-job-header role includes the set-zuul-log-path-fact
+    # role. We can only test for zuul_log_path against changes, though.
+    - name: Assert zuul_log_path by set-zuul-log-path-fact for a change
+      assert:
+        that:
+          - zuul_log_path is defined
+          - zuul.change in zuul_log_path
+          - zuul.patchset in zuul_log_path
+          - zuul.pipeline in zuul_log_path
+          - zuul.job in zuul_log_path
+          - zuul.build[:3] == zuul_log_path[:3]
+
+# Note that the order of these two plays is important. We want the second one
+# to run to be the one the creates the correct log path for the currently
+# running system.
+- name: Test the emit-job-header role without swift
   hosts: all
   roles:
     - role: emit-job-header
@@ -17,3 +42,4 @@
           - zuul.patchset in zuul_log_path
           - zuul.pipeline in zuul_log_path
           - zuul.job in zuul_log_path
+          - zuul.build[:3] != zuul_log_path[:3]