Add missing roles

These roles, originally from openstack-infra/project-config,
are used by jobs in this repo.

Change-Id: Ia8d087dba865eb2db38bb58ee11e2bdaaffe3f74

roles/submit-logstash-jobs/README.rst

@@ -0,0 +1,44 @@
+Submit a log processing job to the logstash workers.
+
+This role examines all of the files in the log subdirectory of the job
+work dir and any matching filenames are submitted to the gearman queue
+for the logstash log processor, along with any tags configured for
+those filenames.
+
+**Role Variables**
+
+.. zuul:rolevar:: logstash_gearman_server
+   :default: logstash.openstack.org
+
+   The gearman server to use.
+
+.. zuul:rolevar:: logstash_processor_config
+   :type: dict
+
+   The default file configuration for the logstash parser.
+
+   This is a dictionary that contains a single entry:
+
+   .. zuul:rolevar:: files
+      :type: list
+
+      A list of files to search for in the ``work/logs/`` directory on
+      the executor.  Each file will be compared to the entries in this
+      list, and if it matches, a processing job will be submitted to
+      the logstash processing queue, along with the tags for the
+      matching entry.  Order is important: the first matcing is used.
+      This field is list of dictionaries, as follows:
+
+      .. zuul:rolevar:: name
+
+         The name of the file to process.  This is treated as an
+         unanchored regular expression.  To match the full path
+         (underneath ``work/logs``) start and end the string with
+         ``^`` and ``$`` respectively.
+
+      .. zuul:rolevar:: tags
+         :type: list
+
+         A list of strings indicating the logstash processing tags
+         associated with this file.  These may be used to indicate the
+         file format to the parser.

roles/submit-logstash-jobs/defaults/main.yaml

@@ -0,0 +1,88 @@
+logstash_gearman_server: logstash.openstack.org
+# For every file found in the logs directory (and its subdirs), the
+# module will attempt to match the filenames below.  If there is a
+# match, the file is submitted to the logstash processing queue, along
+# with the tags for that match.  The first match wins, so be sure to
+# list more specific names first.  The names are un-anchored regular
+# expressions (so if you need to match the root (i.e, the work/logs/
+# directory), be sure to anchor them with ^).
+logstash_processor_config:
+  files:
+    - name: job-output\.txt
+      tags:
+        - console
+        - console.html
+    - name: grenade\.sh\.txt
+      tags:
+        - console
+        - console.html
+    - name: devstacklog\.txt(?!.*summary)
+      tags:
+        - console
+        - console.html
+    - name: apache/keystone\.txt
+      tags:
+        - screen
+        - oslofmt
+    - name: apache/horizon_error\.txt
+      tags:
+        - apacheerror
+    # TODO(clarkb) Add swift proxy logs here.
+    - name: syslog\.txt
+      tags:
+        - syslog
+    - name: tempest\.txt
+      tags:
+        - screen
+        - oslofmt
+    - name: javelin\.txt
+      tags:
+        - screen
+        - oslofmt
+    # Neutron index log files (files with messages from all test cases)
+    - name: dsvm-functional-index\.txt
+      tags:
+        - oslofmt
+    - name: dsvm-fullstack-index\.txt
+      tags:
+        - oslofmt
+    - name: screen-s-account\.txt
+      tags:
+        - screen
+        - apachecombined
+    - name: screen-s-container\.txt
+      tags:
+        - screen
+        - apachecombined
+    - name: screen-s-object\.txt
+      tags:
+        - screen
+        - apachecombined
+    # tripleo logs
+    - name: postci\.txt
+      tags:
+        - console
+        - postci
+    - name: var/log/extra/logstash\.txt
+      tags:
+        - console
+        - postci
+    - name: var/log/extra/errors\.txt
+      tags:
+        - console
+        - errors
+    # wildcard logs
+    - name: devstack-gate-.*\.txt
+      tags:
+        - console
+        - console.html
+    # NOTE(mriedem): Logs that are known logstash index OOM killers are
+    # blacklisted here until fixed.
+    # screen-kubelet.txt: https://bugs.launchpad.net/kuryr-kubernetes/+bug/1795067
+    # screen-mistral-engine.txt: https://bugs.launchpad.net/mistral/+bug/1795068
+    # screen-monasca-persister.txt: https://storyboard.openstack.org/#!/story/2003911
+    # screen-ovn-northd.txt: https://bugs.launchpad.net/networking-ovn/+bug/1795069
+    - name: screen-(?!(peakmem_tracker|dstat|karaf|kubelet|mistral-engine|monasca-persister|ovn-northd)).*\.txt
+      tags:
+        - screen
+        - oslofmt

roles/submit-logstash-jobs/meta/main.yaml

@@ -0,0 +1,2 @@
+dependencies:
+  - role: submit-log-processor-jobs

roles/submit-logstash-jobs/tasks/main.yaml

@@ -0,0 +1,9 @@
+- name: Submit logstash processing jobs to log processors
+  submit_log_processor_jobs:
+    gearman_server: "{{ logstash_gearman_server }}"
+    job: "push-log"
+    config: "{{ logstash_processor_config }}"
+    success: "{{ zuul_success }}"
+    host_vars: "{{ hostvars }}"
+    path: "{{ zuul.executor.log_root }}"
+    log_url: "{{ (lookup('file', zuul.executor.result_data_file) | from_json).get('zuul').get('log_url') }}"

roles/use-cached-repos/README.rst

@@ -0,0 +1,9 @@
+Clone repos from the image cache into the workspace
+
+Clon any repos cached on the remote node (via the image build process)
+that this job uses into the workspace.  If a repo doesn't exist, clone
+it from its source.
+
+These repos are neither up to date, nor do they contain the changes
+that Zuul has prepared.  A separate role should synchronize the local
+and remote repos.

roles/use-cached-repos/tasks/main.yaml

@@ -0,0 +1,41 @@
+- name: Find locally cached git repos
+  stat:
+    path: "/opt/git/{{ item.name }}"
+  with_items: "{{ zuul.projects.values() | list }}"
+  register: cached_repos
+
+- name: Clone cached repo to workspace
+  command: "git clone /opt/git/{{ item.0.name }} {{ ansible_user_dir }}/{{ item.0.src_dir}}"
+  args:
+    creates: "{{ ansible_user_dir }}/{{ item.0.src_dir}}"
+  when: item.1.stat.exists
+  with_together:
+    - "{{ zuul.projects.values() | list }}"
+    - "{{ cached_repos.results }}"
+  # ANSIBLE0006: If we use the git module, we get warning
+  # ANSIBLE0004 since we  do not give an explicit version
+  tags:
+    - skip_ansible_lint
+
+- name: Clone upstream repo to workspace
+  command: "git clone https://{{ item.0.canonical_hostname }}/{{ item.0.name}} {{ ansible_user_dir }}/{{ item.0.src_dir}}"
+  args:
+    creates: "{{ ansible_user_dir }}/{{ item.0.src_dir}}"
+  when: not item.1.stat.exists
+  with_together:
+    - "{{ zuul.projects.values() | list }}"
+    - "{{ cached_repos.results }}"
+  # ANSIBLE0006: If we use the git module, we get warning
+  # ANSIBLE0004 since we  do not give an explicit version
+  tags:
+    - skip_ansible_lint
+
+- name: Remove origin from local git repos
+  # To be idempotent, remove origin only if it's found in the local list.
+  shell: "git remote -v | grep origin && git remote rm origin || true"
+  args:
+    chdir: "{{ ansible_user_dir }}/{{ item.src_dir}}"
+  with_items: "{{ zuul.projects.values() | list }}"
+  # ANSIBLE0006: git remote is not supported by ansible module
+  tags:
+    - skip_ansible_lint