diff --git a/roles/submit-logstash-jobs/README.rst b/roles/submit-logstash-jobs/README.rst new file mode 100644 index 0000000..eff2320 --- /dev/null +++ b/roles/submit-logstash-jobs/README.rst @@ -0,0 +1,44 @@ +Submit a log processing job to the logstash workers. + +This role examines all of the files in the log subdirectory of the job +work dir and any matching filenames are submitted to the gearman queue +for the logstash log processor, along with any tags configured for +those filenames. + +**Role Variables** + +.. zuul:rolevar:: logstash_gearman_server + :default: logstash.openstack.org + + The gearman server to use. + +.. zuul:rolevar:: logstash_processor_config + :type: dict + + The default file configuration for the logstash parser. + + This is a dictionary that contains a single entry: + + .. zuul:rolevar:: files + :type: list + + A list of files to search for in the ``work/logs/`` directory on + the executor. Each file will be compared to the entries in this + list, and if it matches, a processing job will be submitted to + the logstash processing queue, along with the tags for the + matching entry. Order is important: the first matcing is used. + This field is list of dictionaries, as follows: + + .. zuul:rolevar:: name + + The name of the file to process. This is treated as an + unanchored regular expression. To match the full path + (underneath ``work/logs``) start and end the string with + ``^`` and ``$`` respectively. + + .. zuul:rolevar:: tags + :type: list + + A list of strings indicating the logstash processing tags + associated with this file. These may be used to indicate the + file format to the parser. diff --git a/roles/submit-logstash-jobs/defaults/main.yaml b/roles/submit-logstash-jobs/defaults/main.yaml new file mode 100644 index 0000000..261f893 --- /dev/null +++ b/roles/submit-logstash-jobs/defaults/main.yaml @@ -0,0 +1,88 @@ +logstash_gearman_server: logstash.openstack.org +# For every file found in the logs directory (and its subdirs), the +# module will attempt to match the filenames below. If there is a +# match, the file is submitted to the logstash processing queue, along +# with the tags for that match. The first match wins, so be sure to +# list more specific names first. The names are un-anchored regular +# expressions (so if you need to match the root (i.e, the work/logs/ +# directory), be sure to anchor them with ^). +logstash_processor_config: + files: + - name: job-output\.txt + tags: + - console + - console.html + - name: grenade\.sh\.txt + tags: + - console + - console.html + - name: devstacklog\.txt(?!.*summary) + tags: + - console + - console.html + - name: apache/keystone\.txt + tags: + - screen + - oslofmt + - name: apache/horizon_error\.txt + tags: + - apacheerror + # TODO(clarkb) Add swift proxy logs here. + - name: syslog\.txt + tags: + - syslog + - name: tempest\.txt + tags: + - screen + - oslofmt + - name: javelin\.txt + tags: + - screen + - oslofmt + # Neutron index log files (files with messages from all test cases) + - name: dsvm-functional-index\.txt + tags: + - oslofmt + - name: dsvm-fullstack-index\.txt + tags: + - oslofmt + - name: screen-s-account\.txt + tags: + - screen + - apachecombined + - name: screen-s-container\.txt + tags: + - screen + - apachecombined + - name: screen-s-object\.txt + tags: + - screen + - apachecombined + # tripleo logs + - name: postci\.txt + tags: + - console + - postci + - name: var/log/extra/logstash\.txt + tags: + - console + - postci + - name: var/log/extra/errors\.txt + tags: + - console + - errors + # wildcard logs + - name: devstack-gate-.*\.txt + tags: + - console + - console.html + # NOTE(mriedem): Logs that are known logstash index OOM killers are + # blacklisted here until fixed. + # screen-kubelet.txt: https://bugs.launchpad.net/kuryr-kubernetes/+bug/1795067 + # screen-mistral-engine.txt: https://bugs.launchpad.net/mistral/+bug/1795068 + # screen-monasca-persister.txt: https://storyboard.openstack.org/#!/story/2003911 + # screen-ovn-northd.txt: https://bugs.launchpad.net/networking-ovn/+bug/1795069 + - name: screen-(?!(peakmem_tracker|dstat|karaf|kubelet|mistral-engine|monasca-persister|ovn-northd)).*\.txt + tags: + - screen + - oslofmt diff --git a/roles/submit-logstash-jobs/meta/main.yaml b/roles/submit-logstash-jobs/meta/main.yaml new file mode 100644 index 0000000..9f28a12 --- /dev/null +++ b/roles/submit-logstash-jobs/meta/main.yaml @@ -0,0 +1,2 @@ +dependencies: + - role: submit-log-processor-jobs diff --git a/roles/submit-logstash-jobs/tasks/main.yaml b/roles/submit-logstash-jobs/tasks/main.yaml new file mode 100644 index 0000000..5324233 --- /dev/null +++ b/roles/submit-logstash-jobs/tasks/main.yaml @@ -0,0 +1,9 @@ +- name: Submit logstash processing jobs to log processors + submit_log_processor_jobs: + gearman_server: "{{ logstash_gearman_server }}" + job: "push-log" + config: "{{ logstash_processor_config }}" + success: "{{ zuul_success }}" + host_vars: "{{ hostvars }}" + path: "{{ zuul.executor.log_root }}" + log_url: "{{ (lookup('file', zuul.executor.result_data_file) | from_json).get('zuul').get('log_url') }}" diff --git a/roles/use-cached-repos/README.rst b/roles/use-cached-repos/README.rst new file mode 100644 index 0000000..8ce3516 --- /dev/null +++ b/roles/use-cached-repos/README.rst @@ -0,0 +1,9 @@ +Clone repos from the image cache into the workspace + +Clon any repos cached on the remote node (via the image build process) +that this job uses into the workspace. If a repo doesn't exist, clone +it from its source. + +These repos are neither up to date, nor do they contain the changes +that Zuul has prepared. A separate role should synchronize the local +and remote repos. diff --git a/roles/use-cached-repos/tasks/main.yaml b/roles/use-cached-repos/tasks/main.yaml new file mode 100644 index 0000000..4456db7 --- /dev/null +++ b/roles/use-cached-repos/tasks/main.yaml @@ -0,0 +1,41 @@ +- name: Find locally cached git repos + stat: + path: "/opt/git/{{ item.name }}" + with_items: "{{ zuul.projects.values() | list }}" + register: cached_repos + +- name: Clone cached repo to workspace + command: "git clone /opt/git/{{ item.0.name }} {{ ansible_user_dir }}/{{ item.0.src_dir}}" + args: + creates: "{{ ansible_user_dir }}/{{ item.0.src_dir}}" + when: item.1.stat.exists + with_together: + - "{{ zuul.projects.values() | list }}" + - "{{ cached_repos.results }}" + # ANSIBLE0006: If we use the git module, we get warning + # ANSIBLE0004 since we do not give an explicit version + tags: + - skip_ansible_lint + +- name: Clone upstream repo to workspace + command: "git clone https://{{ item.0.canonical_hostname }}/{{ item.0.name}} {{ ansible_user_dir }}/{{ item.0.src_dir}}" + args: + creates: "{{ ansible_user_dir }}/{{ item.0.src_dir}}" + when: not item.1.stat.exists + with_together: + - "{{ zuul.projects.values() | list }}" + - "{{ cached_repos.results }}" + # ANSIBLE0006: If we use the git module, we get warning + # ANSIBLE0004 since we do not give an explicit version + tags: + - skip_ansible_lint + +- name: Remove origin from local git repos + # To be idempotent, remove origin only if it's found in the local list. + shell: "git remote -v | grep origin && git remote rm origin || true" + args: + chdir: "{{ ansible_user_dir }}/{{ item.src_dir}}" + with_items: "{{ zuul.projects.values() | list }}" + # ANSIBLE0006: git remote is not supported by ansible module + tags: + - skip_ansible_lint