openstack
/
ansible-role-collect-logs
Code Issues Proposed changes

Create infrared plugin 

The review adds required files (infrared_plugin/ folder) by Infrared
in order to make the role an infrared plugin.

- adds 'infrared' molecule scenario (run with pytest -k infrared)
- fix missing html report generation from tox molecule

Change-Id: I5db914db44c354c908d0e12726bebf93af239443
Task: https://tree.taiga.io/project/tripleo-ci-board/task/1276
This commit is contained in:
Martin Kopec 2019-09-13 12:58:33 +00:00
parent 075b69a5fe
commit 25d62887fe
11 changed files with 385 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -26,6 +26,7 @@ container_registry.egg-info/
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
!infrared_plugin/plugin.spec
# Installer logs
pip-log.txt

45
README.rst
View File

@ -107,22 +107,23 @@ Publishing related
ssh to connect
- ``artcl_rsync_url`` – rsync target for uploading the logs. The
localhost needs to have passwordless authentication to the target or
the ``PROVISIONER_KEY`` Var specificed in the environment.
the ``PROVISIONER_KEY`` var specified in the environment.
- ``artcl_use_swift``: false/true – use swift object storage to publish
the logs
- ``artcl_swift_auth_url`` – the OpenStack auth URL for Swift
- ``artcl_swift_username`` – OpenStack username for Swift
- ``artcl_swift_password`` – password for the Swift user
- ``artcl_swift_tenant_name`` – OpenStack tenant name for Swift
- ``artcl_swift_tenant_name`` – OpenStack tenant (project) name for Swift
- ``artcl_swift_container`` – the name of the Swift container to use,
default is ``logs``
- ``artcl_swift_delete_after`` – The number of seconds after which
Swift will remove the uploaded objects, the default is 2678400
seconds = 31 days.
- ``artcl_artifact_url`` – a HTTP URL at which the uploaded logs will
- ``artcl_artifact_url`` – An HTTP URL at which the uploaded logs will
be accessible after upload.
- ``artcl_collect_sosreport`` – true/false – If true, create and
collect a sosreport for each host.
- ``artcl_report_server_key`` - A path to a key for an access to the report
server.
Logs parsing
~~~~~~~~~~~~
@ -166,7 +167,7 @@ Example Role Playbook
- collect-logs
** Note:
The tasks that collect data from the nodes is executed with ignore_errors.
The tasks that collect data from the nodes are executed with ignore_errors.
For `example: <https://opendev.org/openstack/ansible-role-collect-logs/src/branch/master/tasks/collect/system.yml#L3>`__
Templated Bash to rST Conversion Notes
@ -228,6 +229,38 @@ The task searches for files containing the sensitive strings
(orig_string) within a file path, and then replaces the sensitive
strings in those files with the sanitized_string.
Usage with InfraRed
-------------------
Run the following steps to execute the role by
`infrared <https://infrared.readthedocs.io/en/latest/>`__.
1. Install infrared and add ansible-role-collect-logs plugin by providing
the url to this repo:
.. code-block::
(infrared)$ ir plugin add https://opendev.org/openstack/ansible-role-collect-logs.git --src-path infrared_plugin
2. Verify that the plugin is imported by:
.. code-block::
(infrared)$ ir plugin list
3. From infrared directory symlink roles path:
.. code-block::
$ ln -s $(pwd)/plugins $(pwd)/plugins/ansible-role-collect-logs/infrared_plugin/roles
4. Run the plugin:
.. code-block::
(infrared)$ ir ansible-role-collect-logs
License
-------

67
infrared_plugin/main.yml Normal file
View File

@ -0,0 +1,67 @@
---
# This file and plugin.spec are required by Infrared project
# This section collects data from the nodes
- hosts: "{{ other.openstack_nodes }}"
remote_user: "{{ other.remote_user }}"
tasks:
- name: Remap infrared parameters to role variables
set_fact:
"{{ item.key }}": "{{ item.value }}"
with_dict: "{{ other }}"
- name: Ansible role collect logs
include_role:
name: ansible-role-collect-logs
# This section takes care of preparing the collected data for publishing
# and for publishing itself
- hosts: localhost
tasks:
- name: Remap infrared parameters to role variables
set_fact:
"{{ item.key }}": "{{ item.value }}"
with_dict: "{{ other }}"
- name: Disable artcl_collect to prepare for publishing
set_fact:
# override artcl_collect to false because in ansible-role-collect-logs
# role collect and publish tasks are complementary
artcl_collect: false
when: artcl_publish|bool
- name: Set path to a report server key
set_fact:
artcl_report_server_key: "-i {{ artcl_report_server_key }}"
when: artcl_report_server_key is defined
- name: Extract the logs
shell: |
cat *.tar | tar xf - -i
args:
chdir: "{{ artcl_collect_dir }}"
executable: /bin/bash
ignore_errors: true
- name: delete the tar file after extraction
shell: |
rm -r *.tar
args:
chdir: "{{ artcl_collect_dir }}"
executable: /bin/bash
ignore_errors: true
- name: Ansible role collect logs
include_role:
name: ansible-role-collect-logs
# TODO: local_working_dir should be unique per each run - a job should
# never fail due to a reason that f.e. the artifacts weren't removed
# properly in a previous run
- name: Delete artifact files from localhost
file:
state: absent
path: "{{ local_working_dir }}"
ignore_errors: true

156
infrared_plugin/plugin.spec Normal file
View File

@ -0,0 +1,156 @@
---
# This file and main.yml are required by Infrared project
config:
plugin_type: other
entry_point: main.yml
subparsers:
ansible-role-collect-logs:
description: An Ansible role for aggregating logs from different nodes.
include_groups: ["Ansible options", "Common options"]
groups:
- title: Collecting
options:
openstack_nodes:
type: Value
help: |
OpenStack nodes ansible-role-collect-logs will be executed on.
default: all:!localhost
artcl_report_server_key:
type: Value
help: |
A path to a key for an access to the report server.
artcl_rsync_path:
type: Value
help: |
Specifies a server hostname and a path where the artifacts will
be stored. Example: username@hostname:/path/to/the/dest
artcl_collect_list:
type: ListValue
help: |
A list of files and directories to gather from the target.
Directories are collected recursively and need to end with a “/”
to get collected. Should be specified as a YaML list, e.g.:
infrared ansible-role-collect-logs \
--artcl_collect_list /etc/nova/,/home/stack/*.log,/var/log/
artcl_collect_list_append:
type: ListValue
help: |
A list of files and directories to be appended in the default
list. This is useful for users that want to keep the original
list and just add more relevant paths.
artcl_exclude_list:
type: ListValue
help: |
A list of files and directories to exclude from collecting. This
list is passed to rsync as an exclude filter and it takes
precedence over the collection list. For details see the
“FILTER RULES” topic in the rsync man page.
local_working_dir:
type: Value
help: |
Destination on the executor host where the logs will be collected
to.
default: /tmp/collect_logs
artcl_collect_dir:
type: Value
help: |
A directory on the executor host within local_working_dir where
the logs should be gathered, without a trailing slash.
artcl_gzip_only:
type: Bool
help: |
When true, gathered files are gzipped one by one
in artcl_collect_dir, when false, a tar.gz file will contain all
the logs.
collect_log_types:
type: ListValue
help: |
A list of which type of logs will be collected, such as openstack
logs, network logs, system logs, etc. Acceptable values are
system, monitoring, network, openstack and container.
artcl_collect_sosreport:
type: Bool
help: |
If true, create and collect a sosreport for each host.
- title: Publishing
options:
artcl_publish:
type: Bool
help: |
If true, the role will attempt to rsync logs to the target
specified by artcl_rsync_url. Uses BUILD_URL, BUILD_TAG vars from
the environment (set during a Jenkins job run) and requires the
next to variables to be set.
artcl_txt_rename:
type: Bool
help: |
Rename compressed text based files to end with txt.gz extension.
artcl_publish_timeout:
type: Value
help: |
The maximum seconds the role can spend uploading the logs.
artcl_use_rsync:
type: Bool
help: |
If true, the role will use rsync to upload the logs.
artcl_rsync_use_daemon:
type: Bool
help: |
If true, the role will use rsync daemon instead of ssh to
connect.
artcl_rsync_url:
type: Value
help: |
rsync target for uploading the logs. The localhost needs to have
passwordless authentication to the target or the PROVISIONER_KEY
var specificed in the environment.
artcl_use_swift:
type: Bool
help: |
If true, the role will use swift object storage to publish
the logs.
artcl_swift_auth_url:
type: Value
help: |
The OpenStack auth URL for Swift.
artcl_swift_username:
type: Value
help: |
OpenStack username for Swift.
artcl_swift_password:
type: Value
help: |
Password for the Swift user.
artcl_swift_tenant_name:
type: Value
help: |
OpenStack tenant name for Swift.
artcl_swift_container:
type: Value
help: |
The name of the Swift container to use.
artcl_swift_delete_after:
type: Value
help: |
The number of seconds after which Swift will remove the uploaded
objects.
artcl_artifact_url:
type: Value
help: |
An HTTP URL at which the uploaded logs will be accessible after
upload.
influxdb_create_data_file:
type: Bool
help: |
Upload data to the InfluxDB database.
default: False
ara_generate_html:
type: Bool
help: |
Whether to generate ara html or not.
default: False
remote_user:
type: Value
help: |
Name of a remote user under which the tasks will be executed.
default: stack

29
molecule/infrared/cleanup.yml Normal file
View File

@ -0,0 +1,29 @@
---
- name: Cleanup
hosts: all
tasks:
- name: Ansible check file exists example.
stat:
path: "{{ infrared_venv }}/bin/activate"
register: former_venv
- name: "Uninstall ansible-role-collect-logs plugin"
shell: |
export PATH=$PATH:/usr/local/sbin:/usr/sbin
source {{ infrared_venv }}/bin/activate
ir plugin remove ansible-role-collect-logs
ir plugin list
args:
executable: /bin/bash
register: plugin_uninstall_output
when: former_venv.stat.exists
- name: "Debug, plugin uninstallation"
debug:
msg: "{{ plugin_uninstall_output }}"
- name: "Clean Infrared venv"
file:
path: "{{ infrared_location }}"
state: absent

1
molecule/infrared/molecule.yml Symbolic link
View File

@ -0,0 +1 @@
../sova/molecule.yml

56
molecule/infrared/playbook.yml Normal file
View File

@ -0,0 +1,56 @@
---
- name: Converge
hosts: all
tasks:
- name: "Download Infrared"
git:
repo: "https://github.com/redhat-openstack/infrared.git"
version: "master"
dest: "{{ infrared_location }}"
update: true
- name: "Create Infrared venv"
pip:
name: "{{ item }}"
virtualenv: "{{ infrared_venv }}"
with_items:
- pip
- setuptools
- pbr
- name: "Install Infrared"
pip:
name: "."
virtualenv: "{{ infrared_venv }}"
chdir: "{{ infrared_location }}"
tags:
# this task is always changed, the problem is on pip module side:
# https://github.com/ansible/ansible/issues/28952
- molecule-idempotence-notest
- name: "Create infrared_plugin dir"
file:
path: "{{ infrared_location }}/infrared_plugin"
state: directory
- name: "Install ansible-role-collect-logs plugin"
shell: |
export PATH=$PATH:/usr/local/sbin:/usr/sbin
source {{ infrared_venv }}/bin/activate
ir plugin add {{ playbook_dir }}/../../ --src-path infrared_plugin
args:
chdir: "{{ infrared_location }}"
executable: /bin/bash
register: plugin_install_output
changed_when: true
tags:
# the task is always changed, skip idempotence
- molecule-idempotence-notest
- name: "Debug: output from plugin installation task main playbook"
debug:
msg: "{{ plugin_install_output }}"
tags:
# the task depends on a taks which skips idempotence
- molecule-idempotence-notest

23
molecule/infrared/verify.yml Normal file
View File

@ -0,0 +1,23 @@
---
- name: Converge
hosts: all
tasks:
- name: "List available plugins"
shell: |
export PATH=$PATH:/usr/local/sbin:/usr/sbin
source {{ infrared_venv }}/bin/activate
ir plugin list
args:
executable: /bin/bash
register: plugin_output
changed_when: false
- name: "Verify playbook list plugins output"
debug:
msg: "{{ plugin_output }}"
- name: "Check if ansible-role-collect-logs is present"
fail:
msg: "ansible-role-collect-logs not installed"
when: "'ansible-role-collect-logs' not in plugin_output.stdout"

20
molecule/sova/molecule.yml
View File

@ -8,20 +8,20 @@ driver:
log: true
lint:
name: yamllint
enabled: false
platforms:
- name: instance
provisioner:
name: ansible
lint:
name: ansible-lint
# Need to fix a whole role
options:
x:
- '301'
- '302'
- '303'
- '305'
config_options:
defaults:
timeout: 30
forks: 500
inventory:
group_vars:
all:
infrared_location: "{{ playbook_dir }}/.venv"
infrared_venv: "{{ infrared_location }}/.infrared"
verifier:
name: ansible
lint:

2
tasks/publish.yml
View File

@ -143,7 +143,7 @@
- name: upload to the artifact server using pubkey auth
shell: >
rsync -av
--quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null"
--quiet -e "ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null {{ artcl_report_server_key|default('') }}"
{{ artcl_collect_dir }}/ {{ artcl_rsync_path }}/{{ lookup('env', 'BUILD_TAG') }}
async: "{{ artcl_publish_timeout }}"
poll: 15

3
tox.ini
View File

@ -1,5 +1,6 @@
[tox]
minversion = 2.0
# for tty option
minversion = 3.4.0
envlist = docs, linters, molecule
skipdist = True