Add ansible role that is configuring logscraper

With this commit, logscraper tool can be started as a service
inside the container.

Change-Id: I7349929bcb314f55d648bb4db4291fd53fa83d07

.zuul.yaml

@@ -4,6 +4,7 @@
       - build-tox-docs
     check: &logcheck
       jobs:
+        - openstack-tox-linters
         - openstack-tox-pep8
         - openstack-tox-py38
     gate: *logcheck

README.rst

@@ -1,4 +1,4 @@
-Openstack CI log processing
+OpenStack CI log processing
 ===========================
 
 The goal of this repository is to provide and check
@@ -16,7 +16,7 @@ the log processing system.
 Testing
 -------
 
-The part of Openstack CI log processing runs a complete testing and
+The part of OpenStack CI log processing runs a complete testing and
 continuous-integration environment, powered by `Zuul
 <https://zuul-ci.org/>`__.
 

ansible/playbooks/roles

@@ -0,0 +1 @@
+../roles

ansible/playbooks/service-logscraper.yml

@@ -0,0 +1,6 @@
+---
+- name: Configure Logscraper tool
+  hosts: logscraper01.openstack.org
+  become: true
+  roles:
+    - logscraper

ansible/roles/logscraper/README.rst

@@ -0,0 +1,80 @@
+Logscraper
+==========
+
+The goal of this role is to setup and configure service related
+to logscraper script which is responsible to to push recent
+zuul builds into log gearman processor.
+
+Requirements
+------------
+
+None
+
+Role Variables
+--------------
+
+The role is automatically deploying service related to the
+log scrape service. Depends of what is set to the `tenant_builds` var,
+it can start multiple services on same host with different name,
+for example:
+
+.. code-block:: yaml
+
+  vars:
+    tenant_builds:
+      - tenant: openstack
+        gearman_port: 4731
+        gearman_server: logstash.openstack.org
+        zuul_api_url: https://zuul.opendev.org/api/tenant/openstack
+        insecure: false
+
+will deploy service with name: `logscraper@openstack.service`.
+It is because on one service we are able to deploy multiple instances
+of logscraper and each of them will be responsible for checking
+and pushing logs for own tenant.
+
+Dependencies
+------------
+
+None
+
+Example Playbook
+----------------
+
+Playbook responsible for deploying service can look like:
+
+Below is a playbook example, responsible for deploying two logscraper
+services, where one will responsible to get logs from `openstack` tenant
+and second one for getting logs from `sometenant` tenant.
+
+.. code-block:: yaml
+
+  - name: Configure Logscraper tool
+    hosts: localhost
+    become: true
+    vars:
+      tenant_builds:
+        - tenant: openstack
+          gearman_port: 4731
+          gearman_server: logstash.openstack.org
+          zuul_api_url: https://zuul.opendev.org/api/tenant/openstack
+          insecure: False
+        - tenant: sometenant
+          gearman_port: 4731
+          gearman_server: someproject.org
+          zuul_api_url: https://zuul.opendev.org/api/tenant/sometenant
+          insecure: True
+    roles:
+      - logscraper
+
+License
+-------
+
+Apache
+
+Author Information
+------------------
+
+Author: OpenStack Contributors
+Author email: openstack-discuss@lists.openstack.org
+Home page: http://docs.openstack.org/infra/ci-log-processing

ansible/roles/logscraper/defaults/main.yml

@@ -0,0 +1,22 @@
+---
+logscraper_user: logscraper
+logscraper_group: logscraper
+logscraper_dir: /etc/logscraper
+
+container_images:
+  # FIXME: Create new project on Docker hub that will contain that image
+  logscraper: quay.rdoproject.org/software-factory/logscraper:latest
+
+# Example:
+# tenant_builds:
+#   - tenant: openstack
+#     gearman_port: 4731
+#     gearman_server: logstash.openstack.org
+#     zuul_api_url: https://zuul.opendev.org/api/tenant/openstack
+#     insecure: False
+#   - tenant: sometenant
+#     gearman_port: 4731
+#     gearman_server: logstash.openstack.org
+#     zuul_api_url: https://zuul.opendev.org/api/tenant/sometenant
+#     insecure: True
+tenant_builds: []

ansible/roles/logscraper/meta/main.yml

@@ -0,0 +1,13 @@
+---
+galaxy_info:
+  author: Openstack Contributors
+  description: Openstack Logscraper tool
+  company: Openstack
+  license: Apache
+  min_ansible_version: 2.9
+  platforms:
+    - name: Centos
+      versions:
+        - 8
+  galaxy_tags: []
+dependencies: []

ansible/roles/logscraper/tasks/main.yml

@@ -0,0 +1,33 @@
+---
+- name: Create dedicated group
+  group:
+    name: "{{ logscraper_group }}"
+    state: present
+
+- name: Create dedicated user
+  user:
+    name: "{{ logscraper_user }}"
+    state: present
+    comment: "Dedicated user for logscraper"
+    group: "{{ logscraper_group }}"
+    shell: "/sbin/nologin"
+    create_home: false
+
+- name: Create dedicated directory
+  file:
+    path: "{{ logscraper_dir }}"
+    state: directory
+    owner: "{{ logscraper_user }}"
+    group: "{{ logscraper_group }}"
+
+- name: Ensure container software is installed
+  package:
+    name: podman
+    state: present
+
+- name: Pull image
+  shell: "podman pull {{ container_images['logscraper'] }}"
+
+- name: Configure logscraper service
+  include_tasks: service.yml
+  loop: "{{ tenant_builds }}"

ansible/roles/logscraper/tasks/service.yml

@@ -0,0 +1,29 @@
+---
+- name: Generate logscraper script
+  template:
+    src: logscraper.sh.j2
+    dest: "/usr/local/bin/logscraper-{{ item.tenant }}"
+    mode: '0755'
+  register: _start_script
+
+- name: Generate systemd unit
+  template:
+    src: logscraper.service.j2
+    dest: "/etc/systemd/system/logscraper-{{ item.tenant }}.service"
+    owner: root
+    group: root
+
+- name: Enable and restart service
+  service:
+    name: logscraper-{{ item.tenant }}
+    state: restarted
+    daemon-reload: true
+    enabled: true
+  when: _start_script.changed
+
+- name: Ensure that service is running
+  service:
+    name: logscraper-{{ item.tenant }}
+    state: started
+    daemon-reload: true
+    enabled: true

ansible/roles/logscraper/templates/logscraper.service.j2

@@ -0,0 +1,16 @@
+[Unit]
+Description=logscraper service for {{ item.tenant }}
+After=syslog.target network.target
+StartLimitInterval=20
+StartLimitBurst=5
+
+[Service]
+Type=simple
+SyslogIdentifier=logscraper-{{ item.tenant }}
+Restart=always
+RestartSec=3s
+ExecStop=-/usr/bin/podman stop -t 10 logscraper-{{ item.tenant }}
+ExecStart=/usr/local/bin/logscraper-{{ item.tenant }}
+
+[Install]
+WantedBy=multi-user.target

ansible/roles/logscraper/templates/logscraper.sh.j2

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+/usr/bin/podman run \
+    --network host \
+    --rm \
+    --name logscraper-{{ item.tenant }} \
+    --volume {{ logscraper_dir }}:{{ logscraper_dir }}:z \
+    {{ container_images['logscraper'] }} \
+    --gearman-port {{ item.gearman_port }} \
+    --gearman-server {{ item.gearman_server }} \
+    --checkpoint-file {{ item.checkpoint_file | default(logscraper_dir + '/checkpoint') }} \
+    --follow \
+    --zuul-api-url {{ item.zuul_api_url }}

doc/source/ansible-role.rst

@@ -0,0 +1 @@
+../../ansible/roles/logscraper/README.rst

doc/source/index.rst

@@ -1,13 +1,13 @@
-Openstack CI Log Processing
+OpenStack CI Log Processing
 ===========================
 
 This documentation covers the installation and maintenance of the
-Openstack CI Log Processing system.
+OpenStack CI Log Processing system.
 
 Security policy
 ---------------
 
-If you find or suspect a security issue with any Openstack CI Log
+If you find or suspect a security issue with any OpenStack CI Log
 Processing services, please inform the administrators via email at
 service-incident@lists.opendev.org.
 
@@ -16,11 +16,13 @@ Contents:
 .. sidebar:: HOWTOs
 
    * :doc:`logscraper`
+   * :doc:`ansible-role`
 
 .. toctree::
    :maxdepth: 2
 
    logscraper
+   ansible-role
 
 Indices and tables
 ==================

test-requirements.txt

@@ -3,3 +3,4 @@ flake8<3.8.5
 pep8<1.7.2
 testtools<2.5.1  # MIT
 stestr<3.3       # Apache-2.0
+yamllint<1.26.4  # GPLv3

tox.ini

@@ -16,7 +16,9 @@ commands =
 commands = flake8
 
 [testenv:linters]
-commands = flake8
+commands =
+  flake8
+  yamllint ansible/
 
 [testenv:venv]
 commands = {posargs}