diff --git a/.zuul.yaml b/.zuul.yaml index 3b2eb1b..e12b9f2 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -4,6 +4,7 @@ - build-tox-docs check: &logcheck jobs: + - openstack-tox-linters - openstack-tox-pep8 - openstack-tox-py38 gate: *logcheck diff --git a/README.rst b/README.rst index 5019f3d..27d16c6 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -Openstack CI log processing +OpenStack CI log processing =========================== The goal of this repository is to provide and check @@ -16,7 +16,7 @@ the log processing system. Testing ------- -The part of Openstack CI log processing runs a complete testing and +The part of OpenStack CI log processing runs a complete testing and continuous-integration environment, powered by `Zuul `__. diff --git a/ansible/playbooks/roles b/ansible/playbooks/roles new file mode 120000 index 0000000..d8c4472 --- /dev/null +++ b/ansible/playbooks/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/ansible/playbooks/service-logscraper.yml b/ansible/playbooks/service-logscraper.yml new file mode 100644 index 0000000..d8d7917 --- /dev/null +++ b/ansible/playbooks/service-logscraper.yml @@ -0,0 +1,6 @@ +--- +- name: Configure Logscraper tool + hosts: logscraper01.openstack.org + become: true + roles: + - logscraper diff --git a/ansible/roles/logscraper/README.rst b/ansible/roles/logscraper/README.rst new file mode 100644 index 0000000..23009fd --- /dev/null +++ b/ansible/roles/logscraper/README.rst @@ -0,0 +1,80 @@ +Logscraper +========== + +The goal of this role is to setup and configure service related +to logscraper script which is responsible to to push recent +zuul builds into log gearman processor. + +Requirements +------------ + +None + +Role Variables +-------------- + +The role is automatically deploying service related to the +log scrape service. Depends of what is set to the `tenant_builds` var, +it can start multiple services on same host with different name, +for example: + +.. code-block:: yaml + + vars: + tenant_builds: + - tenant: openstack + gearman_port: 4731 + gearman_server: logstash.openstack.org + zuul_api_url: https://zuul.opendev.org/api/tenant/openstack + insecure: false + +will deploy service with name: `logscraper@openstack.service`. +It is because on one service we are able to deploy multiple instances +of logscraper and each of them will be responsible for checking +and pushing logs for own tenant. + +Dependencies +------------ + +None + +Example Playbook +---------------- + +Playbook responsible for deploying service can look like: + +Below is a playbook example, responsible for deploying two logscraper +services, where one will responsible to get logs from `openstack` tenant +and second one for getting logs from `sometenant` tenant. + +.. code-block:: yaml + + - name: Configure Logscraper tool + hosts: localhost + become: true + vars: + tenant_builds: + - tenant: openstack + gearman_port: 4731 + gearman_server: logstash.openstack.org + zuul_api_url: https://zuul.opendev.org/api/tenant/openstack + insecure: False + - tenant: sometenant + gearman_port: 4731 + gearman_server: someproject.org + zuul_api_url: https://zuul.opendev.org/api/tenant/sometenant + insecure: True + roles: + - logscraper + +License +------- + +Apache + +Author Information +------------------ + +Author: OpenStack Contributors +Author email: openstack-discuss@lists.openstack.org +Home page: http://docs.openstack.org/infra/ci-log-processing diff --git a/ansible/roles/logscraper/defaults/main.yml b/ansible/roles/logscraper/defaults/main.yml new file mode 100644 index 0000000..4236271 --- /dev/null +++ b/ansible/roles/logscraper/defaults/main.yml @@ -0,0 +1,22 @@ +--- +logscraper_user: logscraper +logscraper_group: logscraper +logscraper_dir: /etc/logscraper + +container_images: + # FIXME: Create new project on Docker hub that will contain that image + logscraper: quay.rdoproject.org/software-factory/logscraper:latest + +# Example: +# tenant_builds: +# - tenant: openstack +# gearman_port: 4731 +# gearman_server: logstash.openstack.org +# zuul_api_url: https://zuul.opendev.org/api/tenant/openstack +# insecure: False +# - tenant: sometenant +# gearman_port: 4731 +# gearman_server: logstash.openstack.org +# zuul_api_url: https://zuul.opendev.org/api/tenant/sometenant +# insecure: True +tenant_builds: [] diff --git a/ansible/roles/logscraper/meta/main.yml b/ansible/roles/logscraper/meta/main.yml new file mode 100644 index 0000000..8df689b --- /dev/null +++ b/ansible/roles/logscraper/meta/main.yml @@ -0,0 +1,13 @@ +--- +galaxy_info: + author: Openstack Contributors + description: Openstack Logscraper tool + company: Openstack + license: Apache + min_ansible_version: 2.9 + platforms: + - name: Centos + versions: + - 8 + galaxy_tags: [] +dependencies: [] diff --git a/ansible/roles/logscraper/tasks/main.yml b/ansible/roles/logscraper/tasks/main.yml new file mode 100644 index 0000000..649593b --- /dev/null +++ b/ansible/roles/logscraper/tasks/main.yml @@ -0,0 +1,33 @@ +--- +- name: Create dedicated group + group: + name: "{{ logscraper_group }}" + state: present + +- name: Create dedicated user + user: + name: "{{ logscraper_user }}" + state: present + comment: "Dedicated user for logscraper" + group: "{{ logscraper_group }}" + shell: "/sbin/nologin" + create_home: false + +- name: Create dedicated directory + file: + path: "{{ logscraper_dir }}" + state: directory + owner: "{{ logscraper_user }}" + group: "{{ logscraper_group }}" + +- name: Ensure container software is installed + package: + name: podman + state: present + +- name: Pull image + shell: "podman pull {{ container_images['logscraper'] }}" + +- name: Configure logscraper service + include_tasks: service.yml + loop: "{{ tenant_builds }}" diff --git a/ansible/roles/logscraper/tasks/service.yml b/ansible/roles/logscraper/tasks/service.yml new file mode 100644 index 0000000..75a186b --- /dev/null +++ b/ansible/roles/logscraper/tasks/service.yml @@ -0,0 +1,29 @@ +--- +- name: Generate logscraper script + template: + src: logscraper.sh.j2 + dest: "/usr/local/bin/logscraper-{{ item.tenant }}" + mode: '0755' + register: _start_script + +- name: Generate systemd unit + template: + src: logscraper.service.j2 + dest: "/etc/systemd/system/logscraper-{{ item.tenant }}.service" + owner: root + group: root + +- name: Enable and restart service + service: + name: logscraper-{{ item.tenant }} + state: restarted + daemon-reload: true + enabled: true + when: _start_script.changed + +- name: Ensure that service is running + service: + name: logscraper-{{ item.tenant }} + state: started + daemon-reload: true + enabled: true diff --git a/ansible/roles/logscraper/templates/logscraper.service.j2 b/ansible/roles/logscraper/templates/logscraper.service.j2 new file mode 100644 index 0000000..ac872d1 --- /dev/null +++ b/ansible/roles/logscraper/templates/logscraper.service.j2 @@ -0,0 +1,16 @@ +[Unit] +Description=logscraper service for {{ item.tenant }} +After=syslog.target network.target +StartLimitInterval=20 +StartLimitBurst=5 + +[Service] +Type=simple +SyslogIdentifier=logscraper-{{ item.tenant }} +Restart=always +RestartSec=3s +ExecStop=-/usr/bin/podman stop -t 10 logscraper-{{ item.tenant }} +ExecStart=/usr/local/bin/logscraper-{{ item.tenant }} + +[Install] +WantedBy=multi-user.target diff --git a/ansible/roles/logscraper/templates/logscraper.sh.j2 b/ansible/roles/logscraper/templates/logscraper.sh.j2 new file mode 100644 index 0000000..07184cc --- /dev/null +++ b/ansible/roles/logscraper/templates/logscraper.sh.j2 @@ -0,0 +1,13 @@ +#!/bin/bash + +/usr/bin/podman run \ + --network host \ + --rm \ + --name logscraper-{{ item.tenant }} \ + --volume {{ logscraper_dir }}:{{ logscraper_dir }}:z \ + {{ container_images['logscraper'] }} \ + --gearman-port {{ item.gearman_port }} \ + --gearman-server {{ item.gearman_server }} \ + --checkpoint-file {{ item.checkpoint_file | default(logscraper_dir + '/checkpoint') }} \ + --follow \ + --zuul-api-url {{ item.zuul_api_url }} diff --git a/doc/source/ansible-role.rst b/doc/source/ansible-role.rst new file mode 120000 index 0000000..ded681e --- /dev/null +++ b/doc/source/ansible-role.rst @@ -0,0 +1 @@ +../../ansible/roles/logscraper/README.rst \ No newline at end of file diff --git a/doc/source/index.rst b/doc/source/index.rst index a36ab83..4cb7f02 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -1,13 +1,13 @@ -Openstack CI Log Processing +OpenStack CI Log Processing =========================== This documentation covers the installation and maintenance of the -Openstack CI Log Processing system. +OpenStack CI Log Processing system. Security policy --------------- -If you find or suspect a security issue with any Openstack CI Log +If you find or suspect a security issue with any OpenStack CI Log Processing services, please inform the administrators via email at service-incident@lists.opendev.org. @@ -16,11 +16,13 @@ Contents: .. sidebar:: HOWTOs * :doc:`logscraper` + * :doc:`ansible-role` .. toctree:: :maxdepth: 2 logscraper + ansible-role Indices and tables ================== diff --git a/test-requirements.txt b/test-requirements.txt index 19c2fb9..03898b9 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -3,3 +3,4 @@ flake8<3.8.5 pep8<1.7.2 testtools<2.5.1 # MIT stestr<3.3 # Apache-2.0 +yamllint<1.26.4 # GPLv3 diff --git a/tox.ini b/tox.ini index 789b1af..4064dd9 100644 --- a/tox.ini +++ b/tox.ini @@ -16,7 +16,9 @@ commands = commands = flake8 [testenv:linters] -commands = flake8 +commands = + flake8 + yamllint ansible/ [testenv:venv] commands = {posargs}