openstack/ci-log-processing
Code Issues Proposed changes

Drop playbook for veryfing old workflow; rename playbook

Browse Source 
It is a leftover after removing loggearman workflow [1].

[1] https://review.opendev.org/c/openstack/ci-log-processing/+/911590

Change-Id: Icda73c1001e34df1cf70ab0b029e807dea0c8695
This commit is contained in:
Daniel Pawlik
2024-04-26 08:31:56 +02:00
parent 13f50c6bf2
commit 1a87138538
5 changed files with 36 additions and 205 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.zuul.yaml
View File

@@ -2,7 +2,7 @@
- job:
name: ci-log-processing-functional-test-centos-9-stream-sender
description: Test is validating Logscraper and logsender services
run: ansible/playbooks/check-services-sender.yml
run: ansible/playbooks/check-services.yml
nodeset:
nodes:
- name: centos-9-stream

2
README.rst
View File

@@ -76,7 +76,7 @@ With this solution, log workflow looks like:
+-----------------+
You can deploy your log workflow by using example Ansible playbook that
you can find in `ansible/playbooks/check-services-sender.yml` in this project.
you can find in `ansible/playbooks/check-services.yml` in this project.
Testing
-------

1
ansible/playbooks/check-services-sender.yml → ansible/playbooks/check-services.yml
View File

@@ -23,7 +23,6 @@
- name: Run check services
include_role:
name: check-services
tasks_from: download.yml
- name: Run backup and restore test
include_role:

157
ansible/roles/check-services/tasks/download.yml
View File

@@ -1,157 +0,0 @@
---
- name: Install packages
package:
name: podman
state: present
### BUILD CONTAINER IMAGES ###
- name: Build container images
block:
- name: Build logscraper container image - Zuul
shell: >
podman build -t quay.io/logscraper:dev -f Dockerfile
args:
chdir: "{{ zuul.projects['opendev.org/openstack/ci-log-processing'].src_dir }}"
when: zuul is defined
- name: Build logscraper container image - non Zuul
shell: >
podman build -t quay.io/logscraper:dev -f Dockerfile
args:
chdir: "{{ playbook_dir }}"
when: zuul is not defined
- name: Get logscraper image id
shell: |
podman images --noheading quay.io/logscraper:dev | awk '{print $3}'
register: _logscraper_image_id
- name: Print all images
shell: |
podman images
- name: Replace container images
set_fact:
container_images:
logscraper: "{{ _logscraper_image_id.stdout }}"
logsender: "{{ _logscraper_image_id.stdout }}"
### OPENSEARCH ####
- name: Setup Opensearch
shell: >
podman run -d --name opensearch \
--network host \
-e "discovery.type=single-node" \
quay.io/software-factory/opensearch:2.11.1
- name: Wait for Opensearch to be up
wait_for:
host: 127.0.0.1
port: 9200
delay: 10
timeout: 300
- name: Wait for Opensearch to be up
uri:
url: "https://127.0.0.1:9200"
user: "admin"
password: "admin"
force_basic_auth: true
method: GET
validate_certs: false
status_code: "200"
register: result
until: result.status == 200
retries: 30
delay: 10
- name: Reconfigure once again security plugin
shell: |
podman exec -it opensearch bash -c "JAVA_HOME=/usr/share/opensearch/jdk /usr/share/opensearch/securityadmin_demo.sh"
### Logscraper ###
- name: Setup logscraper service
include_role:
name: logscraper
### Logsender ###
- name: Setup logsender service
include_role:
name: logsender
# Flush handlers before running test
- name: Force all notified handlers to run now
meta: flush_handlers
### service validation ###
- name: Ensure that all services are available and running
shell: |
systemctl is-active -q {{ item }}
loop:
- logscraper-openstack
- logsender-openstack
register: _service_status
failed_when: _service_status.rc != 0
- name: Get Opensearch indices
block:
- name: Get indices
uri:
url: "https://127.0.0.1:9200/_cat/indices"
user: "admin"
password: "admin"
force_basic_auth: true
method: GET
validate_certs: false
status_code: "200"
return_content: true
register: _opensearch_indices
until: "'logstash-logscraper' in _opensearch_indices.content"
retries: 30
delay: 10
- name: Check if build_branch exists in index content
uri:
url: "https://127.0.0.1:9200/logstash-logscraper"
user: "admin"
password: "admin"
force_basic_auth: true
method: GET
validate_certs: false
status_code: "200"
return_content: true
register: _opensearch_index_content
until: "'build_branch' in _opensearch_index_content.content"
retries: 30
delay: 10
rescue:
- name: List all podman containers
shell: |
podman ps -a
- name: Get opensearch logs
shell: |
podman logs opensearch
- name: Get logscraper logs
shell: |
podman logs logscraper-openstack
- name: Get logsender logs
shell: |
podman logs logsender-openstack
- name: Get indices to fail the test
uri:
url: "https://127.0.0.1:9200/_cat/indices"
user: "admin"
password: "admin"
force_basic_auth: true
method: GET
validate_certs: false
status_code: "200"
return_content: true
register: _opensearch_indices
until: "'logstash-logscraper' in _opensearch_indices.content"
retries: 3
delay: 10

79
ansible/roles/check-services/tasks/main.yml
View File

@@ -34,6 +34,7 @@
set_fact:
container_images:
logscraper: "{{ _logscraper_image_id.stdout }}"
logsender: "{{ _logscraper_image_id.stdout }}"
### OPENSEARCH ####
- name: Setup Opensearch
@@ -68,57 +69,27 @@
shell: |
podman exec -it opensearch bash -c "JAVA_HOME=/usr/share/opensearch/jdk /usr/share/opensearch/securityadmin_demo.sh"
### LOGSTASH ###
- name: Start Logstash service
shell: >
podman run -dit --name logstash --net host quay.io/software-factory/logstash-oss-with-opensearch-output-plugin:7.16.2 -e '
input {
tcp {
port => {{ output_port }}
codec => json_lines {}
type => "zuul"
}
} output {
opensearch {
hosts => ["https://localhost:9200"]
index => "logstash-logscraper-%{+YYYY.MM.dd}"
user => "admin"
password => "admin"
ssl => true
ssl_certificate_verification => false
}
} filter {
grok {
match => ["message", "(?<timestamp>[-0-9]{10}\s+[0-9.:]{12})(?<ms>[0-9]{3}) (?<sep>\|)%{GREEDYDATA:message}"]
overwrite => [ "message" ]
}
if [message] =~ /^\s*$/ {
drop { }
}
date {
match => ["timestamp", "yyyy-MM-dd HH:mm:ss.SSS"]
timezone => "UTC"
}
}'
- name: Wait for Logstash to be up
wait_for:
host: 127.0.0.1
port: "{{ output_port }}"
delay: 10
timeout: 300
### Logscraper ###
- name: Setup logscraper service
include_role:
name: logscraper
### Logsender ###
- name: Setup logsender service
include_role:
name: logsender
# Flush handlers before running test
- name: Force all notified handlers to run now
meta: flush_handlers
### service validation ###
- name: Ensure that all services are available and running
shell: |
systemctl is-active -q {{ item }}
loop:
- logscraper-openstack
- logsender-openstack
register: _service_status
failed_when: _service_status.rc != 0
@@ -139,19 +110,37 @@
retries: 30
delay: 10
- name: Check if build_branch exists in index content
uri:
url: "https://127.0.0.1:9200/logstash-logscraper"
user: "admin"
password: "admin"
force_basic_auth: true
method: GET
validate_certs: false
status_code: "200"
return_content: true
register: _opensearch_index_content
until: "'build_branch' in _opensearch_index_content.content"
retries: 30
delay: 10
rescue:
- name: List all podman containers
shell: |
podman ps -a
- name: Get logstash logs
shell: |
podman logs logstash
- name: Get opensearch logs
shell: |
podman logs opensearch
- name: Get logscraper logs
shell: |
podman logs logscraper-openstack
- name: Get logsender logs
shell: |
podman logs logsender-openstack
- name: Get indices to fail the test
uri:
url: "https://127.0.0.1:9200/_cat/indices"