Remove config management for subunit2sql workers
The openstack health service is being shutdown and retired. That services was the only service that relied on the subunit2sql workers. This means we can shutdown and retire the subunit2sql workers. This is one step of that process. Change-Id: Ibd02faaeba888dfcd1f512f4dd3a7d768497fc16
This commit is contained in:
Clark Boylan
parent
e2bbf3eb4d
commit
a5f06418b6
@ -75,16 +75,16 @@ bottleneck very quickly. This looks something like:
|
||||
zuul post-logs playbook
|
||||
|
|
||||
|
|
||||
gearman-client ---------------
|
||||
/ | \ |
|
||||
/ | \ |
|
||||
gearman gearman gearman subunit gearman
|
||||
worker1 worker2 worker3 worker01
|
||||
| | | |
|
||||
logstash logstash logstash |
|
||||
indexer1 indexer2 indexer3 |
|
||||
\ | / subunit2sql
|
||||
\ | / DB
|
||||
gearman-client
|
||||
/ | \
|
||||
/ | \
|
||||
gearman gearman gearman
|
||||
worker1 worker2 worker3
|
||||
| | |
|
||||
logstash logstash logstash
|
||||
indexer1 indexer2 indexer3
|
||||
\ | /
|
||||
\ | /
|
||||
elasticsearch
|
||||
cluster
|
||||
|
|
||||
@ -113,19 +113,6 @@ can be found at
|
||||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-client.py
|
||||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/log-gearman-worker.py
|
||||
|
||||
Subunit Gearman Worker
|
||||
----------------------
|
||||
|
||||
Using the same mechanism as the Log pushers there is an additional class of
|
||||
gearman worker that takes the subunit output from test runs and stores them in
|
||||
a subunit2SQL database. Right now this is only done with the subunit output
|
||||
from gate and periodic queue tempest runs.
|
||||
|
||||
If you are interested in technical details the source of this script can be
|
||||
found at:
|
||||
|
||||
* https://opendev.org/opendev/puppet-log_processor/src/branch/master/files/subunit-gearman-worker.py
|
||||
|
||||
Logstash
|
||||
--------
|
||||
|
||||
@ -215,27 +202,6 @@ be reached at http://logstash.openstack.org. See
|
||||
:ref:`query-logstash` for more info on using Kibana to perform
|
||||
queries.
|
||||
|
||||
subunit2SQL
|
||||
-----------
|
||||
subunit2SQL is a python project for taking subunit v2 streams and storing them
|
||||
in a SQL database. More information on the subunit protocol can be found here:
|
||||
https://github.com/testing-cabal/subunit/blob/master/README
|
||||
|
||||
subunit2sql provides a database schema, several utilities for interacting with
|
||||
the database, and a python library to build tooling on top of the database.
|
||||
More information about using subunit2sql can be found at:
|
||||
http://docs.openstack.org/developer/subunit2sql/
|
||||
|
||||
Our instance of the subunit2SQL database is running on a MySQL database and has
|
||||
been configured to be remotely accessible to allow for public querying. The
|
||||
public query access is provided with the following credentials::
|
||||
|
||||
username: query
|
||||
password: query
|
||||
hostname: logstash.openstack.org
|
||||
database name: subunit2sql
|
||||
database port: 3306
|
||||
|
||||
simpleproxy
|
||||
-----------
|
||||
Simpleproxy is a simple tcp proxy which allows forwarding tcp connections from
|
||||
|
||||
@ -33,7 +33,6 @@ cacti_hosts:
|
||||
- gitea08.opendev.org
|
||||
- grafana01.opendev.org
|
||||
- graphite02.opendev.org
|
||||
- health.openstack.org
|
||||
- jvb01.opendev.org
|
||||
- jvb02.opendev.org
|
||||
- kdc03.openstack.org
|
||||
@ -94,8 +93,6 @@ cacti_hosts:
|
||||
- status01.openstack.org
|
||||
- storyboard01.opendev.org
|
||||
- storyboard-dev01.opendev.org
|
||||
- subunit-worker01.openstack.org
|
||||
- subunit-worker02.openstack.org
|
||||
- translate.openstack.org
|
||||
- translate-dev.openstack.org
|
||||
- wiki.openstack.org
|
||||
|
||||
@ -3,5 +3,4 @@ iptables_extra_public_tcp_ports:
|
||||
- 3306
|
||||
iptables_extra_allowed_groups:
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'logstash-worker'}
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'subunit-worker'}
|
||||
- {'protocol': 'tcp', 'port': '4730', 'group': 'zuul-executor'}
|
||||
|
||||
@ -139,7 +139,6 @@ groups:
|
||||
- status*.open*.org
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
- storyboard[0-9]*.opendev.org
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
- translate-dev[0-9]*.open*.org
|
||||
- translate[0-9]*.open*.org
|
||||
puppet4:
|
||||
@ -152,7 +151,6 @@ groups:
|
||||
- status*.open*.org
|
||||
- storyboard[0-9]*.opendev.org
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
- translate[0-9]*.open*.org
|
||||
- translate-dev[0-9]*.open*.org
|
||||
refstack:
|
||||
@ -172,8 +170,6 @@ groups:
|
||||
- storyboard[0-9]*.opendev.org
|
||||
storyboard-dev:
|
||||
- storyboard-dev[0-9]*.opendev.org
|
||||
subunit-worker:
|
||||
- subunit-worker[0-9]*.open*.org
|
||||
translate-dev:
|
||||
- translate-dev[0-9]*.open*.org
|
||||
translate:
|
||||
|
||||
@ -61,18 +61,6 @@ node /^logstash-worker\d+\.open.*\.org$/ {
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: xenial
|
||||
node /^subunit-worker\d+\.open.*\.org$/ {
|
||||
$group = "subunit-worker"
|
||||
class { 'openstack_project::server': }
|
||||
class { 'openstack_project::subunit_worker':
|
||||
subunit2sql_db_host => hiera('subunit2sql_db_host', ''),
|
||||
subunit2sql_db_pass => hiera('subunit2sql_db_password', ''),
|
||||
mqtt_pass => hiera('mqtt_service_user_password'),
|
||||
mqtt_ca_cert_contents => hiera('mosquitto_tls_ca_file'),
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: xenial
|
||||
node /^elasticsearch\d+\.open.*\.org$/ {
|
||||
$group = "elasticsearch"
|
||||
|
||||
@ -1,49 +0,0 @@
|
||||
# Copyright 2013 Hewlett-Packard Development Company, L.P.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
# subunit2sql worker glue class.
|
||||
#
|
||||
class openstack_project::subunit_worker (
|
||||
$subunit2sql_db_host,
|
||||
$subunit2sql_db_pass,
|
||||
$mqtt_user = 'infra',
|
||||
$mqtt_pass = undef,
|
||||
$mqtt_ca_cert_contents = undef,
|
||||
) {
|
||||
|
||||
file { '/etc/subunit2sql/subunit-woker.yaml':
|
||||
ensure => file,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0555',
|
||||
content => template('openstack_project/logstash/jenkins-subunit-worker.yaml.erb'),
|
||||
}
|
||||
|
||||
file { '/etc/subunit2sql/mqtt-root-CA.pem.crt':
|
||||
ensure => present,
|
||||
content => $mqtt_ca_cert_contents,
|
||||
replace => true,
|
||||
owner => 'subunit',
|
||||
group => 'subunit',
|
||||
mode => '0555',
|
||||
}
|
||||
|
||||
include subunit2sql
|
||||
subunit2sql::worker { 'A':
|
||||
config_file => '/etc/subunit2sql/subunit-woker.yaml',
|
||||
db_host => $subunit2sql_db_host,
|
||||
db_pass => $subunit2sql_db_pass,
|
||||
}
|
||||
|
||||
}
|
||||
@ -1,9 +0,0 @@
|
||||
gearman-host: logstash.openstack.org
|
||||
gearman-port: 4730
|
||||
config: /etc/subunit2sql/subunit2sql.conf
|
||||
#mqtt-host: firehose.openstack.org
|
||||
#mqtt-port: 8883
|
||||
#mqtt-topic: gearman-subunit/<%= @hostname %>
|
||||
#mqtt-user: <%= @mqtt_user %>
|
||||
#mqtt-pass: <%= @mqtt_pass %>
|
||||
#mqtt-ca-certs: /etc/subunit2sql/mqtt-root-CA.pem.crt
|
||||
Loading…
Reference in New Issue
Block a user