From 313fc008191604e94c222ee052a90f22a072ce41 Mon Sep 17 00:00:00 2001 From: Swann Croiset Date: Wed, 3 Aug 2016 16:34:52 +0200 Subject: [PATCH] Check libvirt status on compute nodes The patche adds a new collectd plugin to test the availability of libvirt and configure AFD for all compute nodes. These AFD are part of nova global cluster. Change-Id: I0944f7da69caf32ed6ac9c908d4241bc8c396994 --- .../puppet/manifests/ceph_osd.pp | 6 ++ .../puppet/manifests/compute.pp | 7 ++ .../puppet/manifests/hiera_override.pp | 4 ++ .../templates/alarming.yaml.erb | 15 +++++ .../templates/gse_filters.yaml.erb | 6 ++ .../files/collectd/collectd_libvirt_check.py | 65 +++++++++++++++++++ .../manifests/collectd/libvirt_check.pp | 36 ++++++++++ ...a_collector_collectd_libvirt_check_spec.rb | 26 ++++++++ doc/user/source/metrics/checks.rst | 1 + 9 files changed, 166 insertions(+) create mode 100644 deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_libvirt_check.py create mode 100644 deployment_scripts/puppet/modules/lma_collector/manifests/collectd/libvirt_check.pp create mode 100644 deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_collectd_libvirt_check_spec.rb diff --git a/deployment_scripts/puppet/manifests/ceph_osd.pp b/deployment_scripts/puppet/manifests/ceph_osd.pp index 486718909..a17bf6123 100644 --- a/deployment_scripts/puppet/manifests/ceph_osd.pp +++ b/deployment_scripts/puppet/manifests/ceph_osd.pp @@ -30,4 +30,10 @@ if hiera('lma::collector::influxdb::server', false) { }else{ notice('ceph_osd_perf not configured to avoid messing of collectd python plugin configuration!') } + + # Due to limitation of Python collectd plugin implementation, the + # libvirt_check is configured here instead of compute.pp manifest. + if $node_profiles['compute'] { + class { 'lma_collector::collectd::libvirt_check': } + } } diff --git a/deployment_scripts/puppet/manifests/compute.pp b/deployment_scripts/puppet/manifests/compute.pp index 53a818a77..7493d4036 100644 --- a/deployment_scripts/puppet/manifests/compute.pp +++ b/deployment_scripts/puppet/manifests/compute.pp @@ -32,6 +32,13 @@ if hiera('lma::collector::influxdb::server', false) { } class { 'lma_collector::collectd::libvirt': } + + # Due to limitation of Python collectd plugin implementation, the + # libvirt_check is configured by ceph_osd manifests if it is a ceph-osd node. + $node_profiles = hiera_hash('lma::collector::node_profiles') + if ! $node_profiles['ceph_osd'] { + class { 'lma_collector::collectd::libvirt_check': } + } } if $ceilometer['enabled'] { diff --git a/deployment_scripts/puppet/manifests/hiera_override.pp b/deployment_scripts/puppet/manifests/hiera_override.pp index 1ec6e4a04..58f71d319 100644 --- a/deployment_scripts/puppet/manifests/hiera_override.pp +++ b/deployment_scripts/puppet/manifests/hiera_override.pp @@ -21,6 +21,8 @@ if ($plugin_data) { $network_metadata = hiera_hash('network_metadata') $is_controller_node = roles_include(['controller', 'primary-controller']) $is_base_os_node = roles_include('base-os') + $is_compute = roles_include('compute') + $is_ceph_osd = roles_include('ceph-osd') $has_controller = count(get_nodes_hash_by_roles($network_metadata, ['primary-controller'])) > 0 # The detached RabbitMQ plugin has no primary role in 8.0 $has_detached_rabbitmq = count(get_nodes_hash_by_roles($network_metadata, ['primary-standalone-rabbitmq', 'standalone-rabbitmq'])) > 0 @@ -169,6 +171,8 @@ lma::collector::node_profiles: rabbitmq: <%= @is_rabbitmq_node %> mysql: <%= @is_mysql_node %> base_os: <%= @is_base_os_node %> + compute: <%= @is_compute %> + ceph_osd: <%= @is_ceph_osd %> lma::collector::monitor::mysql_db: <%= @mysql_db %> lma::collector::monitor::mysql_username: <%= @mysql_username %> diff --git a/deployment_scripts/puppet/modules/fuel_lma_collector/templates/alarming.yaml.erb b/deployment_scripts/puppet/modules/fuel_lma_collector/templates/alarming.yaml.erb index dc38f6435..1525d71b7 100644 --- a/deployment_scripts/puppet/modules/fuel_lma_collector/templates/alarming.yaml.erb +++ b/deployment_scripts/puppet/modules/fuel_lma_collector/templates/alarming.yaml.erb @@ -800,6 +800,18 @@ lma_collector: window: 60 periods: 0 function: last + - name: 'libvirt-check' + description: "Libvirt cannot be checked" + severity: 'down' + enabled: 'true' + trigger: + rules: + - metric: libvirt_check + relational_operator: '==' + threshold: 0 + window: 60 + periods: 0 + function: last # Mapping between the Fuel roles and the AFD node filters node_cluster_roles: @@ -850,6 +862,7 @@ lma_collector: elasticsearch: ['primary-elasticsearch_kibana', 'elasticsearch_kibana'] pacemaker: ['primary-controller', 'controller'] haproxy-openstack: ['primary-controller', 'controller'] + libvirt: ['compute'] # Definition of the AFD node filters node_cluster_alarms: @@ -941,3 +954,5 @@ lma_collector: check: ['haproxy-check'] pacemaker: check: ['pacemaker-check'] + libvirt: + check: ['libvirt-check'] diff --git a/deployment_scripts/puppet/modules/fuel_lma_collector/templates/gse_filters.yaml.erb b/deployment_scripts/puppet/modules/fuel_lma_collector/templates/gse_filters.yaml.erb index 1d690fc3f..9b1839d85 100644 --- a/deployment_scripts/puppet/modules/fuel_lma_collector/templates/gse_filters.yaml.erb +++ b/deployment_scripts/puppet/modules/fuel_lma_collector/templates/gse_filters.yaml.erb @@ -388,6 +388,11 @@ lma_collector: group_by: member members: - check + libvirt: + policy: majority_of_members + group_by: member + members: + - check gse_cluster_node: input_message_types: @@ -523,6 +528,7 @@ lma_collector: - nova-novncproxy-websocket - controller - compute + - libvirt hints: - cinder - glance diff --git a/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_libvirt_check.py b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_libvirt_check.py new file mode 100644 index 000000000..46606090b --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/files/collectd/collectd_libvirt_check.py @@ -0,0 +1,65 @@ +#!/usr/bin/python +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import collectd +import libvirt + +import collectd_base as base + +NAME = 'libvirt' +URI = 'qemu:///system' + + +class LibvirtCheckPlugin(base.Base): + + def __init__(self, *args, **kwargs): + super(LibvirtCheckPlugin, self).__init__(*args, **kwargs) + self.plugin = NAME + self.uri = URI + + def config_callback(self, conf): + super(LibvirtCheckPlugin, self).config_callback(conf) + + for node in conf.children: + if node.key == 'Uri': + self.uri = node.values[0] + + def read_callback(self): + try: + cnx = libvirt.openReadOnly(self.uri) + cnx.numOfDefinedDomains() + self.dispatch_check_metric(self.OK) + except libvirt.libvirtError as e: + msg = 'Fail to query libvirt ({}): {}'.format(self.uri, e) + self.dispatch_check_metric(self.FAIL, msg) + + +plugin = LibvirtCheckPlugin(collectd) + + +def init_callback(): + plugin.restore_sigchld() + + +def config_callback(conf): + plugin.config_callback(conf) + + +def read_callback(): + plugin.read_callback() + +collectd.register_init(init_callback) +collectd.register_config(config_callback) +collectd.register_read(read_callback) diff --git a/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/libvirt_check.pp b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/libvirt_check.pp new file mode 100644 index 000000000..7895aefd9 --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/manifests/collectd/libvirt_check.pp @@ -0,0 +1,36 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# == Class lma_collector::collectd::libvirt_check +# +# Class that configures collectd to check availability of libvirt. +# +# === Parameters: +# +# [*connection*] +# (optional) The hypervisor connection URI. Default is 'qemu:///system'. +# +class lma_collector::collectd::libvirt_check ( + $connection = $lma_collector::params::libvirt_connection, +) inherits lma_collector::params { + + validate_string($connection) + + lma_collector::collectd::python { 'collectd_libvirt_check': + config => { + 'Uri' => "\"${connection}\"", + }, + } +} diff --git a/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_collectd_libvirt_check_spec.rb b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_collectd_libvirt_check_spec.rb new file mode 100644 index 000000000..d06ab5d4c --- /dev/null +++ b/deployment_scripts/puppet/modules/lma_collector/spec/classes/lma_collector_collectd_libvirt_check_spec.rb @@ -0,0 +1,26 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +require 'spec_helper' + +describe 'lma_collector::collectd::libvirt_check' do + let(:facts) do + {:kernel => 'Linux', :operatingsystem => 'Ubuntu', + :osfamily => 'Debian', :concat_basedir => '/foo'} + end + + describe 'with defaults' do + it { is_expected.to contain_lma_collector__collectd__python('collectd_libvirt_check') \ + .with_config({'Uri' => '"qemu:///system"'}) } + end +end diff --git a/doc/user/source/metrics/checks.rst b/doc/user/source/metrics/checks.rst index 95d82b61f..bbaa960e3 100644 --- a/doc/user/source/metrics/checks.rst +++ b/doc/user/source/metrics/checks.rst @@ -11,3 +11,4 @@ The value is ``1`` when successful and ``0`` if it fails. * ``ceph_osd_check``, for Ceph OSD. * ``elasticsearch_check``, for Elasticsearch. * ``influxdb_check``, for InfluxDB. +* ``libvirt_check``, for Libvirt.