x
/
fuel-plugin-lma-collector
Code Issues Proposed changes

Allow deployment without InfluxDB and Elasticsearch 

This allows to support several deployment scenarii where backends are not
deployed initialy, for instance when using the 'virt' nodes to deploy
LMA backends.

The patch factorizes manifests by moving all the configuration data of
InfluxDB and Elasticsearch into hiera.

DocImpact

Fixes-bug: #1570386
Change-Id: I8688bbd10d88bc8ef68b5d31e9edd62a764dc23d
This commit is contained in:
Swann Croiset 2016-05-18 17:31:07 +02:00
parent 85399b5d23
commit debe1883d7
11 changed files with 178 additions and 226 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

127
deployment_scripts/puppet/manifests/base.pp
View File

@ -63,29 +63,6 @@ if $is_controller {
$additional_groups = []
}
$elasticsearch_mode = $lma_collector['elasticsearch_mode']
case $elasticsearch_mode {
'remote': {
$es_server = $lma_collector['elasticsearch_address']
}
'local': {
$vip_name = 'es_vip_mgmt'
if $network_metadata['vips'][$vip_name] {
$es_server = $network_metadata['vips'][$vip_name]['ipaddr']
}else{
# compatibility with elasticsearch-kibana version 0.8
$es_server = $es_nodes[0]['internal_address']
}
}
'disabled': {
# Nothing to do
}
default: {
fail("'${elasticsearch_mode}' mode not supported for Elasticsearch")
}
}
case $::osfamily {
'Debian': {
$heka_user = 'heka'
@ -307,8 +284,7 @@ if $is_controller or $is_rabbitmq or $is_mysql_server {
}
}
$influxdb_mode = $lma_collector['influxdb_mode']
if $elasticsearch_mode != 'disabled' {
if hiera('lma::collector::elasticsearch::server', false) {
class { 'lma_collector::logs::system':
require => Class['lma_collector'],
}
@ -321,7 +297,8 @@ if $elasticsearch_mode != 'disabled' {
}
class { 'lma_collector::elasticsearch':
server => $es_server,
server => hiera('lma::collector::elasticsearch::server'),
port => hiera('lma::collector::elasticsearch::rest_port'),
require => Class['lma_collector'],
}
@ -338,70 +315,41 @@ if $elasticsearch_mode != 'disabled' {
}
}
case $influxdb_mode {
'remote','local': {
if $influxdb_mode == 'remote' {
$influxdb_server = $lma_collector['influxdb_address']
$influxdb_database = $lma_collector['influxdb_database']
$influxdb_user = $lma_collector['influxdb_user']
$influxdb_password = $lma_collector['influxdb_password']
}
else {
# Note: we don't validate data inputs because another manifest
# is responsible to check their coherences.
$influxdb_vip_name = 'influxdb'
if $network_metadata['vips'][$influxdb_vip_name] {
$influxdb_server = $network_metadata['vips'][$influxdb_vip_name]['ipaddr']
} else {
$influxdb_server = $influxdb_nodes[0]['internal_address']
}
$influxdb_database = $influxdb_grafana['influxdb_dbname']
$influxdb_user = $influxdb_grafana['influxdb_username']
$influxdb_password = $influxdb_grafana['influxdb_userpass']
}
# TODO(all): this class is also applied by other role-specific manifests.
# This is sub-optimal and error prone. It needs to be fixed by having all
# collectd resources managed by a single manifest.
class { 'lma_collector::collectd::base':
processes => ['hekad', 'collectd'],
# Purge the default configuration shipped with the collectd package
purge => true,
require => Class['lma_collector'],
}
if $is_mysql_server {
$nova = hiera_hash('nova', {})
class { 'lma_collector::collectd::mysql':
username => 'nova',
password => $nova['db_password'],
require => Class['lma_collector::collectd::base'],
}
lma_collector::collectd::dbi_mysql_status { 'mysql_status':
username => 'nova',
dbname => 'nova',
password => $nova['db_password'],
require => Class['lma_collector::collectd::base'],
}
}
class { 'lma_collector::influxdb':
server => $influxdb_server,
database => $influxdb_database,
user => $influxdb_user,
password => $influxdb_password,
tag_fields => ['deployment_id', 'environment_label', 'tenant_id', 'user_id'],
require => Class['lma_collector'],
}
if hiera('lma::collector::influxdb::server', false) {
# TODO(all): this class is also applied by other role-specific manifests.
# This is sub-optimal and error prone. It needs to be fixed by having all
# collectd resources managed by a single manifest.
class { 'lma_collector::collectd::base':
processes => ['hekad', 'collectd'],
# Purge the default configuration shipped with the collectd package
purge => true,
require => Class['lma_collector'],
}
'disabled': {
# Nothing to do
if $is_mysql_server {
$nova = hiera_hash('nova', {})
class { 'lma_collector::collectd::mysql':
username => 'nova',
password => $nova['db_password'],
require => Class['lma_collector::collectd::base'],
}
lma_collector::collectd::dbi_mysql_status { 'mysql_status':
username => 'nova',
dbname => 'nova',
password => $nova['db_password'],
require => Class['lma_collector::collectd::base'],
}
}
default: {
fail("'${influxdb_mode}' mode not supported for InfluxDB")
class { 'lma_collector::influxdb':
server => hiera('lma::collector::influxdb::server'),
database => hiera('lma::collector::influxdb::database'),
user => hiera('lma::collector::influxdb::user'),
password => hiera('lma::collector::influxdb::password'),
tag_fields => ['deployment_id', 'environment_label', 'tenant_id', 'user_id'],
require => Class['lma_collector'],
}
}
@ -419,8 +367,7 @@ if $is_rabbitmq {
password => $rabbit['password'],
}
if ($influxdb_mode != 'disabled') {
if hiera('lma::collector::influxdb::server', false) {
class { 'lma_collector::notifications::metrics': }
# If the node has the controller role, the collectd Python plugins will be

4
deployment_scripts/puppet/manifests/ceph_osd.pp
View File

@ -14,9 +14,7 @@
notice('fuel-plugin-lma-collector: ceph_osd.pp')
$lma_collector_hash = hiera_hash('lma_collector')
if $lma_collector_hash['influxdb_mode'] != 'disabled' {
if hiera('lma::collector::influxdb::server', false) {
# Only install this python collectd plugin if ceph-osd is not deployed on a
# controller node. This is due to a limitation of the python plugin puppet
# module which can be run only by one manifest otherwise the collectd configuration is

58
deployment_scripts/puppet/manifests/check_environment_configuration.pp
View File

@ -1,58 +0,0 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
notice('fuel-plugin-lma-collector: check_environment_configuration.pp')
# This manifest is only executed on the primrary-controller to verify that the
# plugin's configuration matches with the environment.
$lma_collector = hiera_hash('lma_collector')
$network_metadata = hiera('network_metadata')
$elasticsearch_mode = $lma_collector['elasticsearch_mode']
if $elasticsearch_mode == 'local' {
# Check that the Elasticsearch-Kibana plugin is enabled for that environment
# and that the node names match
$elasticsearch_kibana = hiera_hash('elasticsearch_kibana', false)
if ! $elasticsearch_kibana {
fail('Could not get the Elasticsearch parameters. The Elasticsearch-Kibana plugin is probably not installed.')
}
elsif ! $elasticsearch_kibana['metadata']['enabled'] {
fail('Could not get the Elasticsearch parameters. The Elasticsearch-Kibana plugin is probably not enabled for this environment.')
}
# Check that the Elasticsearch-Kibana node exists in the environment
$es_nodes = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana', 'primary-elasticsearch_kibana'])
if size(keys($es_nodes)) < 1 {
fail("Could not find node with role 'elasticsearch_kibana' in the environment")
}
}
$influxdb_mode = $lma_collector['influxdb_mode']
if $influxdb_mode == 'local' {
# Check that the InfluxDB-Grafana plugin is enabled for that environment
# and that the node names match
$influxdb_grafana = hiera_hash('influxdb_grafana', false)
if ! $influxdb_grafana {
fail('Could not get the InfluxDB parameters. The InfluxDB-Grafana plugin is probably not installed.')
}
elsif ! $influxdb_grafana['metadata']['enabled'] {
fail('Could not get the InfluxDB parameters. The InfluxDB-Grafana plugin is probably not enabled for this environment.')
}
# Check that the InfluxDB-Grafana node exists in the environment
$influxdb_nodes = get_nodes_hash_by_roles($network_metadata, ['influxdb_grafana', 'primary-influxdb_grafana'])
if size(keys($influxdb_nodes)) < 1 {
fail("Could not find node with role 'influxdb_grafana' in the environment")
}
}

5
deployment_scripts/puppet/manifests/cinder.pp
View File

@ -15,7 +15,6 @@
notice('fuel-plugin-lma-collector: cinder.pp')
$ceilometer = hiera_hash('ceilometer', {})
$lma_collector = hiera_hash('lma_collector')
$is_controller = roles_include(['controller', 'primary-controller'])
$is_rabbitmq = roles_include(['standalone-rabbitmq', 'primary-standalone-rabbitmq'])
$is_mysql_server = roles_include(['standalone-database', 'primary-standalone-database'])
@ -31,13 +30,13 @@ if $is_controller or $is_rabbitmq or $is_mysql_server {
}
}
if $lma_collector['influxdb_mode'] != 'disabled' {
if hiera('lma::collector::influxdb::server', false) {
class { 'lma_collector::logs::counter':
hostname => $::hostname,
}
}
if $lma_collector['elasticsearch_mode'] != 'disabled' {
if hiera('lma::collector::elasticsearch::server', false) {
lma_collector::logs::openstack { 'cinder': }
}

7
deployment_scripts/puppet/manifests/compute.pp
View File

@ -14,16 +14,15 @@
notice('fuel-plugin-lma-collector: compute.pp')
$ceilometer = hiera_hash('ceilometer', {})
$lma_collector = hiera_hash('lma_collector')
$ceilometer = hiera_hash('ceilometer', {})
if $lma_collector['elasticsearch_mode'] != 'disabled' {
if hiera('lma::collector::elasticsearch::server', false) {
lma_collector::logs::openstack { 'nova': }
lma_collector::logs::openstack { 'neutron': }
class { 'lma_collector::logs::libvirt': }
}
if $lma_collector['influxdb_mode'] != 'disabled' {
if hiera('lma::collector::influxdb::server', false) {
class { 'lma_collector::logs::counter':
hostname => $::hostname,
}

33
deployment_scripts/puppet/manifests/controller.pp
View File

@ -261,14 +261,12 @@ if ! $storage_options['objects_ceph'] {
}
}
# Logs
if $lma_collector['elasticsearch_mode'] != 'disabled' {
if hiera('lma::collector::elasticsearch::server', false) {
class { 'lma_collector::logs::pacemaker': }
}
# Metrics
$influxdb_mode = $lma_collector['influxdb_mode']
if $influxdb_mode != 'disabled' {
if hiera('lma::collector::influxdb::server', false) {
$nova = hiera_hash('nova', {})
$neutron = hiera_hash('quantum_settings', {})
@ -414,29 +412,10 @@ if $influxdb_mode != 'disabled' {
class { 'lma_collector::afd::workers': }
# VIP checks
if $influxdb_mode == 'remote' {
$use_local_influxdb = false
$use_remote_influxdb = true
$influxdb_server = $lma_collector['influxdb_address']
} elsif $influxdb_mode == 'local'{
$use_local_influxdb = true
$use_remote_influxdb = false
$influxdb_vip_name = 'influxdb'
if $network_metadata['vips'][$influxdb_vip_name] {
$influxdb_server = $network_metadata['vips'][$influxdb_vip_name]['ipaddr']
} else {
# compatibility with the InfluxDB-Grafana plugin 0.8
$influxdb_grafana = hiera_hash('influxdb_grafana', {})
$influxdb_nodes = get_nodes_hash_by_roles($network_metadata, ['influxdb_grafana'])
$influxdb_server = $influxdb_nodes[0]['internal_address']
}
} else {
$use_local_influxdb = false
$use_remote_influxdb = false
}
if $use_local_influxdb or $use_remote_influxdb {
$influxdb_url = "http://${influxdb_server}:8086/ping"
if hiera('lma::collector::influxdb::server', false) {
$influxdb_server = hiera('lma::collector::influxdb::server')
$influxdb_port = hiera('lma::collector::influxdb::port')
$influxdb_url = "http://${influxdb_server}:${influxdb_port}/ping"
}
$vip_urls = {

110
deployment_scripts/puppet/manifests/hiera_override.pp
View File

@ -14,9 +14,107 @@
notice('fuel-plugin-lma-collector: hiera_override.pp')
prepare_network_config(hiera('network_scheme', {}))
$plugin_data = hiera_hash('lma_collector', undef)
if ($plugin_data) {
$network_metadata = hiera_hash('network_metadata')
# Elasticsearch
$elasticsearch_mode = $plugin_data['elasticsearch_mode']
$es_nodes = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana', 'primary-elasticsearch_kibana'])
$es_nodes_count = count($es_nodes)
case $elasticsearch_mode {
'remote': {
$es_server = $plugin_data['elasticsearch_address']
$monitor_elasticsearch = false
}
'local': {
$es_vip_name = 'es_vip_mgmt'
if $network_metadata['vips'][$es_vip_name] {
$es_server = $network_metadata['vips'][$es_vip_name]['ipaddr']
$monitor_elasticsearch = true
} elsif $es_nodes_count > 0 {
$es_server = $es_nodes[0]['internal_address']
$monitor_elasticsearch = true
} else {
$es_server = undef
$monitor_elasticsearch = false
}
}
default: {
fail("'${elasticsearch_mode}' mode not supported for Elasticsearch")
}
}
if $es_nodes_count > 0 or $es_server {
$es_is_deployed = true
} else {
$es_is_deployed = false
}
# InfluxDB
$influxdb_mode = $plugin_data['influxdb_mode']
$influxdb_nodes = get_nodes_hash_by_roles($network_metadata, ['influxdb_grafana', 'primary-influxdb_grafana'])
$influxdb_nodes_count = count($influxdb_nodes)
$influxdb_grafana = hiera_hash('influxdb_grafana', {})
case $influxdb_mode {
'remote': {
$influxdb_server = $plugin_data['influxdb_address']
$influxdb_database = $lma_collector['influxdb_database']
$influxdb_user = $lma_collector['influxdb_user']
$influxdb_password = $lma_collector['influxdb_password']
$monitor_influxdb = false
}
'local': {
$influxdb_vip_name = 'influxdb'
if $network_metadata['vips'][$influxdb_vip_name] {
$influxdb_server = $network_metadata['vips'][$influxdb_vip_name]['ipaddr']
$monitor_influxdb = true
} elsif $influxdb_nodes_count > 0 {
$influxdb_server = $influxdb_nodes[0]['internal_address']
$monitor_influxdb = true
} else {
$monitor_influxdb = false
$influxdb_server = undef
}
$influxdb_database = $influxdb_grafana['influxdb_dbname']
$influxdb_user = $influxdb_grafana['influxdb_username']
$influxdb_password = $influxdb_grafana['influxdb_userpass']
}
default: {
fail("'${influxdb_mode}' mode not supported for InfluxDB")
}
}
if $influxdb_nodes_count > 0 or $influxdb_server {
$influxdb_is_deployed = true
} else {
$influxdb_is_deployed = false
}
$hiera_file = '/etc/hiera/plugins/lma_collector.yaml'
$calculated_content = inline_template('
---
<% if @es_is_deployed -%>
lma::collector::elasticsearch::server: <%= @es_server %>
lma::collector::elasticsearch::rest_port: 9200
<% end -%>
<% if @influxdb_is_deployed -%>
lma::collector::influxdb::server: <%= @influxdb_server %>
lma::collector::influxdb::port: 8086
lma::collector::influxdb::database: <%= @influxdb_database %>
lma::collector::influxdb::user: <%= @influxdb_user %>
lma::collector::influxdb::password: <%= @influxdb_password %>
<% end -%>
')
file { $hiera_file:
ensure => file,
content => $calculated_content,
}
$storage_options = hiera_hash('storage', {})
$tls_enabled = hiera('public_ssl', false)
$ceilometer = hiera_hash('ceilometer', {})
@ -31,18 +129,6 @@ if ($plugin_data) {
$detach_database = hiera('detach-database', {})
$detach_database_enabled = $detach_database['metadata'] and $detach_database['metadata']['enabled']
$elasticsearch_mode = $plugin_data['elasticsearch_mode']
$monitor_elasticsearch = $elasticsearch_mode ? {
'local' => true,
default => false,
}
$influxdb_mode = $plugin_data['influxdb_mode']
$monitor_influxdb = $influxdb_mode ? {
'local' => true,
default => false,
}
fuel_lma_collector::hiera_data { 'gse_filters':
content => template('fuel_lma_collector/gse_filters.yaml.erb')
}

11
deployment_scripts/puppet/manifests/lma_backends.pp
View File

@ -17,10 +17,9 @@ notice('fuel-plugin-lma-collector: lma_backends.pp')
prepare_network_config(hiera('network_scheme', {}))
$mgmt_address = get_network_role_property('management', 'ipaddr')
$lma_collector_hash = hiera_hash('lma_collector')
$influxdb_grafana = hiera('influxdb_grafana')
if $lma_collector_hash['influxdb_mode'] != 'disabled' {
if hiera('lma::collector::influxdb::server', false) {
$network_metadata = hiera('network_metadata')
$is_elasticsearch_node = roles_include(['elasticsearch_kibana', 'primary-elasticsearch_kibana'])
@ -47,15 +46,15 @@ if $lma_collector_hash['influxdb_mode'] != 'disabled' {
class { 'lma_collector::collectd::influxdb':
username => 'root',
password => $influxdb_grafana['influxdb_rootpass'],
address => hiera('lma::influxdb::listen_address', $mgmt_address),
port => hiera('lma::influxdb::influxdb_port', 8086)
address => hiera('lma::collector::influxdb::listen_address', $mgmt_address),
port => hiera('lma::collector::influxdb::influxdb_port', 8086)
}
}
if $is_elasticsearch_node {
class { 'lma_collector::collectd::elasticsearch':
address => hiera('lma::elasticsearch::vip', $mgmt_address),
port => hiera('lma::elasticsearch::rest_port', 9200)
address => hiera('lma::collector::elasticsearch::vip', $mgmt_address),
port => hiera('lma::collector::elasticsearch::rest_port', 9200)
}
}

2
deployment_scripts/puppet/modules/lma_collector/manifests/init.pp
View File

@ -34,6 +34,8 @@ class lma_collector (
$tags = {},
) {
include lma_collector::params
include lma_collector::service::log
include lma_collector::service::metric
validate_hash($tags)

33
deployment_tasks.yaml
View File

@ -10,17 +10,8 @@
puppet_manifest: puppet/manifests/hiera_override.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
- id: lma-check-configuration
type: puppet
version: 2.0.0
requires: [post_deployment_start]
required_for: [post_deployment_end]
role: [primary-controller]
parameters:
puppet_manifest: puppet/manifests/check_environment_configuration.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 300
reexecute_on:
- deploy_changes
- id: lma-configure-apt
type: puppet
@ -35,6 +26,8 @@
puppet_manifest: puppet/manifests/configure_apt.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-base
type: puppet
@ -46,6 +39,8 @@
puppet_manifest: puppet/manifests/base.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
# All tasks lma-main-* must be executed before lma-aggregator. So we don't
# need to add a requirement to post_deployment_end because it is implied
@ -60,6 +55,8 @@
puppet_manifest: puppet/manifests/controller.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-main-compute
type: puppet
@ -71,6 +68,8 @@
puppet_manifest: puppet/manifests/compute.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-main-cinder
type: puppet
@ -82,6 +81,8 @@
puppet_manifest: puppet/manifests/cinder.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-main-ceph-osd
type: puppet
@ -93,6 +94,8 @@
puppet_manifest: puppet/manifests/ceph_osd.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-aggregator
type: puppet
@ -104,6 +107,8 @@
puppet_manifest: puppet/manifests/aggregator.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-configure-afd-filters
type: puppet
@ -122,6 +127,8 @@
puppet_manifest: puppet/manifests/configure_afd_filters.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
- id: lma-backends
type: puppet
@ -137,6 +144,8 @@
puppet_manifest: puppet/manifests/lma_backends.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes
# This task must be executed at the very end of the deployment.
- id: lma-cleanup-apt-config
@ -151,4 +160,6 @@
puppet_manifest: puppet/manifests/cleanup_apt_config.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 600
reexecute_on:
- deploy_changes

14
environment_config.yaml
View File

@ -17,16 +17,10 @@ attributes:
value: "local"
label: "Events analytics (logs and notifications)"
values:
- data: "disabled"
label: "Disabled"
- data: "local"
label: "Local node"
label: "Local node (if deployed)"
- data: "remote"
label: "Remote server"
restrictions: &all_disabled_msg
- condition: "settings:lma_collector.elasticsearch_mode.value == 'disabled' and settings:lma_collector.influxdb_mode.value == 'disabled'"
action: "none"
message: "Please enable Events or Metrics analytics or both"
elasticsearch_address:
value: ''
@ -47,13 +41,10 @@ attributes:
value: "local"
label: "Metrics analytics"
values:
- data: "disabled"
label: "Disabled"
- data: "local"
label: "Local node"
label: "Local node (if deployed)"
- data: "remote"
label: "Remote server"
restrictions: *all_disabled_msg
influxdb_address:
value: ''
@ -109,7 +100,6 @@ attributes:
label: "Alerts sent by email (requires a SMTP server)"
- data: "remote"
label: "Alerts sent to a remote Nagios server"
restrictions: *all_disabled_msg
alerting_send_to:
value: ''