x
/
fuel-plugin-lma-collector
Code Issues Proposed changes

Add a new datasource to Hiera 

This change modifies the Hiera configuration to add a datasource for
the LMA collector (located at /etc/hiera/override/lma_collector.yaml').

It also modifies the Puppet manifests to use hiera_hash() instead of
hiera() when looking up hashes: this ensures that hashes are merged with
the 'deeper-merge' strategy. Otherwise the 'lma_collector' key in
/etc/astute.yaml would be hidden.

Change-Id: Ia9fd3505aae05f63dfae99fc4bac11e831b63133
This commit is contained in:
Simon Pasquier 2015-09-18 15:02:18 +02:00 committed by Guillaume Thouvenin
parent ddd62cd1d5
commit 5c7ddad02a
11 changed files with 310 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deployment_scripts/puppet/manifests/aggregator.pp
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
$lma_collector = hiera('lma_collector')
$lma_collector = hiera_hash('lma_collector')
$roles = node_roles(hiera('nodes'), hiera('uid'))
$is_controller = member($roles, 'controller') or member($roles, 'primary-controller')

6
deployment_scripts/puppet/manifests/base.pp
View File

@ -14,17 +14,17 @@
#
# TODO(spasquier): fail if Neutron isn't used
$lma_collector = hiera('lma_collector')
$lma_collector = hiera_hash('lma_collector')
$roles = node_roles(hiera('nodes'), hiera('uid'))
$is_controller = member($roles, 'controller') or member($roles, 'primary-controller')
$is_base_os = member($roles, 'base-os')
$current_node_name = hiera('user_node_name')
$current_roles = hiera('roles')
$elasticsearch_kibana = hiera('elasticsearch_kibana', {})
$elasticsearch_kibana = hiera_hash('elasticsearch_kibana', {})
$es_nodes = filter_nodes(hiera('nodes'), 'role', 'elasticsearch_kibana')
$influxdb_grafana = hiera('influxdb_grafana', {})
$influxdb_grafana = hiera_hash('influxdb_grafana', {})
$influxdb_nodes = filter_nodes(hiera('nodes'), 'role', 'influxdb_grafana')
$tags = {

2
deployment_scripts/puppet/manifests/ceph_osd.pp
View File

@ -12,7 +12,7 @@
# License for the specific language governing permissions and limitations
# under the License.
#
$lma_collector_hash = hiera('lma_collector')
$lma_collector_hash = hiera_hash('lma_collector')
if $lma_collector_hash['influxdb_mode'] != 'disabled' {
$nodes_hash = hiera('nodes', {})

8
deployment_scripts/puppet/manifests/check_environment_configuration.pp
View File

@ -15,13 +15,13 @@
# This manifest is only executed on the primrary-controller to verify that the
# plugin's configuration matches with the environment.
$lma_collector = hiera('lma_collector')
$lma_collector = hiera_hash('lma_collector')
$elasticsearch_mode = $lma_collector['elasticsearch_mode']
if $elasticsearch_mode == 'local' {
# Check that the Elasticsearch-Kibana plugin is enabled for that environment
# and that the node names match
$elasticsearch_kibana = hiera('elasticsearch_kibana', false)
$elasticsearch_kibana = hiera_hash('elasticsearch_kibana', false)
if ! $elasticsearch_kibana {
fail('Could not get the Elasticsearch parameters. The Elasticsearch-Kibana plugin is probably not installed.')
}
@ -40,7 +40,7 @@ $influxdb_mode = $lma_collector['influxdb_mode']
if $influxdb_mode == 'local' {
# Check that the InfluxDB-Grafana plugin is enabled for that environment
# and that the node names match
$influxdb_grafana = hiera('influxdb_grafana', false)
$influxdb_grafana = hiera_hash('influxdb_grafana', false)
if ! $influxdb_grafana {
fail('Could not get the InfluxDB parameters. The InfluxDB-Grafana plugin is probably not installed.')
}
@ -58,7 +58,7 @@ $alerting_mode = $lma_collector['alerting_mode']
if $alerting_mode == 'local' {
# Check that the LMA-Infrastructure-Alerting plugin is enabled for that environment
# and that the node names match
$infra_alerting = hiera('lma_infrastructure_alerting', false)
$infra_alerting = hiera_hash('lma_infrastructure_alerting', false)
if ! $infra_alerting {
fail('Could not get the LMA Infrastructure Alerting parameters. The LMA-Infrastructure-Alerting plugin is probably not installed.')
}

4
deployment_scripts/puppet/manifests/cinder.pp
View File

@ -14,8 +14,8 @@
#
include lma_collector::params
$ceilometer = hiera('ceilometer')
$lma_collector = hiera('lma_collector')
$ceilometer = hiera_hash('ceilometer', {})
$lma_collector = hiera_hash('lma_collector')
if $lma_collector['elasticsearch_mode'] != 'disabled' {

4
deployment_scripts/puppet/manifests/compute.pp
View File

@ -14,8 +14,8 @@
#
include lma_collector::params
$ceilometer = hiera('ceilometer')
$lma_collector = hiera('lma_collector')
$ceilometer = hiera_hash('ceilometer', {})
$lma_collector = hiera_hash('lma_collector')
if $lma_collector['elasticsearch_mode'] != 'disabled' {

16
deployment_scripts/puppet/manifests/controller.pp
View File

@ -14,11 +14,11 @@
#
include lma_collector::params
$ceilometer = hiera('ceilometer')
$lma_collector = hiera('lma_collector')
$rabbit = hiera('rabbit')
$ceilometer = hiera_hash('ceilometer', {})
$lma_collector = hiera_hash('lma_collector')
$rabbit = hiera_hash('rabbit')
$management_vip = hiera('management_vip')
$storage_options = hiera('storage', {})
$storage_options = hiera_hash('storage', {})
if $ceilometer['enabled'] {
$notification_topics = [$lma_collector::params::openstack_topic, $lma_collector::params::lma_topic]
@ -81,9 +81,9 @@ if $lma_collector['elasticsearch_mode'] != 'disabled' {
# Metrics
if $lma_collector['influxdb_mode'] != 'disabled' {
$nova = hiera('nova')
$neutron = hiera('quantum_settings')
$cinder = hiera('cinder')
$nova = hiera_hash('nova', {})
$neutron = hiera_hash('quantum_settings', {})
$cinder = hiera_hash('cinder', {})
if $ha_deployment {
$haproxy_socket = '/var/lib/haproxy/stats'
@ -188,7 +188,7 @@ if $alerting_mode != 'disabled' {
$nagios_password = $lma_collector['nagios_password']
} elsif $alerting_mode == 'local' {
$use_nagios = true
$lma_infra_alerting = hiera('lma_infrastructure_alerting', false)
$lma_infra_alerting = hiera_hash('lma_infrastructure_alerting', false)
$nagios_nodes = filter_nodes(hiera('nodes'), 'role', 'infrastructure_alerting')
$nagios_server = $nagios_nodes[0]['internal_address']
$nagios_user = $lma_infra_alerting['nagios_user']

165
deployment_scripts/puppet/manifests/hiera_override.pp Normal file
View File

@ -0,0 +1,165 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
$plugin_data = hiera('lma_collector', undef)
if ($plugin_data) {
$storage_options = hiera_hash('storage', {})
$tls_enabled = hiera('public_ssl', false)
$ceilometer = hiera_hash('ceilometer', {})
$ceilometer_enabled = pick($ceilometer['enabled'], false)
$gse_filters = inline_template('---
lma_collector:
gse_cluster_service:
input_message_types:
- afd_service_metric
aggregator_flag: true
entity_field: service
output_message_type: gse_service_cluster_metric
output_metric_name: cluster_service_status
interval: 10
level_1_dependencies:
nova-api:
- nova-api-backends
- nova-ec2-api-backends
- nova-endpoint
nova-novncproxy:
- nova-novncproxy-websocket-backends
nova-metadata:
- nova-api-metadata-backends
nova-scheduler:
- nova-scheduler
nova-compute:
- nova-compute
nova-conductor:
- nova-conductor
cinder-api:
- cinder-api-backends
- cinder-endpoint
- cinder-v2-endpoint
cinder-scheduler:
- cinder-scheduler
cinder-volume:
- cinder-volume
neutron-api:
- neutron-api-backends
- neutron-endpoint
neutron-l3:
- l3
neutron-dhcp:
- dhcp
neutron-ovs:
- openvswitch
keystone-api:
- keystone-public-api-backends
- keystone-admin-api-backends
- keystone-endpoint
glance-api:
- glance-api-backends
- glance-endpoint
glance-registry:
- glance-registry-api-backends
heat-api:
- heat-api-backends
- heat-cfn-api-backends
- heat-endpoint
horizon-ui:
<% if @tls_enabled then -%>
- horizon-https-backends
<% else -%>
- horizon-web-backends
<% end -%>
<% if not @storage_options["objects_ceph"] then -%>
swift-api:
- swift-api-backends
- swift-endpoint
- swift-s3-endpoint
<% end -%>
<% if @ceilometer_enabled -%>
ceilometer-api:
- ceilometer-api-backends
- ceilometer-endpoint
<% end -%>
level_2_dependencies: {}
gse_cluster_node:
input_message_types:
- afd_node_metric
aggregator_flag: true
entity_field: hostname
output_message_type: gse_node_cluster_metric
output_metric_name: cluster_node_status
interval: 10
level_1_dependencies: {}
level_2_dependencies: {}
gse_cluster_global:
input_message_types:
- gse_service_cluster_metric
- gse_node_cluster_metric
aggregator_flag: false
entity_field: cluster_name
output_message_type: gse_cluster_metric
output_metric_name: cluster_status
interval: 10
level_1_dependencies:
nova:
- nova-api
- nova-scheduler
- nova-compute
- nova-conductor
- nova-novncproxy
- nova-metadata
cinder:
- cinder-api
- cinder-scheduler
- cinder-volume
neutron:
- neutron-api
- neutron-l3
- neutron-dhcp
- neutron-metadata
- neutron-ovs
keystone:
- keystone-api
glance:
- glance-api
- glance-registry
heat:
- heat-api
horizon:
- horizon-ui
<% if not @storage_options["objects_ceph"] then -%>
swift:
- swift-api
<% end -%>
level_2_dependencies:
nova-api:
- neutron-api
- keystone-api
- cinder-api
- glance-api
cinder-api:
- keystone-api
neutron-api:
- keystone-api
glance-api:
- keystone-api
heat-api:
- keystone-api
')
lma_collector::hiera_data { 'gse_filters':
content => $gse_filters
}
}

50
deployment_scripts/puppet/modules/lma_collector/manifests/hiera_data.pp Normal file
View File

@ -0,0 +1,50 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
define lma_collector::hiera_data (
$content,
$ensure = present,
) {
$hiera_directory = '/etc/hiera/override'
$parsed_yaml = parseyaml($content)
if ! $parsed_yaml {
# With stlib <= 4.9, parseyaml() will raise an exception if the generated
# YAML is invalid so the Puppet parse will never get to the fail()
# instruction.
fail('Invalid YAML content!')
}
validate_hash($parsed_yaml)
validate_hash($parsed_yaml['lma_collector'])
package {'ruby-deep-merge':
ensure => 'installed',
}
file { $hiera_directory:
ensure => directory,
}
file { "${hiera_directory}/${name}.yaml":
ensure => file,
content => $content,
require => File[$hiera_directory],
}
file_line { "${name}_hiera_override":
path => '/etc/hiera.yaml',
line => " - override/${name}",
after => ' - override/module/%{calling_module}',
}
}

66
deployment_scripts/puppet/modules/lma_collector/spec/defines/lma_collector_hiera_data_spec.rb Normal file
View File

@ -0,0 +1,66 @@
# Copyright 2015 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
require 'spec_helper'
require 'yaml'
describe 'lma_collector::hiera_data' do
let(:title) { :foo }
let(:facts) do
{:kernel => 'Linux', :operatingsystem => 'Ubuntu',
:osfamily => 'Debian'}
end
describe 'with valid YAML' do
yaml =<<EOF
lma_collector:
some_parameter: 42
EOF
let(:params) do
{:content => yaml}
end
it { is_expected.to contain_file('/etc/hiera/override/foo.yaml') }
end
describe 'with invalid YAML' do
yaml =<<EOF
lma_collector:
some_parameter: "ee
EOF
let(:params) do
{:content => yaml}
end
it { is_expected.to raise_error(Psych::SyntaxError) }
end
describe 'with data which is not a hahs' do
yaml =<<EOF
lma_collector
EOF
let(:params) do
{:content => yaml}
end
it { is_expected.to raise_error(Puppet::Error) }
end
describe 'with missing key in YAML' do
yaml =<<EOF
not_lma_collector:
some_parameter: 42
EOF
let(:params) do
{:content => yaml}
end
it { is_expected.to raise_error(Puppet::Error) }
end
end

8
tasks.yaml
View File

@ -20,6 +20,14 @@
puppet_modules: puppet/modules
timeout: 600
- role: '*'
stage: post_deployment/8200
type: puppet
parameters:
puppet_manifest: puppet/manifests/hiera_override.pp
puppet_modules: puppet/modules
timeout: 600
- role: '*'
stage: post_deployment/8200
type: puppet