x
/
fuel-plugin-openstack-telemetry
Code Issues Proposed changes

Adds hindsight 

- downloads packages
- installs packages
- puts configs and scripts
- starts service

Change-Id: I3feaa1a4afcb016db71b76cf975f0ab8f23801d3
This commit is contained in:
Volodymyr Kornylyuk 2016-09-02 18:29:48 +03:00
parent e81d7ee353
commit f47afa82ea
15 changed files with 344 additions and 59 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
deployment_scripts/puppet/manifests/configure-influxdb.pp
View File

@ -2,7 +2,7 @@ notice('MODULAR: fuel-plugin-telemetry: influxdb-create-db.pp')
$influxdb_mode = hiera('telemetry::influxdb::mode')
if $influxdb_mode == 'remote' {
if $influxdb_mode == 'local' {
class { 'telemetry::create_influxdb_database': }

45
deployment_scripts/puppet/manifests/heka.pp
View File

@ -20,6 +20,9 @@ $influxdb_database = hiera('telemetry::influxdb::database')
$influxdb_user = hiera('telemetry::influxdb::user')
$influxdb_password = hiera('telemetry::influxdb::password')
#TODO move to hiera
$module_directory = '/usr/share/telemetry_lua_modules'
### Heka configuration
file {
@ -35,48 +38,6 @@ file {
"${config_dir}/output-influxdb-samples.toml": content => template( 'telemetry/heka/output-influxdb-samples.toml.erb' );
}
### Heka lua scripts
$heka_dir = '/usr/share/heka'
$modules_dir = '/usr/share/heka/lua_modules'
file {
$heka_dir: ensure => 'directory';
$modules_dir: ensure => 'directory';
}
file {
"${modules_dir}/decoders": ensure => 'directory';
"${modules_dir}/encoders": ensure => 'directory';
"${modules_dir}/filters": ensure => 'directory';
}
file {
"${modules_dir}/decoders/metering.lua":
source => 'puppet:///modules/telemetry/decoders/metering.lua'
;
"${modules_dir}/encoders/es_bulk.lua":
source => 'puppet:///modules/telemetry/encoders/es_bulk.lua'
;
"${modules_dir}/filters/influxdb_ceilometer_accumulator.lua":
source => 'puppet:///modules/telemetry/filters/influxdb_ceilometer_accumulator.lua'
;
}
### Heka extra modules
file {
"${modules_dir}/extra_fields.lua":
source => 'puppet:///modules/telemetry/extra_fields.lua'
;
"${modules_dir}/lma_utils.lua":
source => 'puppet:///modules/telemetry/lma_utils.lua'
;
"${modules_dir}/patterns.lua":
source => 'puppet:///modules/telemetry/patterns.lua'
;
}
# Heka Installation
$version = hiera('telemetry::heka::version')

126
deployment_scripts/puppet/manifests/hindsight.pp Normal file
View File

@ -0,0 +1,126 @@
notice('MODULAR: fuel-plugin-telemetry: hindsight.pp')
$user = 'hindsight'
$group = 'hindsight'
$influxdb_address = hiera('telemetry::influxdb::address')
$influxdb_port = hiera('telemetry::influxdb::port')
$influxdb_database = hiera('telemetry::influxdb::database')
$influxdb_user = hiera('telemetry::influxdb::user')
$influxdb_password = hiera('telemetry::influxdb::password')
# TDOD field in UI, add in hiera.pp
$metadata_fields = join(['status deleted container_format min_ram updated_at ',
'min_disk is_public size checksum created_at disk_format protected instance_host ',
'host display_name instance_id instance_type status state'])
#hiera('telemetry::metadata_fields')
# TODO settings/hiera
$topics = 'metering.sample'
#TODO kafka integration
$brokerlist = '"broker1:9092"'
# Install packages
package { 'libluasandbox-dev': }
package { 'libluasandbox1': }
package { 'hindsight': }
package { 'librdkafka1': }
package { 'lua-sandbox-extensions': }
package { 'python-oslo.messaging': }
# User/group
user { $user:
ensure => 'present',
groups => $group,
}
group { $group:
ensure => 'present',
}
# Directories
$conf_dir = '/etc/telemetry_hindsight'
$hindsight_dir = '/usr/share/telemetry_hindsight'
$run_dir = "${hindsight_dir}/run"
# parent /var/lib/hindsight?
$output_dir = '/var/lib/hindsight/output'
$sandbox_dir = '/usr/lib/x86_64-linux-gnu/luasandbox'
$templates = 'telemetry/hindsight/'
$dirs = [
$conf_dir,
$hindsight_dir,
$run_dir,
"${run_dir}/analysis",
"${run_dir}/input",
"${run_dir}/output",
$output_dir
]
file { $dirs:
ensure => 'directory',
owner => $user,
group => $group,
recurse => true,
require => Package['hindsight']
}
$files_defaults = {
owner => $user,
group => $group,
before => Service['hindsight']
}
# Config files
file { '/etc/telemetry_hindsight/hindsight.cfg':
ensure => 'present',
owner => $user,
group => $group,
content => template( 'telemetry/hindsight/hindsight.cfg.erb' ),
require => Package['hindsight']
}
# Templates
$configs = {
"${run_dir}/output/influxdb_ceilometer.cfg" => {
content => template( "${templates}/output/influxdb_ceilometer.cfg.erb"),
},
"${run_dir}/input/ceilometer_kafka.cfg" => {
content => template( "${templates}/input/kafka_input.cfg.erb"),
}
}
create_resources(file, $configs, $files_defaults)
# Files
$scripts = {
"${run_dir}/output/influxdb_tcp.lua" => {
source => 'puppet:///modules/telemetry/hindsight/run/output/influxdb_tcp.lua'
},
"${run_dir}/input/kafka_input.lua" => {
source => 'puppet:///modules/telemetry/hindsight/run/input/kafka_input.lua'
}
}
create_resources(file, $scripts, $files_defaults)
file { '/etc/init/hindsight.conf':
content => template( "${templates}/init.conf.erb"),
before => Service['hindsight']
}
service { 'hindsight':
ensure => 'running',
enable => true,
provider => 'upstart',
require => File['/etc/init/hindsight.conf']
}
# TODO move to separated manifest
#ceilometer_config { 'notification/messaging_urls': value => ['http1','http2'] }

74
deployment_scripts/puppet/manifests/lua-scripts.pp Normal file
View File

@ -0,0 +1,74 @@
notice('MODULAR: fuel-plugin-telemetry: lua-scripts.pp')
$modules_dir = '/usr/share/telemetry_lua_modules'
$dirs = [
$modules_dir,
"${modules_dir}/common",
"${modules_dir}/decoders",
"${modules_dir}/encoders",
"${modules_dir}/filters",
'/usr/share/heka',
'/usr/share/heka/lua_modules'
]
file { $dirs:
ensure => 'directory',
# owner => $user,
# group => $group,
recurse => true,
}
# Common script for heka and hindsight
$scripts = {
"${modules_dir}/common/ceilometer.lua" => {
source => 'puppet:///modules/telemetry/common/ceilometer.lua'
},
"${modules_dir}/common/samples.lua" => {
source => 'puppet:///modules/telemetry/common/samples.lua'
},
"${modules_dir}/decoders/metering.lua" => {
source => 'puppet:///modules/telemetry/decoders/metering.lua'
}
}
#TODO
# files shoud by readable for hindsight user
# $files_defaults = {
# owner => $user,
# group => $group;
# before => Service['hindsight']
# }
create_resources(file, $scripts)
# Scripts imported from lam_collector moduele
# see pre_build_hook
$lma_scripts = {
"${modules_dir}/common/lma_utils.lua" => {
source => 'puppet:///modules/telemetry/import/common/lma_utils.lua'
},
"${modules_dir}/common/patterns.lua" => {
source => 'puppet:///modules/telemetry/import/common/patterns.lua'
},
"${modules_dir}/common/influxdb.lua" => {
source => 'puppet:///modules/telemetry/import/common/influxdb.lua'
},
"${modules_dir}/common/accumulator.lua" => {
source => 'puppet:///modules/telemetry/import/common/accumulator.lua'
},
"${modules_dir}/filters/influxdb_accumulator.lua" => {
source => 'puppet:///modules/telemetry/import/filters/influxdb_accumulator.lua'
},
"${modules_dir}/encoders/es_ceilometer_resources.lua" => {
source => 'puppet:///modules/telemetry/import/encoders/es_ceilometer_resources.lua'
},
"${modules_dir}/common/extra_fields.lua" => {
source => 'puppet:///modules/telemetry/common/extra_fields.lua'
}
}
create_resources(file, $lma_scripts)

7
deployment_scripts/puppet/modules/telemetry/templates/heka/decoder-sample.toml.erb
View File

@ -1,6 +1,9 @@
[sample_decoder]
type = "SandboxDecoder"
filename = "/usr/share/heka/lua_modules/decoders/metering.lua"
filename = "/usr/share/telemetry_lua_modules/decoders/metering.lua"
module_directory = "<%= @module_directory %>/common;/usr/share/heka/lua_modules"
[sample_decoder.config]
metadata_fields = 'status deleted container_format min_ram updated_at min_disk is_public size checksum created_at disk_format protected instance_host host display_name instance_id instance_type status state'
metadata_fields = 'status deleted container_format min_ram updated_at min_disk is_public size checksum created_at disk_format protected instance_host host display_name instance_id instance_type status state'
decoder = "ceilometer"

11
deployment_scripts/puppet/modules/telemetry/templates/heka/encoder-resource-elasticsearch.toml.erb
View File

@ -1,7 +1,8 @@
[elasticsearch_resource_encoder]
type = "SandboxEncoder"
filename = "/usr/share/heka/lua_modules/encoders/es_bulk.lua"
type = "SandboxEncoder"
filename = "/usr/share/telemetry_lua_modules/encoders/es_ceilometer_resources.lua"
module_directory = "<%= @module_directory %>/common;/usr/share/heka/lua_modules"
[elasticsearch_resource_encoder.config]
index = "ceilometer_resource"
type_name = "source"
[elasticsearch_resource_encoder.config]
index = "ceilometer_resource"
type_name = "source"

11
deployment_scripts/puppet/modules/telemetry/templates/heka/filter-influxdb_accumulator_sample.toml.erb
View File

@ -1,12 +1,13 @@
[ceilometer_influxdb_accumulator_filter]
type = "SandboxFilter"
filename = "/usr/share/heka/lua_modules/filters/influxdb_ceilometer_accumulator.lua"
filename = "/usr/share/telemetry_lua_modules/filters/influxdb_accumulator.lua"
module_directory = "<%= @module_directory %>/common;/usr/share/heka/lua_modules"
preserve_data = false
message_matcher = "Fields[aggregator] == NIL && Type =~ /sample.bulk_metric$/"
ticker_interval = 1
[ceilometer_influxdb_accumulator_filter.config]
tag_fields = 'deployment_id environment_label hostname tenant_id user_id'
time_precision = 'ns'
payload_name = 'sample_data'
flush_count = 10
tag_fields = 'deployment_id environment_label hostname tenant_id user_id'
time_precision = 'ns'
payload_name = 'sample_data'
flush_count = 10

7
deployment_scripts/puppet/modules/telemetry/templates/hindsight/analysis/influxdb_accumulator.cfg.erb Normal file
View File

@ -0,0 +1,7 @@
filename = "influxdb_accumulator.lua"
message_matcher = "Type == 'ceilometer_samples'"
ticker_interval = 1
log_level = 7
payload_name = 'ceilometer'
time_precision = 'ns'
bulk_metric_type_matcher = 'ceilometer_samples'

27
deployment_scripts/puppet/modules/telemetry/templates/hindsight/hindsight.cfg.erb Normal file
View File

@ -0,0 +1,27 @@
max_message_size = 3 * 1024 * 1024
backpressure_disk_free = 100
output_size = 100 * 1024 * 1024
backpressure = 25
output_path = "<%= @output_dir %>"
sandbox_load_path = "<%= @hindsight_dir %>/load"
sandbox_run_path = "<%= @hindsight_dir %>/run"
analysis_lua_path = "<%= @sandbox_dir %>/modules/?.lua;<%= @sandbox_dir %>/modules/lpeg/?.lua;/usr/share/telemetry_lua_modules/common/?.lua"
analysis_lua_cpath = "<%= @sandbox_dir %>/modules/?.so"
io_lua_path = analysis_lua_path .. ";<%= @sandbox_dir %>/io_modules/?.lua"
io_lua_cpath = analysis_lua_cpath .. ";<%= @sandbox_dir %>/?.so"
input_defaults = {
-- output_limit = 64 * 1024
-- memory_limit = 8 * 1024 * 1024
-- instruction_limit = 1e6
-- preserve_data = false
-- ticker_interval = 0
}
analysis_defaults = {
}
output_defaults = {
}

6
deployment_scripts/puppet/modules/telemetry/templates/hindsight/init.conf.erb Normal file
View File

@ -0,0 +1,6 @@
description "Hindsight service"
start on runlevel [2345]
stop on shutdown
exec start-stop-daemon --start -c hindsight --exec /usr/bin/hindsight -- /etc/hindsight/hindsight.cfg

11
deployment_scripts/puppet/modules/telemetry/templates/hindsight/input/kafka_input.cfg.erb Normal file
View File

@ -0,0 +1,11 @@
filename = "kafka_input.lua"
brokerlist = <%= @brokerlist %>
topics = {"<%= @topics %>"}
consumer_conf = {
["group.id"] = "telemetry_collector",
["message.max.bytes"] = 16024,
}
topic_conf = {}
metadata_fields = "<%= @metadata_fields %>"
instuction_limit = 0
ticker_interval = 60

10
deployment_scripts/puppet/modules/telemetry/templates/hindsight/output/influxdb_ceilometer.cfg.erb Normal file
View File

@ -0,0 +1,10 @@
filename = "influxdb_tcp.lua"
host = "<%= @influxdb_address %>"
port = <%= @influxdb_port %>
database = "<%= @influxdb_database %>"
username = "<%= @influxdb_user %>"
password = "<%= @influxdb_password %>"
batch_max_lines = 5000
message_matcher = "Fields[payload_name] == 'ceilometer'"
ticker_interval = 1
memory_limit = 100 * 1024 * 1024

24
deployment_tasks.yaml
View File

@ -229,17 +229,39 @@
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 300
- id: telemetry-heka
- id: telemetry-lua-scripts
type: puppet
version: 2.1.0
groups: [primary-controller, controller]
required_for: [post_deployment_end]
requires: [telemetry-influxdb-create-db]
parameters:
puppet_manifest: puppet/manifests/lua-scripts.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 300
- id: telemetry-heka
type: puppet
version: 2.1.0
groups: [primary-controller, controller]
required_for: [post_deployment_end]
requires: [telemetry-lua-scripts]
parameters:
puppet_manifest: puppet/manifests/heka.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 300
- id: telemetry-hindsight
type: puppet
version: 2.1.0
groups: [primary-controller, controller]
required_for: [post_deployment_end]
requires: [telemetry-heka]
parameters:
puppet_manifest: puppet/manifests/hindsight.pp
puppet_modules: puppet/modules:/etc/puppet/modules
timeout: 300
# skip base tasks
- id: ceilometer-radosgw-user
type: skipped

4
functions.sh
View File

@ -17,8 +17,8 @@ set -eux
ROOT="$(dirname "$(readlink -f "$0")")"
MODULES_DIR="${ROOT}"/deployment_scripts/puppet/modules
RPM_REPO="${ROOT}"/repositories/centos/
DEB_REPO="${ROOT}"/repositories/ubuntu/
RPM_REPO="${ROOT}"/repositories/centos
DEB_REPO="${ROOT}"/repositories/ubuntu
function get_package_path {
FILE=$(basename "$1")

38
pre_build_hook
View File

@ -6,6 +6,20 @@ set -eux
HEKA_VERSION="0.10.0"
COLLECTOR_TAG="0.10.0"
REPO_URL="http://mirror.fuel-infra.org/mos-repos/ubuntu/snapshots/9.0-2016-09-03-050322/pool/main"
# Hindsight packages
download_packages \
$REPO_URL/h/hindsight/hindsight_0.9.0-1~ub14.04%2bmos2_amd64.deb \
$REPO_URL/libr/librdkafka/librdkafka1_0.9.1-1~ub14.04%2bmos1_amd64.deb \
$REPO_URL/l/lua-sandbox/libluasandbox1_1.0.3-5~ub14.04%2bmos1_amd64.deb \
$REPO_URL/l/lua-sandbox/libluasandbox-dev_1.0.3-5~ub14.04%2bmos1_amd64.deb \
$REPO_URL/l/lua-sandbox-extensions/lua-sandbox-extensions_0.0~git20160812.98065e76-1~ub14.04%2bmos3_amd64.deb
# Oslo messaging
download_packages $REPO_URL/p/python-oslo.messaging/python-oslo-messaging_4.6.1-3~u14.04+mos9_all.deb
# Download Heka deb package
@ -13,6 +27,28 @@ download_packages \
https://github.com/elemoine/heka/releases/download/ratelimit-1/heka_${HEKA_VERSION}_amd64.deb
check_md5sum heka_${HEKA_VERSION}_amd64.deb 69514d94173181a8d1dcab769062fdac
# Lua scripts from lma-collector plugin
# TODO use version 1.0 after release (not master)
URL="https://github.com/openstack/fuel-plugin-lma-collector/archive/master.tar.gz"
SCRIPT_PATH="fuel-plugin-lma-collector-master/deployment_scripts/puppet/modules/lma_collector/files/plugins"
DESTINATION="deployment_scripts/puppet/modules/telemetry/files/import"
TEMP_DIR=`mktemp -u`
download_file $URL master.tar.gz $TEMP_DIR
tar -xf $TEMP_DIR/master.tar.gz -C $TEMP_DIR --strip-components=7 $SCRIPT_PATH
mkdir -p $DESTINATION/common/
cp -f $TEMP_DIR/common/lma_utils.lua $DESTINATION/common/
cp -f $TEMP_DIR/common/patterns.lua $DESTINATION/common/
cp -f $TEMP_DIR/common/influxdb.lua $DESTINATION/common/
cp -f $TEMP_DIR/common/accumulator.lua $DESTINATION/common/
mkdir -p $DESTINATION/filters/
cp -f $TEMP_DIR/filters/influxdb_accumulator.lua $DESTINATION/filters/
mkdir -p $DESTINATION/encoders/
cp -f $TEMP_DIR/encoders/es_ceilometer_resources.lua $DESTINATION/encoders/
rm -fr $TEMP_DIR
# Download Heka puppet module from lma collector plugin
URL="https://github.com/openstack/fuel-plugin-lma-collector/archive/${COLLECTOR_TAG}.tar.gz"
@ -21,4 +57,4 @@ DESTINATION=deployment_scripts/puppet/modules
TEMP_DIR=`mktemp -u`
download_file $URL $COLLECTOR_TAG.tar.gz $TEMP_DIR
tar -xf $TEMP_DIR/$COLLECTOR_TAG.tar.gz -C $DESTINATION --strip-components=4 $HEKA_MODULE_PATH
rm -fr $TEMP_DIR
rm -fr $TEMP_DIR