Update vertica projections for better performance

Update for lint compliance as well

Change-Id: I55f6b9a74c1c4be17c6df9c3436c5d990efb6100
Closes-Bug: #1467625
This commit is contained in:
bklei 2015-06-18 09:17:38 -06:00
parent 4657141992
commit 569975deea
31 changed files with 488 additions and 54 deletions

View File

@ -5,11 +5,10 @@ group :development, :test do
gem 'rspec-puppet', '~> 2.1.0', :require => false
gem 'metadata-json-lint'
# Needs fix for LP: #1467625
# gem 'puppet-lint-param-docs'
# gem 'puppet-lint-absolute_classname-check'
# gem 'puppet-lint-absolute_template_path'
# gem 'puppet-lint-trailing_newline-check'
gem 'puppet-lint-param-docs'
gem 'puppet-lint-absolute_classname-check'
gem 'puppet-lint-absolute_template_path'
gem 'puppet-lint-trailing_newline-check'
# Puppet 4.x related lint checks
gem 'puppet-lint-unquoted_string-check'

View File

@ -39,9 +39,9 @@ CREATE TABLE MonMetrics.DefinitionDimensions (
CREATE PROJECTION Measurements_DBD_1_rep_MonMetrics /*+createtype(D)*/
(
definition_dimensions_id ENCODING RLE,
time_stamp ENCODING DELTAVAL,
time_stamp ENCODING COMMONDELTA_COMP,
value ENCODING AUTO,
value_meta ENCODING AUTO
value_meta ENCODING RLE
)
AS
SELECT definition_dimensions_id,
@ -50,12 +50,12 @@ AS
value_meta
FROM MonMetrics.Measurements
ORDER BY definition_dimensions_id,
time_stamp
UNSEGMENTED ALL NODES;
value_meta
SEGMENTED BY MODULARHASH (definition_dimensions_id) ALL NODES OFFSET 0;
CREATE PROJECTION Definitions_DBD_2_rep_MonMetrics /*+createtype(D)*/
(
id ENCODING RLE,
id ENCODING AUTO,
name ENCODING AUTO,
tenant_id ENCODING RLE,
region ENCODING RLE
@ -66,16 +66,15 @@ AS
tenant_id,
region
FROM MonMetrics.Definitions
ORDER BY id,
ORDER BY region,
tenant_id,
region,
name
UNSEGMENTED ALL NODES;
CREATE PROJECTION Dimensions_DBD_3_rep_MonMetrics /*+createtype(D)*/
(
id ENCODING RLE,
name ENCODING AUTO,
dimension_set_id ENCODING AUTO,
name ENCODING RLE,
value ENCODING AUTO
)
AS
@ -83,15 +82,16 @@ AS
name,
value
FROM MonMetrics.Dimensions
ORDER BY dimension_set_id,
name
ORDER BY name,
dimension_set_id,
value
UNSEGMENTED ALL NODES;
CREATE PROJECTION DefinitionDimensions_DBD_4_rep_MonMetrics /*+createtype(D)*/
(
id ENCODING RLE,
definition_id,
dimension_set_id
id ENCODING AUTO,
definition_id ENCODING RLE,
dimension_set_id ENCODING AUTO
)
AS
SELECT id,
@ -99,8 +99,7 @@ AS
dimension_set_id
FROM MonMetrics.DefinitionDimensions
ORDER BY definition_id,
dimension_set_id,
id
dimension_set_id
UNSEGMENTED ALL NODES;
select refresh('MonMetrics.Measurements, MonMetrics.Definitions, MonMetrics.Dimensions, MonMetrics.DefinitionDimensions');

View File

@ -5,7 +5,113 @@
# === Parameters
#
# [*enabled*]
# TODO:add comments here.
# flag to enable/disable the monasca agent
#
# [*url*]
# url of the monasca api server to POST metrics to
#
# [*username*]
# monasca agent name
#
# [*password*]
# monasca agent password
#
# [*keystone_url*]
# keystone endpoint for authentication
#
# [*project_name*]
# name of keystone project to POST metrics for
#
# [*project_domain_id*]
# domain id of the keystone project to POST metrics for
#
# [*project_domain_name*]
# domain name of the keystone project to POST metrics for
#
# [*project_id*]
# id of keystone project to POST metrics for
#
# [*ca_file*]
# certificate file to use in keystone authentication
#
# [*max_buffer_size*]
# buffer size for metrics POSTing
#
# [*backlog_send_rate*]
# how name metrics to POST from backlog at a time
#
# [*amplifier*]
# multiplier for testing, allows POSTing the same metric multiple times
#
# [*hostname*]
# hostname for this monasca agent
#
# [*dimensions*]
# list of dimensions for this monasca agent
#
# [*recent_point_threshold*]
# number of seconds to consider a metric 'recent'
#
# [*check_freq*]
# how frequently (in seconds) to run the agent
#
# [*listen_port*]
# port for the monasca agent to listen on
#
# [*non_local_traffic*]
# flag for whether or not to support non-local traffic
# (see monasca documentation for more details)
#
# [*statsd_port*]
# port for the statsd server
#
# [*statsd_interval*]
# frequency to poll statsd
#
# [*statsd_forward_host*]
# host for statsd server
#
# [*statsd_forward_port*]
# port for statsd server
#
# [*log_level*]
# logging level -- INFO, DEBUG, ALL...
#
# [*collector_log_file*]
# logfile for monasca collector
#
# [*forwarder_log_file*]
# logfile for monasca forwarder
#
# [*monstatsd_log_file*]
# logfile for monasca statsd collector
#
# [*log_to_syslog*]
# flag for whether or not to log to syslog
#
# [*syslog_host*]
# host of the syslog server
#
# [*syslog_port*]
# port of the syslog server
#
# [*virtual_env*]
# directory of python virtual environment
#
# [*agent_user*]
# name of the monasca agent user
#
# [*agent_ensure*]
# install ensure option (present, latest..)
#
# [*install_python_deps*]
# flag for whether or not to install python dependencies
#
# [*python_dep_ensure*]
# flag for whether or not to ensure/update python dependencies
#
# [*pip_install_args*]
# arguments to pass to the pip install command
#
class monasca::agent(
$enabled = true,
@ -45,8 +151,8 @@ class monasca::agent(
$python_dep_ensure = 'present',
$pip_install_args = '',
) {
include monasca
include monasca::params
include ::monasca
include ::monasca::params
$agent_dir = "${::monasca::monasca_dir}/agent"
$additional_checksd = "${agent_dir}/checks.d"

View File

@ -1,6 +1,36 @@
# == Class: monasca::alarmdefs
#
# Class for bootstrapping monasca alarm definitions
#
# === Parameters:
#
# [*alarm_definition_config_source*]
# location of alarm definitions to bootstrap in mysql database
#
# [*admin_username*]
# name of the monasca admin user
#
# [*admin_password*]
# password of the monasca admin user
#
# [*api_server_url*]
# monasca api server endpoint
#
# [*auth_url*]
# keystone endpoint
#
# [*project_name*]
# keystone project name to bootstrap alarm definitions for
#
# [*virtual_env*]
# location of python virtual environment to install to
#
# [*install_python_deps*]
# flag for whether or not to install python dependencies
#
# [*python_dep_ensure*]
# flag for whether or not to ensure/update python dependencies
#
class monasca::alarmdefs(
$alarm_definition_config_source = 'puppet:///modules/monasca/alarm_definition_config.json',
$admin_username = 'monasca-admin',
@ -13,7 +43,7 @@ class monasca::alarmdefs(
$python_dep_ensure = 'present',
)
{
include monasca::params
include ::monasca::params
$alarm_definition_config = '/tmp/alarm_definition_config.json'
$script_name = 'bootstrap-alarm-definitions.py'

View File

@ -1,6 +1,54 @@
# == Class: monasca::api
#
# Class to setup monasca api
#
# === Parameters:
#
# [*api_user*]
# name of the monasca api user
#
# [*blobmirror*]
# url of server to install debians from
#
# [*database_type*]
# type of database backend, influxdb or vertica
#
# [*db_admin_password*]
# database admin password
#
# [*gzip_setting*]
# flag for whether to use gzip for monasca api and persister
#
# [*kafka_brokers*]
# list of kafka brokers and ports
#
# [*keystone_endpoint*]
# url of keystone server
#
# [*keystone_admin_token*]
# token for keystone admin
#
# [*mon_api_build_ver*]
# build version of the monasca api debian package
#
# [*mon_api_deb*]
# name of the monasca api debian package
#
# [*region_name*]
# openstack region name for this install
#
# [*role_delegate*]
# name of the monasca delegate role
#
# [*roles_default*]
# name of the monasca default role
#
# [*roles_agent*]
# name of the monasca agent role
#
# [*zookeeper_servers*]
# list of zookeeper servers and ports
#
class monasca::api (
$api_user = 'monasca_api',
$blobmirror = undef,
@ -18,8 +66,8 @@ class monasca::api (
$roles_agent = ['monasca-agent'],
$zookeeper_servers = undef,
) {
include monasca
include monasca::params
include ::monasca
include ::monasca::params
$api_fetch_url = "http://${blobmirror}/repos/monasca/monasca_api"
$latest_api_deb = "/tmp/${mon_api_deb}"

View File

@ -37,4 +37,4 @@ class monasca::checks::apache(
}
create_resources('monasca::checks::instances::apache', $instances)
}
}
}

View File

@ -37,4 +37,4 @@ class monasca::checks::cpu(
}
create_resources('monasca::checks::instances::cpu', $instances)
}
}
}

View File

@ -41,4 +41,4 @@ class monasca::checks::disk(
}
create_resources('monasca::checks::instances::disk', $instances)
}
}
}

View File

@ -52,4 +52,4 @@ class monasca::checks::host_alive(
}
create_resources('monasca::checks::instances::host_alive', $instances)
}
}
}

View File

@ -50,4 +50,4 @@ class monasca::checks::http_check(
}
create_resources('monasca::checks::instances::http_check', $instances)
}
}
}

View File

@ -5,12 +5,19 @@
# === Parameters
#
# [*admin_user*]
# name of the monasca admin
#
# [*admin_password*]
# password for the monasca admin
#
# [*admin_tenant_name*]
# name of the monasca admin tenant/project
#
# [*identity_uri*]
# uri of the keystone instance
#
# [*region_name*]
# openstack keystone region for this install
#
# [*cache_dir*]
# Cache directory to persist data.

View File

@ -36,4 +36,4 @@ class monasca::checks::load(
}
create_resources('monasca::checks::instances::load', $instances)
}
}
}

View File

@ -36,4 +36,4 @@ class monasca::checks::memory(
}
create_resources('monasca::checks::instances::memory', $instances)
}
}
}

View File

@ -54,4 +54,4 @@ class monasca::checks::mysql(
before => Service['monasca-agent'],
install_args => $::monasca::agent::pip_install_args,
}
}
}

View File

@ -61,4 +61,4 @@ class monasca::checks::nagios_wrapper(
if($instances){
create_resources('monasca::checks::instances::nagios_wrapper', $instances)
}
}
}

View File

@ -40,4 +40,4 @@ class monasca::checks::network(
}
create_resources('monasca::checks::instances::network', $instances)
}
}
}

View File

@ -43,4 +43,4 @@ class monasca::checks::process(
}
create_resources('monasca::checks::instances::process', $instances)
}
}
}

View File

@ -50,4 +50,4 @@ class monasca::checks::rabbitmq(
}
create_resources('monasca::checks::instances::rabbitmq', $instances)
}
}
}

View File

@ -41,4 +41,4 @@ class monasca::checks::zk(
}
create_resources('monasca::checks::instances::zk', $instances)
}
}
}

View File

@ -18,10 +18,10 @@
# DEFAULT/bar:
# value: barValue
#
# [**monasca_config**]
# [*monasca_config*]
# (optional) Allow configuration of monasca.conf configurations.
#
# [**monasca_ini**]
# [*monasca_ini*]
# (optional) Allow configuration of monasca.ini configurations.
#
# NOTE: The configuration MUST NOT be already handled by this module

View File

@ -3,7 +3,7 @@
# for configuration of thresholds, alarms, etc.
#
class monasca::db::mysql {
include monasca::params
include ::monasca::params
$sql_host = $::monasca::params::sql_host
$sql_user = $::monasca::params::sql_user

View File

@ -1,6 +1,30 @@
# == Class: monasca::influxdb::bootstrap
#
# Class for bootstrapping influxdb for monasca
#
# === Parameters:
#
# [*influxdb_password*]
# password for the influxdb admin
#
# [*influxdb_dbuser_ro_password*]
# password for the influxdb read-only user
#
# [*influxdb_def_ret_pol_name*]
# default retention policy name
#
# [*influxdb_def_ret_pol_duration*]
# default influxdb retention policy duration
#
# [*influxdb_tmp_ret_pol_name*]
# temporary retention policy
#
# [*influxdb_tmp_ret_pol_duration*]
# temporary influxdb retention policy duration
#
# [*influxdb_retention_replication*]
# influxdb retention policy replication factor
#
class monasca::influxdb::bootstrap(
$influxdb_password = undef,
$influxdb_dbuser_ro_password = undef,
@ -11,7 +35,7 @@ class monasca::influxdb::bootstrap(
$influxdb_retention_replication = 1,
)
{
include monasca::params
include ::monasca::params
$influxdb_dbuser_password = $::monasca::params::api_db_password
$script = 'bootstrap-influxdb.sh'

View File

@ -1,6 +1,32 @@
# == Class: monasca::kakfa::config
#
# Class for creating kafka topics needed by monasca
#
# === Parameters:
#
# [*kafka_zookeeper_connections*]
# list of zookeeper servers and ports
#
# [*kafka_replication_factor*]
# replication factor for kafka
#
# [*topic_config*]
# topic specific topic configuration, sample hiera:
#
# monasca::kafka::config::topic_config:
# metrics:
# partitions: 4
# events:
# partitions: 4
# alarm-notifications:
# partitions: 8
# alarm-state-transitions:
# partitions: 8
# retry-notifications:
# partitions: 2
# healthcheck:
# partitions: 4
#
class monasca::kafka::config (
$kafka_zookeeper_connections = undef,
$kafka_replication_factor = undef,

View File

@ -97,6 +97,15 @@
# no trailing '/'
# Setting this variable overrides other $internal_* parameters.
#
# [*role_agent*]
# keystone role to use for monasca agent
#
# [*role_delegate*]
# keystone role to use for monasca delegate
#
# [*role_user*]
# keystone role to use for monasca general user
#
class monasca::keystone::auth (
$auth_name = 'monasca',
$admin_email = 'monasca@localhost',
@ -121,7 +130,7 @@ class monasca::keystone::auth (
$role_delegate = 'monitoring-delegate',
$role_user = 'monasca-user',
) {
include monasca::params
include ::monasca::params
$admin_name = $::monasca::params::admin_name
$agent_name = $::monasca::params::agent_name

View File

@ -1,6 +1,48 @@
# == Class: monasca::notifications
#
# Class for configuring monasca notifications
#
# === Parameters:
#
# [*notification_user*]
# name of the monasca notification user
#
# [*from_email_address*]
# email address to send notifications from
#
# [*install_python_deps*]
# flag for whether or not to install python dependencies
#
# [*kafka_brokers*]
# list of kafka broker servers and ports
#
# [*pagerduty_url*]
# url of pager duty if used as a notification method
#
# [*python_dep_ensure*]
# flag for whether or not to ensure/update python dependencies
#
# [*smtp_password*]
# password for the smtp server
#
# [*smtp_port*]
# port on the smtp server to send mail to
#
# [*smtp_server*]
# host of the smtp server
#
# [*smtp_user*]
# name to use when authenticating agains the smtp server
#
# [*virtual_env*]
# directory of python virtual environment
#
# [*webhook_url*]
# url for webhook notifications
#
# [*zookeeper_servers*]
# list of zookeeper servers and ports
#
class monasca::notification(
$notification_user = 'monasca-notification',
$from_email_address = '',
@ -17,7 +59,7 @@ class monasca::notification(
$zookeeper_servers = 'localhost:2181',
)
{
include monasca::params
include ::monasca::params
# variables for the template
$sql_host = $::monasca::params::sql_host

View File

@ -4,6 +4,55 @@
# across all monasca services.
#
# === Parameters:
#
# [*api_db_user*]
# name of the monasca api user for the database
#
# [*api_db_password*]
# password for the monasca api database user
#
# [*port*]
# port to run monasca api server on
#
# [*api_version*]
# version of the monasca api to configure
#
# [*region*]
# default openstack region for this monasca api instance
#
# [*admin_name*]
# name of the monasca admin user
#
# [*agent_name*]
# name of the monasca agent user
#
# [*user_name*]
# name of the default monasca user
#
# [*auth_method*]
# keystone auth method, token or password
#
# [*admin_password*]
# password for the monasca admin user
#
# [*agent_password*]
# password for the monasca agent user
#
# [*user_password*]
# password for the monasca default user
#
# [*sql_host*]
# host of the mysql instance
#
# [*sql_user*]
# name of the mysql user
#
# [*sql_password*]
# password for the mysql user
#
# [*persister_config_defaults*]
# defaults for monasca persister settings
#
class monasca::params(
$api_db_user = 'mon_api',
$api_db_password = undef,

View File

@ -1,6 +1,42 @@
# == Class: monasca::persister
#
# Class to setup monasca persister
#
# === Parameters:
#
# [*blobmirror*]
# location of server to pull debian package from
#
# [*consumer_id*]
# id of the kafka consumer for this persister
#
# [*batch_size*]
# batch size of metrics/alarm to persist at the same time
#
# [*num_threads*]
# number of persister threads
#
# [*batch_seconds*]
# frequency for this perisiter to write to db
#
# [*config*]
# persister specific configuration -- allows running multiple persisters.
#
# [*db_admin_password*]
# admin password for database
#
# [*mon_pers_build_ver*]
# version of the persister to install
#
# [*mon_pers_deb*]
# name of the debian package for the persister
#
# [*pers_user*]
# name of the monasca perisister user
#
# [*zookeeper_servers*]
# list of zookeeper servers
#
class monasca::persister (
$blobmirror = undef,
$consumer_id = 1,
@ -14,8 +50,8 @@ class monasca::persister (
$pers_user = 'persister',
$zookeeper_servers = undef,
) {
include monasca
include monasca::params
include ::monasca
include ::monasca::params
$pers_fetch_url = "http://${blobmirror}/repos/monasca/monasca_persister"
$latest_pers_deb = "/tmp/${mon_pers_deb}"

View File

@ -16,7 +16,7 @@ define monasca::persister::config (
$retention_policy = 'raw',
$zookeeper_servers = $monasca::persister::zookeeper_servers,
) {
include monasca::params
include ::monasca::params
$persister_config = deep_merge($monasca::params::persister_config_defaults, $config)
$persister_service_name = $name

View File

@ -1,6 +1,27 @@
#
# Class for configuring misc storm packages for use by monasca api server
#
# [*storm_version*]
# version of apache-storm to use
#
# [*mirror*]
# location of apache-storm mirror
#
# [*install_dir*]
# location to install storm
#
# [*storm_user*]
# name of the storm user
#
# [*storm_group*]
# name of the storm group
#
# [*log_dir*]
# directory for storm logs
#
# [*nimbus_server*]
# name of the nimbus server
#
class monasca::storm::config (
$storm_version = 'apache-storm-0.9.3',
$mirror = 'http://apache.arvixe.com/storm',

View File

@ -1,6 +1,21 @@
#
# Class to install monasca api server
#
# [*blobmirror*]
# location of the server to pull debians from
#
# [*kafka_brokers*]
# list of kafka brokers
#
# [*mon_thresh_build_ver*]
# version of the thresh debian package
#
# [*mon_thresh_deb*]
# name of the thresh debian package
#
# [*zookeeper_servers*]
# list of zookeeper servers
#
class monasca::thresh (
$blobmirror = undef,
$kafka_brokers = undef,
@ -8,8 +23,8 @@ class monasca::thresh (
$mon_thresh_deb = undef,
$zookeeper_servers = undef,
) {
include monasca
include monasca::params
include ::monasca
include ::monasca::params
# variables for the template
$sql_host = $::monasca::params::sql_host

View File

@ -1,10 +1,34 @@
#
# Class for vertica specific files
#
# === Parameters
#
# [*db_user*]
# name of the database user
#
# [*db_group*]
# name of the database group
#
# [*db_admin_password*]
# database admin password
#
# [*metrics_schema*]
# location of the metrics schema/projections file
#
# [*mon_api_password*]
# database api user password
#
# [*mon_persister_password*]
# database persister user password
#
# [*monitor_password*]
# database monitor user password
#
class monasca::vertica::config (
$db_user = 'dbadmin',
$db_group = 'verticadba',
$db_admin_password = unset,
$metrics_schema = 'puppet:///modules/monasca/vertica/mon_metrics_schema.sql',
$mon_api_password = unset,
$mon_persister_password = unset,
$monitor_password = unset,
@ -15,7 +39,6 @@ class monasca::vertica::config (
$install_dir = '/var/vertica'
$alarms_schema = 'mon_alarms_schema.sql'
$grants_schema = 'mon_grants.sql'
$metrics_schema = 'mon_metrics_schema.sql'
$config_schema = 'mon_schema.sql'
$users_schema = 'mon_users.sql'
$cluster_script = 'create_mon_db_cluster.sh'
@ -46,9 +69,9 @@ class monasca::vertica::config (
require => File[$install_dir],
}
file { "${install_dir}/${metrics_schema}":
file { "${install_dir}/mon_metrics_schema.sql":
ensure => file,
source => "${files}/${metrics_schema}",
source => $metrics_schema,
mode => '0644',
owner => $db_user,
group => $db_group,