Update vertica projections for better performance

Update for lint compliance as well

Change-Id: I55f6b9a74c1c4be17c6df9c3436c5d990efb6100
Closes-Bug: #1467625

Gemfile

@@ -5,11 +5,10 @@ group :development, :test do
   gem 'rspec-puppet', '~> 2.1.0', :require => false
 
   gem 'metadata-json-lint'
-# Needs fix for LP: #1467625
-# gem 'puppet-lint-param-docs'
-# gem 'puppet-lint-absolute_classname-check'
-# gem 'puppet-lint-absolute_template_path'
-# gem 'puppet-lint-trailing_newline-check'
+  gem 'puppet-lint-param-docs'
+  gem 'puppet-lint-absolute_classname-check'
+  gem 'puppet-lint-absolute_template_path'
+  gem 'puppet-lint-trailing_newline-check'
 
   # Puppet 4.x related lint checks
   gem 'puppet-lint-unquoted_string-check'

files/vertica/mon_metrics_schema.sql

@@ -39,9 +39,9 @@ CREATE TABLE MonMetrics.DefinitionDimensions (
 CREATE PROJECTION Measurements_DBD_1_rep_MonMetrics /*+createtype(D)*/
 (
  definition_dimensions_id ENCODING RLE,
- time_stamp ENCODING DELTAVAL,
+ time_stamp ENCODING COMMONDELTA_COMP,
  value ENCODING AUTO,
- value_meta ENCODING AUTO
+ value_meta ENCODING RLE
 )
 AS
  SELECT definition_dimensions_id,
@@ -50,12 +50,12 @@ AS
         value_meta
  FROM MonMetrics.Measurements
  ORDER BY definition_dimensions_id,
-          time_stamp
-UNSEGMENTED ALL NODES;
+          value_meta
+SEGMENTED BY MODULARHASH (definition_dimensions_id) ALL NODES OFFSET 0;
 
 CREATE PROJECTION Definitions_DBD_2_rep_MonMetrics /*+createtype(D)*/
 (
- id ENCODING RLE,
+ id ENCODING AUTO,
  name ENCODING AUTO,
  tenant_id ENCODING RLE,
  region ENCODING RLE
@@ -66,16 +66,15 @@ AS
         tenant_id,
         region
  FROM MonMetrics.Definitions
- ORDER BY id,
+ ORDER BY region,
           tenant_id,
-          region,
           name
 UNSEGMENTED ALL NODES;
 
 CREATE PROJECTION Dimensions_DBD_3_rep_MonMetrics /*+createtype(D)*/
 (
- id ENCODING RLE,
- name ENCODING AUTO,
+ dimension_set_id ENCODING AUTO,
+ name ENCODING RLE,
  value ENCODING AUTO
 )
 AS
@@ -83,15 +82,16 @@ AS
         name,
         value
  FROM MonMetrics.Dimensions
- ORDER BY dimension_set_id,
-          name
+ ORDER BY name,
+          dimension_set_id,
+          value
 UNSEGMENTED ALL NODES;
 
 CREATE PROJECTION DefinitionDimensions_DBD_4_rep_MonMetrics /*+createtype(D)*/
 (
- id ENCODING RLE,
- definition_id,
- dimension_set_id
+ id ENCODING AUTO,
+ definition_id ENCODING RLE,
+ dimension_set_id ENCODING AUTO
 )
 AS
  SELECT id,
@@ -99,8 +99,7 @@ AS
         dimension_set_id
  FROM MonMetrics.DefinitionDimensions
  ORDER BY definition_id,
-          dimension_set_id,
-          id
+          dimension_set_id
 UNSEGMENTED ALL NODES;
 
 select refresh('MonMetrics.Measurements, MonMetrics.Definitions, MonMetrics.Dimensions, MonMetrics.DefinitionDimensions');

manifests/agent.pp

@@ -5,7 +5,113 @@
 # === Parameters
 #
 # [*enabled*]
-#   TODO:add comments here.
+#   flag to enable/disable the monasca agent
+#
+# [*url*]
+#   url of the monasca api server to POST metrics to
+#
+# [*username*]
+#   monasca agent name
+#
+# [*password*]
+#   monasca agent password
+#
+# [*keystone_url*]
+#   keystone endpoint for authentication
+#
+# [*project_name*]
+#   name of keystone project to POST metrics for
+#
+# [*project_domain_id*]
+#   domain id of the keystone project to POST metrics for
+#
+# [*project_domain_name*]
+#   domain name of the keystone project to POST metrics for
+#
+# [*project_id*]
+#   id of keystone project to POST metrics for
+#
+# [*ca_file*]
+#   certificate file to use in keystone authentication
+#
+# [*max_buffer_size*]
+#   buffer size for metrics POSTing
+#
+# [*backlog_send_rate*]
+#   how name metrics to POST from backlog at a time
+#
+# [*amplifier*]
+#   multiplier for testing, allows POSTing the same metric multiple times
+#
+# [*hostname*]
+#   hostname for this monasca agent
+#
+# [*dimensions*]
+#   list of dimensions for this monasca agent
+#
+# [*recent_point_threshold*]
+#   number of seconds to consider a metric 'recent'
+#
+# [*check_freq*]
+#   how frequently (in seconds) to run the agent
+#
+# [*listen_port*]
+#   port for the monasca agent to listen on
+#
+# [*non_local_traffic*]
+#   flag for whether or not to support non-local traffic
+#   (see monasca documentation for more details)
+#
+# [*statsd_port*]
+#   port for the statsd server
+#
+# [*statsd_interval*]
+#   frequency to poll statsd
+#
+# [*statsd_forward_host*]
+#   host for statsd server
+#
+# [*statsd_forward_port*]
+#   port for statsd server
+#
+# [*log_level*]
+#   logging level -- INFO, DEBUG, ALL...
+#
+# [*collector_log_file*]
+#   logfile for monasca collector
+#
+# [*forwarder_log_file*]
+#   logfile for monasca forwarder
+#
+# [*monstatsd_log_file*]
+#   logfile for monasca statsd collector
+#
+# [*log_to_syslog*]
+#   flag for whether or not to log to syslog
+#
+# [*syslog_host*]
+#   host of the syslog server
+#
+# [*syslog_port*]
+#   port of the syslog server
+#
+# [*virtual_env*]
+#   directory of python virtual environment
+#
+# [*agent_user*]
+#   name of the monasca agent user
+#
+# [*agent_ensure*]
+#   install ensure option (present, latest..)
+#
+# [*install_python_deps*]
+#   flag for whether or not to install python dependencies
+#
+# [*python_dep_ensure*]
+#   flag for whether or not to ensure/update python dependencies
+#
+# [*pip_install_args*]
+#   arguments to pass to the pip install command
 #
 class monasca::agent(
   $enabled                 = true,
@@ -45,8 +151,8 @@ class monasca::agent(
   $python_dep_ensure       = 'present',
   $pip_install_args        = '',
 ) {
-  include monasca
-  include monasca::params
+  include ::monasca
+  include ::monasca::params
 
   $agent_dir = "${::monasca::monasca_dir}/agent"
   $additional_checksd = "${agent_dir}/checks.d"

manifests/alarmdefs.pp

@@ -1,6 +1,36 @@
+# == Class: monasca::alarmdefs
 #
 # Class for bootstrapping monasca alarm definitions
 #
+# === Parameters:
+#
+# [*alarm_definition_config_source*]
+#   location of alarm definitions to bootstrap in mysql database
+#
+# [*admin_username*]
+#   name of the monasca admin user
+#
+# [*admin_password*]
+#   password of the monasca admin user
+#
+# [*api_server_url*]
+#   monasca api server endpoint
+#
+# [*auth_url*]
+#   keystone endpoint
+#
+# [*project_name*]
+#   keystone project name to bootstrap alarm definitions for
+#
+# [*virtual_env*]
+#   location of python virtual environment to install to
+#
+# [*install_python_deps*]
+#   flag for whether or not to install python dependencies
+#
+# [*python_dep_ensure*]
+#   flag for whether or not to ensure/update python dependencies
+#
 class monasca::alarmdefs(
   $alarm_definition_config_source = 'puppet:///modules/monasca/alarm_definition_config.json',
   $admin_username = 'monasca-admin',
@@ -13,7 +43,7 @@ class monasca::alarmdefs(
   $python_dep_ensure       = 'present',
 )
 {
-  include monasca::params
+  include ::monasca::params
 
   $alarm_definition_config = '/tmp/alarm_definition_config.json'
   $script_name = 'bootstrap-alarm-definitions.py'

manifests/api.pp

@@ -1,6 +1,54 @@
+# == Class: monasca::api
 #
 # Class to setup monasca api
 #
+# === Parameters:
+#
+# [*api_user*]
+#   name of the monasca api user
+#
+# [*blobmirror*]
+#   url of server to install debians from
+#
+# [*database_type*]
+#   type of database backend, influxdb or vertica
+#
+# [*db_admin_password*]
+#   database admin password
+#
+# [*gzip_setting*]
+#   flag for whether to use gzip for monasca api and persister
+#
+# [*kafka_brokers*]
+#   list of kafka brokers and ports
+#
+# [*keystone_endpoint*]
+#   url of keystone server
+#
+# [*keystone_admin_token*]
+#   token for keystone admin
+#
+# [*mon_api_build_ver*]
+#   build version of the monasca api debian package
+#
+# [*mon_api_deb*]
+#   name of the monasca api debian package
+#
+# [*region_name*]
+#   openstack region name for this install
+#
+# [*role_delegate*]
+#   name of the monasca delegate role
+#
+# [*roles_default*]
+#   name of the monasca default role
+#
+# [*roles_agent*]
+#   name of the monasca agent role
+#
+# [*zookeeper_servers*]
+#   list of zookeeper servers and ports
+#
 class monasca::api (
   $api_user             = 'monasca_api',
   $blobmirror           = undef,
@@ -18,8 +66,8 @@ class monasca::api (
   $roles_agent          = ['monasca-agent'],
   $zookeeper_servers    = undef,
 ) {
-  include monasca
-  include monasca::params
+  include ::monasca
+  include ::monasca::params
 
   $api_fetch_url = "http://${blobmirror}/repos/monasca/monasca_api"
   $latest_api_deb = "/tmp/${mon_api_deb}"

manifests/checks/libvirt.pp

@@ -5,12 +5,19 @@
 # === Parameters
 #
 # [*admin_user*]
+#   name of the monasca admin
 #
 # [*admin_password*]
+#   password for the monasca admin
 #
 # [*admin_tenant_name*]
+#   name of the monasca admin tenant/project
 #
 # [*identity_uri*]
+#   uri of the keystone instance
+#
+# [*region_name*]
+#   openstack keystone region for this install
 #
 # [*cache_dir*]
 #   Cache directory to persist data.

manifests/config.pp

@@ -18,10 +18,10 @@
 #     DEFAULT/bar:
 #       value: barValue
 #
-# [**monasca_config**]
+# [*monasca_config*]
 #   (optional) Allow configuration of monasca.conf configurations.
 #
-# [**monasca_ini**]
+# [*monasca_ini*]
 #   (optional) Allow configuration of monasca.ini configurations.
 #
 #   NOTE: The configuration MUST NOT be already handled by this module

manifests/db/mysql.pp

@@ -3,7 +3,7 @@
 # for configuration of thresholds, alarms, etc.
 #
 class monasca::db::mysql {
-  include monasca::params
+  include ::monasca::params
 
   $sql_host            = $::monasca::params::sql_host
   $sql_user            = $::monasca::params::sql_user

manifests/influxdb/bootstrap.pp

@@ -1,6 +1,30 @@
+# == Class: monasca::influxdb::bootstrap
 #
 # Class for bootstrapping influxdb for monasca
 #
+# === Parameters:
+#
+# [*influxdb_password*]
+#   password for the influxdb admin
+#
+# [*influxdb_dbuser_ro_password*]
+#   password for the influxdb read-only user
+#
+# [*influxdb_def_ret_pol_name*]
+#   default retention policy name
+#
+# [*influxdb_def_ret_pol_duration*]
+#   default influxdb retention policy duration
+#
+# [*influxdb_tmp_ret_pol_name*]
+#   temporary retention policy
+#
+# [*influxdb_tmp_ret_pol_duration*]
+#   temporary influxdb retention policy duration
+#
+# [*influxdb_retention_replication*]
+#   influxdb retention policy replication factor
+#
 class monasca::influxdb::bootstrap(
   $influxdb_password = undef,
   $influxdb_dbuser_ro_password = undef,
@@ -11,7 +35,7 @@ class monasca::influxdb::bootstrap(
   $influxdb_retention_replication = 1,
 )
 {
-  include monasca::params
+  include ::monasca::params
 
   $influxdb_dbuser_password = $::monasca::params::api_db_password
   $script = 'bootstrap-influxdb.sh'

manifests/kafka/config.pp

@@ -1,6 +1,32 @@
+# == Class: monasca::kakfa::config
 #
 # Class for creating kafka topics needed by monasca
 #
+# === Parameters:
+#
+# [*kafka_zookeeper_connections*]
+#   list of zookeeper servers and ports
+#
+# [*kafka_replication_factor*]
+#   replication factor for kafka
+#
+# [*topic_config*]
+#   topic specific topic configuration, sample hiera:
+#
+#   monasca::kafka::config::topic_config:
+#     metrics:
+#       partitions: 4
+#     events:
+#       partitions: 4
+#     alarm-notifications:
+#       partitions: 8
+#     alarm-state-transitions:
+#       partitions: 8
+#     retry-notifications:
+#       partitions: 2
+#     healthcheck:
+#       partitions: 4
+#
 class monasca::kafka::config (
   $kafka_zookeeper_connections = undef,
   $kafka_replication_factor    = undef,

manifests/keystone/auth.pp

@@ -97,6 +97,15 @@
 #    no trailing '/'
 #    Setting this variable overrides other $internal_* parameters.
 #
+# [*role_agent*]
+#   keystone role to use for monasca agent
+#
+# [*role_delegate*]
+#   keystone role to use for monasca delegate
+#
+# [*role_user*]
+#   keystone role to use for monasca general user
+#
 class monasca::keystone::auth (
   $auth_name           = 'monasca',
   $admin_email         = 'monasca@localhost',
@@ -121,7 +130,7 @@ class monasca::keystone::auth (
   $role_delegate       = 'monitoring-delegate',
   $role_user           = 'monasca-user',
 ) {
-  include monasca::params
+  include ::monasca::params
 
   $admin_name = $::monasca::params::admin_name
   $agent_name = $::monasca::params::agent_name

manifests/notification.pp

@@ -1,6 +1,48 @@
+# == Class: monasca::notifications
 #
 # Class for configuring monasca notifications
 #
+# === Parameters:
+#
+# [*notification_user*]
+#   name of the monasca notification user
+#
+# [*from_email_address*]
+#   email address to send notifications from
+#
+# [*install_python_deps*]
+#   flag for whether or not to install python dependencies
+#
+# [*kafka_brokers*]
+#   list of kafka broker servers and ports
+#
+# [*pagerduty_url*]
+#   url of pager duty if used as a notification method
+#
+# [*python_dep_ensure*]
+#   flag for whether or not to ensure/update python dependencies
+#
+# [*smtp_password*]
+#   password for the smtp server
+#
+# [*smtp_port*]
+#   port on the smtp server to send mail to
+#
+# [*smtp_server*]
+#   host of the smtp server
+#
+# [*smtp_user*]
+#   name to use when authenticating agains the smtp server
+#
+# [*virtual_env*]
+#   directory of python virtual environment
+#
+# [*webhook_url*]
+#   url for webhook notifications
+#
+# [*zookeeper_servers*]
+#   list of zookeeper servers and ports
+#
 class monasca::notification(
   $notification_user   = 'monasca-notification',
   $from_email_address  = '',
@@ -17,7 +59,7 @@ class monasca::notification(
   $zookeeper_servers   = 'localhost:2181',
 )
 {
-  include monasca::params
+  include ::monasca::params
 
   # variables for the template
   $sql_host = $::monasca::params::sql_host

manifests/params.pp

@@ -4,6 +4,55 @@
 # across all monasca services.
 #
 # === Parameters:
+#
+# [*api_db_user*]
+#   name of the monasca api user for the database
+#
+# [*api_db_password*]
+#   password for the monasca api database user
+#
+# [*port*]
+#   port to run monasca api server on
+#
+# [*api_version*]
+#   version of the monasca api to configure
+#
+# [*region*]
+#   default openstack region for this monasca api instance
+#
+# [*admin_name*]
+#   name of the monasca admin user
+#
+# [*agent_name*]
+#   name of the monasca agent user
+#
+# [*user_name*]
+#   name of the default monasca user
+#
+# [*auth_method*]
+#   keystone auth method, token or password
+#
+# [*admin_password*]
+#   password for the monasca admin user
+#
+# [*agent_password*]
+#   password for the monasca agent user
+#
+# [*user_password*]
+#   password for the monasca default user
+#
+# [*sql_host*]
+#   host of the mysql instance
+#
+# [*sql_user*]
+#   name of the mysql user
+#
+# [*sql_password*]
+#   password for the mysql user
+#
+# [*persister_config_defaults*]
+#   defaults for monasca persister settings
+#
 class monasca::params(
     $api_db_user     = 'mon_api',
     $api_db_password = undef,

manifests/persister.pp

@@ -1,6 +1,42 @@
+# == Class: monasca::persister
 #
 # Class to setup monasca persister
 #
+# === Parameters:
+#
+# [*blobmirror*]
+#   location of server to pull debian package from
+#
+# [*consumer_id*]
+#   id of the kafka consumer for this persister
+#
+# [*batch_size*]
+#   batch size of metrics/alarm to persist at the same time
+#
+# [*num_threads*]
+#   number of persister threads
+#
+# [*batch_seconds*]
+#   frequency for this perisiter to write to db
+#
+# [*config*]
+#   persister specific configuration -- allows running multiple persisters.
+#
+# [*db_admin_password*]
+#   admin password for database
+#
+# [*mon_pers_build_ver*]
+#   version of the persister to install
+#
+# [*mon_pers_deb*]
+#   name of the debian package for the persister
+#
+# [*pers_user*]
+#   name of the monasca perisister user
+#
+# [*zookeeper_servers*]
+#   list of zookeeper servers
+#
 class monasca::persister (
   $blobmirror         = undef,
   $consumer_id        = 1,
@@ -14,8 +50,8 @@ class monasca::persister (
   $pers_user          = 'persister',
   $zookeeper_servers  = undef,
 ) {
-  include monasca
-  include monasca::params
+  include ::monasca
+  include ::monasca::params
 
   $pers_fetch_url = "http://${blobmirror}/repos/monasca/monasca_persister"
   $latest_pers_deb = "/tmp/${mon_pers_deb}"

manifests/persister/config.pp

@@ -16,7 +16,7 @@ define monasca::persister::config (
   $retention_policy   = 'raw',
   $zookeeper_servers  = $monasca::persister::zookeeper_servers,
 ) {
-  include monasca::params
+  include ::monasca::params
   $persister_config = deep_merge($monasca::params::persister_config_defaults, $config)
 
   $persister_service_name = $name

manifests/storm/config.pp

@@ -1,6 +1,27 @@
 #
 # Class for configuring misc storm packages for use by monasca api server
 #
+# [*storm_version*]
+#   version of apache-storm to use
+#
+# [*mirror*]
+#   location of apache-storm mirror
+#
+# [*install_dir*]
+#   location to install storm
+#
+# [*storm_user*]
+#   name of the storm user
+#
+# [*storm_group*]
+#   name of the storm group
+#
+# [*log_dir*]
+#   directory for storm logs
+#
+# [*nimbus_server*]
+#   name of the nimbus server
+#
 class monasca::storm::config (
   $storm_version = 'apache-storm-0.9.3',
   $mirror = 'http://apache.arvixe.com/storm',

manifests/thresh.pp

@@ -1,6 +1,21 @@
 #
 # Class to install monasca api server
 #
+# [*blobmirror*]
+#   location of the server to pull debians from
+#
+# [*kafka_brokers*]
+#   list of kafka brokers
+#
+# [*mon_thresh_build_ver*]
+#   version of the thresh debian package
+#
+# [*mon_thresh_deb*]
+#   name of the thresh debian package
+#
+# [*zookeeper_servers*]
+#   list of zookeeper servers
+#
 class monasca::thresh (
   $blobmirror           = undef,
   $kafka_brokers        = undef,
@@ -8,8 +23,8 @@ class monasca::thresh (
   $mon_thresh_deb       = undef,
   $zookeeper_servers    = undef,
 ) {
-  include monasca
-  include monasca::params
+  include ::monasca
+  include ::monasca::params
 
   # variables for the template
   $sql_host = $::monasca::params::sql_host

manifests/vertica/config.pp

@@ -1,10 +1,34 @@
 #
 # Class for vertica specific files
 #
+# === Parameters
+#
+# [*db_user*]
+#   name of the database user
+#
+# [*db_group*]
+#   name of the database group
+#
+# [*db_admin_password*]
+#   database admin password
+#
+# [*metrics_schema*]
+#   location of the metrics schema/projections file
+#
+# [*mon_api_password*]
+#   database api user password
+#
+# [*mon_persister_password*]
+#   database persister user password
+#
+# [*monitor_password*]
+#   database monitor user password
+#
 class monasca::vertica::config (
   $db_user                = 'dbadmin',
   $db_group               = 'verticadba',
   $db_admin_password      = unset,
+  $metrics_schema         = 'puppet:///modules/monasca/vertica/mon_metrics_schema.sql',
   $mon_api_password       = unset,
   $mon_persister_password = unset,
   $monitor_password       = unset,
@@ -15,7 +39,6 @@ class monasca::vertica::config (
   $install_dir = '/var/vertica'
   $alarms_schema = 'mon_alarms_schema.sql'
   $grants_schema = 'mon_grants.sql'
-  $metrics_schema = 'mon_metrics_schema.sql'
   $config_schema = 'mon_schema.sql'
   $users_schema = 'mon_users.sql'
   $cluster_script = 'create_mon_db_cluster.sh'
@@ -46,9 +69,9 @@ class monasca::vertica::config (
     require => File[$install_dir],
   }
 
-  file { "${install_dir}/${metrics_schema}":
+  file { "${install_dir}/mon_metrics_schema.sql":
     ensure  => file,
-    source  => "${files}/${metrics_schema}",
+    source  => $metrics_schema,
     mode    => '0644',
     owner   => $db_user,
     group   => $db_group,