Tune mysql and galera backend settings on scale

- Tune MySQL config settings to be more clouds-specific (reads much
  more common, reads:writes rate is 10(100):1)
  * Limit innodb_buffer_pool_size to 10G and set is as 30% of RAM
  * Update innodb_log_file_size as:
    * If mysql ib_logfiles do not match 25% of innodb_buffer_pool_size (max 2G),
      delete them and update innodb_log_file_size in the config appropriately;
    * Otherwise, use existing ib_logfile0 size to re-evaluate the
      innodb_buffer_pool_size config value to match its real size.
  * Reduce innodb_write_io_threads to 2/2 from 2/8
  * Use innodb_flush_log_at_trx_commit=2 for HA (was 1)
  * Raise FD imits to 102400/102400 from 22058/22058,
  * Raise max connections to 3000/4096 from 2048/2048,
  * Raise open tables limits 10000/10000 from 10000/400.
  * Lower wait_timeout to 3600/3600 (1h) from 28800/28800 (8h)
  * Adjust myisam_sort_buffer_size to 64M from 8388608 bytes
  * Adjust key_buffer_size to 64M from 16777216 bytes
- Sync Puppet galera module params for MySQL module as well
Note: doubled values should be treated as simple/HA case
CI retest

Partial blueprint: optimize-db-performance-at-scale
Change-Id: Ie341da55711e70e38116389a54bda3b4357dafe7
Signed-off-by: Bogdan Dobrelya <bdobrelia@mirantis.com>

deployment/puppet/galera/lib/facter/mysql_log_file_size_real.rb

@@ -0,0 +1,15 @@
+# Fact: mysql_log_file_size_real
+#
+# Purpose: Return size (M) of ib_logfile0, if exists
+#
+require 'facter'
+Facter.add(:mysql_log_file_size_real) do
+  setcode do
+    f = '/var/lib/mysql/ib_logfile0'
+    if File.exists?(f)
+      (File.size(f).to_f / 1048576).round.to_s + 'M' rescue '0'
+    else
+      '0'
+    end
+  end
+end

deployment/puppet/galera/manifests/init.pp

@@ -67,6 +67,15 @@ class galera (
   $mysql_password = $::galera::params::mysql_password
   $libgalera_prefix = $::galera::params::libgalera_prefix
   $mysql_buffer_pool_size = $::galera::params::mysql_buffer_pool_size
+  $mysql_log_file_size = $::galera::params::mysql_log_file_size
+  $max_connections = $::galera::params::max_connections
+  $table_open_cache = $::galera::params::table_open_cache
+  $key_buffer_size = $::galera::params::key_buffer_size
+  $myisam_sort_buffer_size = $::galera::params::myisam_sort_buffer_size
+  $wait_timeout = $::galera::params::wait_timeout
+  $open_files_limit= $::galera::params::open_files_limit
+  $innodb_flush_log_at_trx_commit=$::galera::params::innodb_flush_log_at_trx_commit
+  $datadir=$::mysql::params::datadir
 
   case $::osfamily {
     'RedHat' : {
@@ -238,6 +247,22 @@ class galera (
   file { ["/etc/mysql", "/etc/mysql/conf.d"]: ensure => directory, }
 
   if $::galera_gcomm_empty == "true" {
+    #FIXME(bogdando): dirtyhack to pervert imperative puppet nature.
+    if $::mysql_log_file_size_real != $mysql_log_file_size {
+      # delete MySQL ib_logfiles, if log file size does not match the one
+      # from params
+      exec { 'delete_logfiles':
+        command     => "rm -f ${datadir}/ib_logfile* || true",
+        path        => [ '/sbin/', '/usr/sbin/', '/usr/bin/' ,'/bin/' ],
+        before      => File['/etc/mysql/conf.d/wsrep.cnf'],
+      }
+      # use predefined value for log file size
+      $innodb_log_file_size_real = $mysql_log_file_size
+    } else {
+      # evaluate existing log file size and use it as a value
+      $innodb_log_file_size_real = $::mysql_log_file_size_real
+    }
+
     file { "/etc/mysql/conf.d/wsrep.cnf":
       ensure  => present,
       content => template("galera/wsrep.cnf.erb"),

deployment/puppet/galera/manifests/params.pp

@@ -17,9 +17,22 @@
 # should be considered to be constant
 class galera::params {
 
-  $mysql_user             = "wsrep_sst"
+  $mysql_user             = 'wsrep_sst'
-  $mysql_password         = "password"
+  $mysql_password         = 'password'
-  $mysql_buffer_pool_size = inline_template("<%= ($::memorysize_mb * 0.3).floor %>M")
+  #TODO(bogdando) remove code duplication for galera and mysql manifests to openstack::db in 'I' release
+  #limit buffer size to 10G
+  $buffer_size             =
+    inline_template("<%= [(${::memorysize_mb} * 0.3 + 0).floor, 10000].min %>")
+  $mysql_buffer_pool_size  =  "${buffer_size}M"
+  $mysql_log_file_size     =
+    inline_template("<%= [(${buffer_size} * 0.25 + 0).floor, 2047].min %>M")
+  $wait_timeout            = '3600'
+  $myisam_sort_buffer_size = '64M'
+  $key_buffer_size         = '64M'
+  $table_open_cache        = '10000'
+  $open_files_limit        = '102400'
+  $max_connections         = '4096'
+  $innodb_flush_log_at_trx_commit = '2'
 
   case $::osfamily {
     'RedHat': {

deployment/puppet/galera/templates/wsrep.cnf.erb

@@ -33,7 +33,7 @@ query_cache_type=0
 bind-address=<%= @node_address %>
 port=3307
 
-max_connections=2048
+max_connections=<%= max_connections %>
 
 ##
 ## WSREP options
@@ -134,11 +134,23 @@ skip-name-resolve
 <% end -%>
 <% if @mysql_buffer_pool_size %>
 innodb_buffer_pool_size=<%= @mysql_buffer_pool_size %>
+<% if @innodb_log_file_size_real != '0' %>
+innodb_log_file_size=<%= @innodb_log_file_size_real %>
+<% end %>
 <% end %>
 innodb_thread_concurrency=0
-innodb_write_io_threads=8
+innodb_write_io_threads=2
 innodb_read_io_threads=8
 innodb_io_capacity=500
-table_cache=10000
 innodb_file_per_table=1
+<% if @operatingsystem != 'RedHat' %>
+table_open_cache=<%= @table_open_cache %>
 innodb_file_format=Barracuda
+<% else %>
+table_cache=<%= @table_open_cache %>
+<% end %>
+key_buffer_size=<%= @key_buffer_size %>
+myisam_sort_buffer_size=<%= @myisam_sort_buffer_size %>
+wait_timeout=<%= @wait_timeout %>
+open_files_limit=<%= @open_files_limit %>
+innodb_flush_log_at_trx_commit=<%= @innodb_flush_log_at_trx_commit %>

deployment/puppet/mysql/manifests/config.pp

@@ -52,6 +52,15 @@ class mysql::config(
   $server_id         = $mysql::params::server_id,
 ) inherits mysql::params {
 
+  $mysql_buffer_pool_size = $::mysql::params::mysql_buffer_pool_size
+  $mysql_log_file_size    = $::mysql::params::mysql_log_file_size
+  $max_connections = $::mysql::params::max_connections
+  $table_open_cache = $::mysql::params::table_open_cache
+  $key_buffer_size = $::mysql::params::key_buffer_size
+  $myisam_sort_buffer_size = $::mysql::params::myisam_sort_buffer_size
+  $wait_timeout = $::mysql::params::wait_timeout
+  $open_files_limit= $::mysql::params::open_files_limit
+
   if $custom_setup_class != "pacemaker_mysql" {
     File {
       owner  => 'root',
@@ -128,6 +137,22 @@ class mysql::config(
     mode   => '0755',
   }
 
+  #FIXME(bogdando): dirtyhack to pervert imperative puppet nature.
+  if $::mysql_log_file_size_real != $mysql_log_file_size {
+    # delete MySQL ib_logfiles, if log file size does not match the one
+    # from params
+    exec { 'delete_logfiles':
+      command     => "rm -f ${datadir}/ib_logfile* || true",
+      path        => [ '/sbin/', '/usr/sbin/', '/usr/bin/' ,'/bin/' ],
+      before      => File[$config_file],
+    }
+    # use predefined value for log file size
+    $innodb_log_file_size_real = $mysql_log_file_size
+  } else {
+    # evaluate existing log file size and use it as a value
+    $innodb_log_file_size_real = $::mysql_log_file_size_real
+  }
+
   file { $config_file:
     content => template('mysql/my.cnf.erb'),
     mode    => '0644',

deployment/puppet/mysql/manifests/params.pp

@@ -16,10 +16,21 @@ class mysql::params {
   $port                = 3306
   $etc_root_password   = false
   $ssl                 = false
-  $server_id           = delete(delete(delete("$::hostname",'controller-'),'fuel-'),"node-")
+  $server_id           = delete(delete(delete("${::hostname}",'controller-'),'fuel-'),"node-")
   $service_provider = undef
+  #TODO(bogdando) remove code duplication for galera and mysql manifests to openstack::db in 'I' release
   #Set buffer pool size to 30% of memory, but not greater than 10G
-  $mysql_buffer_pool_size = inline_template("<%= [($::memorysize_mb * 0.3 + 0).floor, 10000].min %>M")
+  $buffer_size             =
+    inline_template("<%= [(${::memorysize_mb} * 0.3 + 0).floor, 10000].min %>")
+  $mysql_buffer_pool_size  =  "${buffer_size}M"
+  $mysql_log_file_size     =
+    inline_template("<%= [(${buffer_size} * 0.25 + 0).floor, 2047].min %>M")
+  $wait_timeout            = '3600'
+  $myisam_sort_buffer_size = '64M'
+  $key_buffer_size         = '64M'
+  $table_open_cache        = '10000'
+  $open_files_limit        = '102400'
+  $max_connections         = '3000'
 
   case $::osfamily {
     'RedHat': {

deployment/puppet/mysql/manifests/server.pp

@@ -32,9 +32,7 @@ class mysql::server (
   $rep_pass = 'replicant666',
   $replication_roles = "SELECT, PROCESS, FILE, SUPER, REPLICATION CLIENT, REPLICATION SLAVE, RELOAD",
   $use_syslog              = false,
-  $initscript_file         = 'puppet:///modules/mysql/mysql-single.init',
+  $initscript_file         = 'puppet:///modules/mysql/mysql-single.init'
-  $mysql_buffer_pool_size  = $::mysql::params::mysql_buffer_pool_size
-
 ) inherits mysql::params {
 
   Exec {path => '/usr/bin:/bin:/usr/sbin:/sbin'}

deployment/puppet/mysql/templates/my.cnf.erb

@@ -10,8 +10,11 @@ syslog
 [mysqld]
 innodb_file_per_table=1
 <% if @operatingsystem != 'RedHat' %>
+table_open_cache=<%= @table_open_cache %>
 innodb_file_format=Barracuda
-<% end -%>
+<% else %>
+table_cache=<%= @table_open_cache %>
+<% end %>
 user    = mysql
 #pid-file  = <%= @pidfile %>
 socket    = <%= @socket %>
@@ -28,6 +31,20 @@ thread_cache_size  = 8
 myisam-recover     = BACKUP
 query_cache_limit  = 1M
 query_cache_size   = 16M
+<% if @mysql_buffer_pool_size %>
+innodb_buffer_pool_size=<%= @mysql_buffer_pool_size %>
+<% if @innodb_log_file_size_real != '0' %>
+innodb_log_file_size=<%= @innodb_log_file_size_real %>
+<% end %>
+<% end %>
+innodb_thread_concurrency= 0
+innodb_write_io_threads  = 2
+innodb_read_io_threads   = 8
+innodb_io_capacity       = 500
+key_buffer_size          = <%= @key_buffer_size %>
+myisam_sort_buffer_size  = <%= @myisam_sort_buffer_size %>
+wait_timeout             = <%= @wait_timeout %>
+open_files_limit         = <%= @open_files_limit %>
 <% if @use_syslog -%>
 #log_error          = syslog:local1
 <% else -%>
@@ -35,14 +52,7 @@ log_error          = <%= @log_error %>
 <% end -%>
 expire_logs_days   = 10
 max_binlog_size    = 100M
-max_connections    = 2048
+max_connections    =  <%= @max_connections %>
-table_open_cache   = 10000
-wait_timeout       = 30
-innodb_buffer_pool_size   = <%= @mysql_buffer_pool_size %>
-innodb_thread_concurrency = 0
-innodb_write_io_threads   = 2
-innodb_read_io_threads    = 8
-innodb_io_capacity        = 500
 <% if @default_engine != 'UNSET' %>
 default-storage-engine = <%= @default_engine %>
 <% end %>