From 24c87cd7bb20b59388c72cf024ebee27564cbe58 Mon Sep 17 00:00:00 2001 From: Bogdan Dobrelya Date: Fri, 14 Feb 2014 17:11:36 +0200 Subject: [PATCH] Tune mysql and galera backend settings on scale - Tune MySQL config settings to be more clouds-specific (reads much more common, reads:writes rate is 10(100):1) * Limit innodb_buffer_pool_size to 10G and set is as 30% of RAM * Update innodb_log_file_size as: * If mysql ib_logfiles do not match 25% of innodb_buffer_pool_size (max 2G), delete them and update innodb_log_file_size in the config appropriately; * Otherwise, use existing ib_logfile0 size to re-evaluate the innodb_buffer_pool_size config value to match its real size. * Reduce innodb_write_io_threads to 2/2 from 2/8 * Use innodb_flush_log_at_trx_commit=2 for HA (was 1) * Raise FD imits to 102400/102400 from 22058/22058, * Raise max connections to 3000/4096 from 2048/2048, * Raise open tables limits 10000/10000 from 10000/400. * Lower wait_timeout to 3600/3600 (1h) from 28800/28800 (8h) * Adjust myisam_sort_buffer_size to 64M from 8388608 bytes * Adjust key_buffer_size to 64M from 16777216 bytes - Sync Puppet galera module params for MySQL module as well Note: doubled values should be treated as simple/HA case CI retest Partial blueprint: optimize-db-performance-at-scale Change-Id: Ie341da55711e70e38116389a54bda3b4357dafe7 Signed-off-by: Bogdan Dobrelya --- .../lib/facter/mysql_log_file_size_real.rb | 15 ++++++++++ deployment/puppet/galera/manifests/init.pp | 25 +++++++++++++++++ deployment/puppet/galera/manifests/params.pp | 19 +++++++++++-- .../puppet/galera/templates/wsrep.cnf.erb | 18 ++++++++++-- deployment/puppet/mysql/manifests/config.pp | 27 +++++++++++++++++- deployment/puppet/mysql/manifests/params.pp | 15 ++++++++-- deployment/puppet/mysql/manifests/server.pp | 4 +-- deployment/puppet/mysql/templates/my.cnf.erb | 28 +++++++++++++------ 8 files changed, 130 insertions(+), 21 deletions(-) create mode 100644 deployment/puppet/galera/lib/facter/mysql_log_file_size_real.rb diff --git a/deployment/puppet/galera/lib/facter/mysql_log_file_size_real.rb b/deployment/puppet/galera/lib/facter/mysql_log_file_size_real.rb new file mode 100644 index 0000000000..dee05b80c6 --- /dev/null +++ b/deployment/puppet/galera/lib/facter/mysql_log_file_size_real.rb @@ -0,0 +1,15 @@ +# Fact: mysql_log_file_size_real +# +# Purpose: Return size (M) of ib_logfile0, if exists +# +require 'facter' +Facter.add(:mysql_log_file_size_real) do + setcode do + f = '/var/lib/mysql/ib_logfile0' + if File.exists?(f) + (File.size(f).to_f / 1048576).round.to_s + 'M' rescue '0' + else + '0' + end + end +end diff --git a/deployment/puppet/galera/manifests/init.pp b/deployment/puppet/galera/manifests/init.pp index 6f466bea13..cc057fe35c 100644 --- a/deployment/puppet/galera/manifests/init.pp +++ b/deployment/puppet/galera/manifests/init.pp @@ -67,6 +67,15 @@ class galera ( $mysql_password = $::galera::params::mysql_password $libgalera_prefix = $::galera::params::libgalera_prefix $mysql_buffer_pool_size = $::galera::params::mysql_buffer_pool_size + $mysql_log_file_size = $::galera::params::mysql_log_file_size + $max_connections = $::galera::params::max_connections + $table_open_cache = $::galera::params::table_open_cache + $key_buffer_size = $::galera::params::key_buffer_size + $myisam_sort_buffer_size = $::galera::params::myisam_sort_buffer_size + $wait_timeout = $::galera::params::wait_timeout + $open_files_limit= $::galera::params::open_files_limit + $innodb_flush_log_at_trx_commit=$::galera::params::innodb_flush_log_at_trx_commit + $datadir=$::mysql::params::datadir case $::osfamily { 'RedHat' : { @@ -238,6 +247,22 @@ class galera ( file { ["/etc/mysql", "/etc/mysql/conf.d"]: ensure => directory, } if $::galera_gcomm_empty == "true" { + #FIXME(bogdando): dirtyhack to pervert imperative puppet nature. + if $::mysql_log_file_size_real != $mysql_log_file_size { + # delete MySQL ib_logfiles, if log file size does not match the one + # from params + exec { 'delete_logfiles': + command => "rm -f ${datadir}/ib_logfile* || true", + path => [ '/sbin/', '/usr/sbin/', '/usr/bin/' ,'/bin/' ], + before => File['/etc/mysql/conf.d/wsrep.cnf'], + } + # use predefined value for log file size + $innodb_log_file_size_real = $mysql_log_file_size + } else { + # evaluate existing log file size and use it as a value + $innodb_log_file_size_real = $::mysql_log_file_size_real + } + file { "/etc/mysql/conf.d/wsrep.cnf": ensure => present, content => template("galera/wsrep.cnf.erb"), diff --git a/deployment/puppet/galera/manifests/params.pp b/deployment/puppet/galera/manifests/params.pp index e463f53326..e563e1455f 100644 --- a/deployment/puppet/galera/manifests/params.pp +++ b/deployment/puppet/galera/manifests/params.pp @@ -17,9 +17,22 @@ # should be considered to be constant class galera::params { - $mysql_user = "wsrep_sst" - $mysql_password = "password" - $mysql_buffer_pool_size = inline_template("<%= ($::memorysize_mb * 0.3).floor %>M") + $mysql_user = 'wsrep_sst' + $mysql_password = 'password' + #TODO(bogdando) remove code duplication for galera and mysql manifests to openstack::db in 'I' release + #limit buffer size to 10G + $buffer_size = + inline_template("<%= [(${::memorysize_mb} * 0.3 + 0).floor, 10000].min %>") + $mysql_buffer_pool_size = "${buffer_size}M" + $mysql_log_file_size = + inline_template("<%= [(${buffer_size} * 0.25 + 0).floor, 2047].min %>M") + $wait_timeout = '3600' + $myisam_sort_buffer_size = '64M' + $key_buffer_size = '64M' + $table_open_cache = '10000' + $open_files_limit = '102400' + $max_connections = '4096' + $innodb_flush_log_at_trx_commit = '2' case $::osfamily { 'RedHat': { diff --git a/deployment/puppet/galera/templates/wsrep.cnf.erb b/deployment/puppet/galera/templates/wsrep.cnf.erb index 5e2927a9a0..a774d7b28a 100644 --- a/deployment/puppet/galera/templates/wsrep.cnf.erb +++ b/deployment/puppet/galera/templates/wsrep.cnf.erb @@ -33,7 +33,7 @@ query_cache_type=0 bind-address=<%= @node_address %> port=3307 -max_connections=2048 +max_connections=<%= max_connections %> ## ## WSREP options @@ -134,11 +134,23 @@ skip-name-resolve <% end -%> <% if @mysql_buffer_pool_size %> innodb_buffer_pool_size=<%= @mysql_buffer_pool_size %> +<% if @innodb_log_file_size_real != '0' %> +innodb_log_file_size=<%= @innodb_log_file_size_real %> +<% end %> <% end %> innodb_thread_concurrency=0 -innodb_write_io_threads=8 +innodb_write_io_threads=2 innodb_read_io_threads=8 innodb_io_capacity=500 -table_cache=10000 innodb_file_per_table=1 +<% if @operatingsystem != 'RedHat' %> +table_open_cache=<%= @table_open_cache %> innodb_file_format=Barracuda +<% else %> +table_cache=<%= @table_open_cache %> +<% end %> +key_buffer_size=<%= @key_buffer_size %> +myisam_sort_buffer_size=<%= @myisam_sort_buffer_size %> +wait_timeout=<%= @wait_timeout %> +open_files_limit=<%= @open_files_limit %> +innodb_flush_log_at_trx_commit=<%= @innodb_flush_log_at_trx_commit %> diff --git a/deployment/puppet/mysql/manifests/config.pp b/deployment/puppet/mysql/manifests/config.pp index 655bb1dd1c..ca7b2312c3 100644 --- a/deployment/puppet/mysql/manifests/config.pp +++ b/deployment/puppet/mysql/manifests/config.pp @@ -52,6 +52,15 @@ class mysql::config( $server_id = $mysql::params::server_id, ) inherits mysql::params { + $mysql_buffer_pool_size = $::mysql::params::mysql_buffer_pool_size + $mysql_log_file_size = $::mysql::params::mysql_log_file_size + $max_connections = $::mysql::params::max_connections + $table_open_cache = $::mysql::params::table_open_cache + $key_buffer_size = $::mysql::params::key_buffer_size + $myisam_sort_buffer_size = $::mysql::params::myisam_sort_buffer_size + $wait_timeout = $::mysql::params::wait_timeout + $open_files_limit= $::mysql::params::open_files_limit + if $custom_setup_class != "pacemaker_mysql" { File { owner => 'root', @@ -66,7 +75,7 @@ class mysql::config( mode => '0400', notify => Service['mysql'], } - } + } if $ssl and $ssl_ca == undef { fail('The ssl_ca parameter is required when ssl is true') @@ -128,6 +137,22 @@ class mysql::config( mode => '0755', } + #FIXME(bogdando): dirtyhack to pervert imperative puppet nature. + if $::mysql_log_file_size_real != $mysql_log_file_size { + # delete MySQL ib_logfiles, if log file size does not match the one + # from params + exec { 'delete_logfiles': + command => "rm -f ${datadir}/ib_logfile* || true", + path => [ '/sbin/', '/usr/sbin/', '/usr/bin/' ,'/bin/' ], + before => File[$config_file], + } + # use predefined value for log file size + $innodb_log_file_size_real = $mysql_log_file_size + } else { + # evaluate existing log file size and use it as a value + $innodb_log_file_size_real = $::mysql_log_file_size_real + } + file { $config_file: content => template('mysql/my.cnf.erb'), mode => '0644', diff --git a/deployment/puppet/mysql/manifests/params.pp b/deployment/puppet/mysql/manifests/params.pp index 1c3959ede6..0ef4e1618b 100644 --- a/deployment/puppet/mysql/manifests/params.pp +++ b/deployment/puppet/mysql/manifests/params.pp @@ -16,10 +16,21 @@ class mysql::params { $port = 3306 $etc_root_password = false $ssl = false - $server_id = delete(delete(delete("$::hostname",'controller-'),'fuel-'),"node-") + $server_id = delete(delete(delete("${::hostname}",'controller-'),'fuel-'),"node-") $service_provider = undef + #TODO(bogdando) remove code duplication for galera and mysql manifests to openstack::db in 'I' release #Set buffer pool size to 30% of memory, but not greater than 10G - $mysql_buffer_pool_size = inline_template("<%= [($::memorysize_mb * 0.3 + 0).floor, 10000].min %>M") + $buffer_size = + inline_template("<%= [(${::memorysize_mb} * 0.3 + 0).floor, 10000].min %>") + $mysql_buffer_pool_size = "${buffer_size}M" + $mysql_log_file_size = + inline_template("<%= [(${buffer_size} * 0.25 + 0).floor, 2047].min %>M") + $wait_timeout = '3600' + $myisam_sort_buffer_size = '64M' + $key_buffer_size = '64M' + $table_open_cache = '10000' + $open_files_limit = '102400' + $max_connections = '3000' case $::osfamily { 'RedHat': { diff --git a/deployment/puppet/mysql/manifests/server.pp b/deployment/puppet/mysql/manifests/server.pp index ebbf38b46f..29b6b4d249 100644 --- a/deployment/puppet/mysql/manifests/server.pp +++ b/deployment/puppet/mysql/manifests/server.pp @@ -32,9 +32,7 @@ class mysql::server ( $rep_pass = 'replicant666', $replication_roles = "SELECT, PROCESS, FILE, SUPER, REPLICATION CLIENT, REPLICATION SLAVE, RELOAD", $use_syslog = false, - $initscript_file = 'puppet:///modules/mysql/mysql-single.init', - $mysql_buffer_pool_size = $::mysql::params::mysql_buffer_pool_size - + $initscript_file = 'puppet:///modules/mysql/mysql-single.init' ) inherits mysql::params { Exec {path => '/usr/bin:/bin:/usr/sbin:/sbin'} diff --git a/deployment/puppet/mysql/templates/my.cnf.erb b/deployment/puppet/mysql/templates/my.cnf.erb index 8e456f5411..dd0bfe4e57 100644 --- a/deployment/puppet/mysql/templates/my.cnf.erb +++ b/deployment/puppet/mysql/templates/my.cnf.erb @@ -10,8 +10,11 @@ syslog [mysqld] innodb_file_per_table=1 <% if @operatingsystem != 'RedHat' %> +table_open_cache=<%= @table_open_cache %> innodb_file_format=Barracuda -<% end -%> +<% else %> +table_cache=<%= @table_open_cache %> +<% end %> user = mysql #pid-file = <%= @pidfile %> socket = <%= @socket %> @@ -28,6 +31,20 @@ thread_cache_size = 8 myisam-recover = BACKUP query_cache_limit = 1M query_cache_size = 16M +<% if @mysql_buffer_pool_size %> +innodb_buffer_pool_size=<%= @mysql_buffer_pool_size %> +<% if @innodb_log_file_size_real != '0' %> +innodb_log_file_size=<%= @innodb_log_file_size_real %> +<% end %> +<% end %> +innodb_thread_concurrency= 0 +innodb_write_io_threads = 2 +innodb_read_io_threads = 8 +innodb_io_capacity = 500 +key_buffer_size = <%= @key_buffer_size %> +myisam_sort_buffer_size = <%= @myisam_sort_buffer_size %> +wait_timeout = <%= @wait_timeout %> +open_files_limit = <%= @open_files_limit %> <% if @use_syslog -%> #log_error = syslog:local1 <% else -%> @@ -35,14 +52,7 @@ log_error = <%= @log_error %> <% end -%> expire_logs_days = 10 max_binlog_size = 100M -max_connections = 2048 -table_open_cache = 10000 -wait_timeout = 30 -innodb_buffer_pool_size = <%= @mysql_buffer_pool_size %> -innodb_thread_concurrency = 0 -innodb_write_io_threads = 2 -innodb_read_io_threads = 8 -innodb_io_capacity = 500 +max_connections = <%= @max_connections %> <% if @default_engine != 'UNSET' %> default-storage-engine = <%= @default_engine %> <% end %>