update the mysql/galera and haproxy timeouts

Update the database timeouts to higher values to prevent the services
from disconnecting too frequently:
- set the Galera HAProxy timeout to 90 minutes (with parameter)
- set the database_idle_timeout to 5000 seconds (83 minutes) to allow
the services to drop the connection before HAProxy does
- add `on-marked-down shutdown-sessions` to Galera HAProxy balancer
members to allow shutting down sessions when a Galera host goes down.

Change-Id: I62c70c906a009df43acaf172eb27729d5d257b0f
This commit is contained in:
François Charlier 2015-04-08 17:12:09 +02:00
parent 735a760c40
commit 17684bd1e3
30 changed files with 291 additions and 212 deletions

View File

@ -36,6 +36,10 @@
# (optional) Password to connect to nova database
# Defaults to 'novapassword'
#
# [*nova_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*rabbit_hosts*]
# (optional) List of RabbitMQ servers. Should be an array.
# Defaults to ['127.0.0.1:5672']
@ -105,6 +109,7 @@ class cloud::compute(
$nova_db_use_slave = false,
$nova_db_user = 'nova',
$nova_db_password = 'novapassword',
$nova_db_idle_timeout = 5000,
$rabbit_hosts = ['127.0.0.1:5672'],
$rabbit_password = 'rabbitpassword',
$ks_glance_internal_host = '127.0.0.1',
@ -147,6 +152,7 @@ class cloud::compute(
class { 'nova':
database_connection => "mysql://${encoded_user}:${encoded_password}@${nova_db_host}/nova?charset=utf8",
database_idle_timeout => $nova_db_idle_timeout,
mysql_module => '2.2',
rabbit_userid => 'nova',
rabbit_hosts => $rabbit_hosts,

View File

@ -31,6 +31,10 @@
# (optional) Password to connect to trove database
# Defaults to 'trovepassword'
#
# [*trove_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*rabbit_hosts*]
# (optional) List of RabbitMQ servers. Should be an array.
# Defaults to ['127.0.0.1:5672']
@ -55,6 +59,7 @@ class cloud::database::dbaas(
$trove_db_host = '127.0.0.1',
$trove_db_user = 'trove',
$trove_db_password = 'trovepassword',
$trove_db_idle_timeout = 5000,
$rabbit_hosts = ['127.0.0.1:5672'],
$rabbit_password = 'rabbitpassword',
$nova_admin_username = 'trove',
@ -67,6 +72,7 @@ class cloud::database::dbaas(
class { 'trove':
database_connection => "mysql://${encoded_user}:${encoded_password}@${trove_db_host}/trove?charset=utf8",
database_idle_timeout => $trove_db_idle_timeout,
mysql_module => '2.2',
rabbit_hosts => $rabbit_hosts,
rabbit_password => $rabbit_password,

View File

@ -553,7 +553,7 @@ class cloud::database::sql::mysql (
ipaddresses => $api_eth,
ports => '3306',
options =>
inline_template('check inter 2000 rise 2 fall 5 port 8200 <% if @hostname != @galera_master_name -%>backup<% end %>')
inline_template('check inter 2000 rise 2 fall 5 port 8200 <% if @hostname != @galera_master_name -%>backup<% end %> on-marked-down shutdown-sessions')
}
@@haproxy::balancermember{"${::fqdn}-readonly":
@ -562,6 +562,6 @@ class cloud::database::sql::mysql (
ipaddresses => $api_eth,
ports => '3306',
options =>
inline_template('check inter 2000 rise 2 fall 5 port 8200 <% if @hostname == @galera_master_name -%>backup<% end %>')
inline_template('check inter 2000 rise 2 fall 5 port 8200 <% if @hostname == @galera_master_name -%>backup<% end %> on-marked-down shutdown-sessions')
}
}

View File

@ -35,6 +35,10 @@
# (optional) Password to connect to keystone database
# Defaults to 'keystonepassword'
#
# [*keystone_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*memcache_servers*]
# (optionnal) Memcached servers used by Keystone. Should be an array.
# Defaults to ['127.0.0.1:11211']
@ -421,6 +425,7 @@ class cloud::identity (
$keystone_db_host = '127.0.0.1',
$keystone_db_user = 'keystone',
$keystone_db_password = 'keystonepassword',
$keystone_db_idle_timeout = 5000,
$memcache_servers = ['127.0.0.1:11211'],
$ks_admin_email = 'no-reply@keystone.openstack',
$ks_admin_password = 'adminpassword',
@ -536,7 +541,7 @@ class cloud::identity (
admin_token => $ks_admin_token,
compute_port => $ks_nova_public_port,
debug => $debug,
database_idle_timeout => 60,
database_idle_timeout => $keystone_db_idle_timeout,
log_facility => $log_facility,
database_connection => "mysql://${encoded_user}:${encoded_password}@${keystone_db_host}/keystone?charset=utf8",
token_provider => 'keystone.token.providers.uuid.Provider',

View File

@ -31,6 +31,10 @@
# (optional) Password to connect to glance database
# Defaults to 'glancepassword'
#
# [*glance_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
# [*ks_keystone_internal_host*]
# (optional) Internal Hostname or IP to connect to Keystone API
# Defaults to '127.0.0.1'
@ -139,6 +143,7 @@ class cloud::image::api(
$glance_db_host = '127.0.0.1',
$glance_db_user = 'glance',
$glance_db_password = 'glancepassword',
$glance_db_idle_timeout = 5000,
$ks_keystone_internal_host = '127.0.0.1',
$ks_keystone_internal_proto = 'http',
$ks_glance_internal_host = '127.0.0.1',
@ -187,6 +192,7 @@ class cloud::image::api(
class { 'glance::api':
database_connection => "mysql://${encoded_glance_user}:${encoded_glance_password}@${glance_db_host}/glance?charset=utf8",
database_idle_timeout => $glance_db_idle_timeout,
mysql_module => '2.2',
registry_host => $openstack_vip,
registry_port => $ks_glance_registry_internal_port,

View File

@ -31,6 +31,10 @@
# (optional) Password to connect to glance database
# Defaults to 'glancepassword'
#
# [*glance_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults 5000
#
# [*ks_keystone_internal_host*]
# (optional) Internal Hostname or IP to connect to Keystone API
# Defaults to '127.0.0.1'
@ -80,6 +84,7 @@ class cloud::image::registry(
$glance_db_host = '127.0.0.1',
$glance_db_user = 'glance',
$glance_db_password = 'glancepassword',
$glance_db_idle_timeout = 5000,
$ks_keystone_internal_host = '127.0.0.1',
$ks_keystone_internal_proto = 'http',
$ks_glance_internal_host = '127.0.0.1',
@ -115,6 +120,7 @@ class cloud::image::registry(
class { 'glance::registry':
database_connection => "mysql://${encoded_glance_user}:${encoded_glance_password}@${glance_db_host}/glance?charset=utf8",
database_idle_timeout => $glance_db_idle_timeout,
mysql_module => '2.2',
verbose => $verbose,
debug => $debug,

View File

@ -478,6 +478,13 @@
# (optional) Port of redis service.
# Defaults to '6379'
#
# [*galera_timeout*]
# (optional) Timeout for galera connections
# Defaults to '90m'.
# Note: when changing this parameter you should also change the
# *_db_idle_timeout for all services to be a little less
# than this timeout.
#
# [*vip_public_ip*]
# (optional) Array or string for public VIP
# Should be part of keepalived_public_ips
@ -594,6 +601,7 @@ class cloud::loadbalancer(
$sensu_dashboard_port = 3000,
$sensu_api_port = 4568,
$redis_port = 6379,
$galera_timeout = '90m',
$vip_public_ip = ['127.0.0.1'],
$vip_internal_ip = false,
$vip_monitor_ip = false,
@ -945,8 +953,8 @@ class cloud::loadbalancer(
'mode' => 'tcp',
'balance' => 'roundrobin',
'option' => ['tcpka', 'tcplog', 'httpchk'], #httpchk mandatory expect 200 on port 9000
'timeout client' => '400s',
'timeout server' => '400s',
'timeout client' => $galera_timeout,
'timeout server' => $galera_timeout,
},
bind_options => $galera_bind_options,
}
@ -968,8 +976,8 @@ class cloud::loadbalancer(
'mode' => 'tcp',
'balance' => 'roundrobin',
'option' => ['tcpka', 'tcplog', 'httpchk'], #httpchk mandatory expect 200 on port 9000
'timeout client' => '400s',
'timeout server' => '400s',
'timeout client' => $galera_timeout,
'timeout server' => $galera_timeout,
},
bind_options => $galera_bind_options,
}

View File

@ -29,6 +29,10 @@
# (optional) Password that will be used for the neutron db user.
# Defaults to 'neutronpassword'
#
# [*neutron_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*ks_neutron_password*]
# (optional) Password used by Neutron to connect to Keystone API
# Defaults to 'neutronpassword'
@ -57,7 +61,6 @@
# (optional) Admin tenant name in Keystone
# Defaults to 'admin'
#
#
# [*nova_url*]
# (optional) URL for connection to nova (Only supports one nova region
# currently).
@ -160,6 +163,7 @@ class cloud::network::controller(
$neutron_db_host = '127.0.0.1',
$neutron_db_user = 'neutron',
$neutron_db_password = 'neutronpassword',
$neutron_db_idle_timeout = 5000,
$ks_neutron_password = 'neutronpassword',
$ks_keystone_admin_host = '127.0.0.1',
$ks_keystone_admin_proto = 'http',
@ -212,6 +216,7 @@ class cloud::network::controller(
auth_protocol => $ks_keystone_admin_proto,
auth_port => $ks_keystone_public_port,
database_connection => "mysql://${encoded_user}:${encoded_password}@${neutron_db_host}/neutron?charset=utf8",
database_idle_timeout => $neutron_db_idle_timeout,
mysql_module => '2.2',
api_workers => $::processorcount,
agent_down_time => '60',

View File

@ -67,6 +67,10 @@
# (optional) Password to connect to heat database
# Defaults to 'heatpassword'
#
# [*heat_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*rabbit_hosts*]
# (optional) List of RabbitMQ servers. Should be an array.
# Defaults to ['127.0.0.1:5672']
@ -108,6 +112,7 @@ class cloud::orchestration(
$heat_db_host = '127.0.0.1',
$heat_db_user = 'heat',
$heat_db_password = 'heatpassword',
$heat_db_idle_timeout = 5000,
$rabbit_hosts = ['127.0.0.1:5672'],
$rabbit_password = 'rabbitpassword',
$verbose = true,
@ -140,7 +145,8 @@ class cloud::orchestration(
keystone_password => $ks_heat_password,
auth_uri => "${ks_keystone_internal_proto}://${ks_keystone_internal_host}:${ks_keystone_internal_port}/v2.0",
keystone_ec2_uri => "${ks_keystone_internal_proto}://${ks_keystone_internal_host}:${ks_keystone_internal_port}/v2.0/ec2tokens",
sql_connection => "mysql://${encoded_user}:${encoded_password}@${heat_db_host}/heat?charset=utf8",
database_connection => "mysql://${encoded_user}:${encoded_password}@${heat_db_host}/heat?charset=utf8",
database_idle_timeout => $heat_db_idle_timeout,
mysql_module => '2.2',
rabbit_hosts => $rabbit_hosts,
rabbit_password => $rabbit_password,

View File

@ -32,6 +32,10 @@
# (optional) Cinder database password
# Defaults to 'cinderpassword'
#
# [*cinder_db_idle_timeout*]
# (optional) Timeout before idle SQL connections are reaped.
# Defaults to 5000
#
# [*rabbit_hosts*]
# (optional) List of RabbitMQ servers. Should be an array.
# Defaults to ['127.0.0.1:5672']
@ -68,6 +72,7 @@ class cloud::volume(
$cinder_db_host = '127.0.0.1',
$cinder_db_user = 'cinder',
$cinder_db_password = 'cinderpassword',
$cinder_db_idle_timeout = 5000,
$rabbit_hosts = ['127.0.0.1:5672'],
$rabbit_password = 'rabbitpassword',
$verbose = true,
@ -96,7 +101,8 @@ class cloud::volume(
class { 'cinder':
sql_connection => "mysql://${encoded_user}:${encoded_password}@${cinder_db_host}/cinder?charset=utf8",
database_connection => "mysql://${encoded_user}:${encoded_password}@${cinder_db_host}/cinder?charset=utf8",
database_idle_timeout => $cinder_db_idle_timeout,
mysql_module => '2.2',
rabbit_userid => 'cinder',
rabbit_hosts => $rabbit_hosts,

View File

@ -67,6 +67,7 @@ describe 'cloud::compute::api' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -55,6 +55,7 @@ describe 'cloud::compute::cert' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -55,6 +55,7 @@ describe 'cloud::compute::conductor' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -55,6 +55,7 @@ describe 'cloud::compute::consoleauth' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -59,6 +59,7 @@ describe 'cloud::compute::consoleproxy' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -91,6 +91,7 @@ describe 'cloud::compute::hypervisor' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false,
:nova_shell => '/bin/bash'

View File

@ -61,6 +61,7 @@ describe 'cloud::compute::scheduler' do
:rabbit_virtual_host => '/',
:memcached_servers => ['10.0.0.1','10.0.0.2'],
:database_connection => 'mysql://nova:secrete@10.0.0.1/nova?charset=utf8',
:database_idle_timeout => '5000',
:glance_api_servers => 'http://10.0.0.1:9292',
:log_dir => false
)

View File

@ -55,6 +55,7 @@ describe 'cloud::database::dbaas::api' do
:nova_proxy_admin_user => 'trove',
:nova_proxy_admin_tenant_name => 'services',
:database_connection => 'mysql://trove:secrete@10.0.0.1/trove?charset=utf8',
:database_idle_timeout => '5000',
)
end

View File

@ -53,6 +53,7 @@ describe 'cloud::database::dbaas::conductor' do
:nova_proxy_admin_user => 'trove',
:nova_proxy_admin_tenant_name => 'services',
:database_connection => 'mysql://trove:secrete@10.0.0.1/trove?charset=utf8',
:database_idle_timeout => '5000',
)
end

View File

@ -53,6 +53,7 @@ describe 'cloud::database::dbaas::taskmanager' do
:nova_proxy_admin_user => 'trove',
:nova_proxy_admin_tenant_name => 'services',
:database_connection => 'mysql://trove:secrete@10.0.0.1/trove?charset=utf8',
:database_idle_timeout => '5000',
)
end

View File

@ -124,9 +124,10 @@ describe 'cloud::identity' do
:compute_port => '8774',
:debug => true,
:verbose => true,
:database_idle_timeout => '60',
:database_idle_timeout => '5000',
:log_facility => 'LOG_LOCAL0',
:database_connection => 'mysql://keystone:secrete@10.0.0.1/keystone?charset=utf8',
:database_idle_timeout => '5000',
:token_driver => 'keystone.token.persistence.backends.sql.Token',
:token_provider => 'keystone.token.providers.uuid.Provider',
:use_syslog => true,

View File

@ -53,6 +53,7 @@ describe 'cloud::image::api' do
it 'configure glance-api' do
is_expected.to contain_class('glance::api').with(
:database_connection => 'mysql://glance:secrete@10.0.0.1/glance?charset=utf8',
:database_idle_timeout => '5000',
:keystone_password => 'secrete',
:registry_host => '10.0.0.42',
:registry_port => '9191',

View File

@ -41,6 +41,7 @@ describe 'cloud::image::registry' do
it 'configure glance-registry' do
is_expected.to contain_class('glance::registry').with(
:database_connection => 'mysql://glance:secrete@10.0.0.1/glance?charset=utf8',
:database_idle_timeout => '5000',
:keystone_password => 'secrete',
:keystone_tenant => 'services',
:keystone_user => 'glance',

View File

@ -233,8 +233,8 @@ describe 'cloud::loadbalancer' do
'mode' => 'tcp',
'balance' => 'roundrobin',
'option' => ['tcpka','tcplog','httpchk'],
'timeout client' => '400s',
'timeout server' => '400s'
'timeout client' => '90m',
'timeout server' => '90m'
}
)}
end # configure monitor haproxy listen
@ -255,8 +255,8 @@ describe 'cloud::loadbalancer' do
'mode' => 'tcp',
'balance' => 'roundrobin',
'option' => ['tcpka','tcplog','httpchk'],
'timeout client' => '400s',
'timeout server' => '400s'
'timeout client' => '90m',
'timeout server' => '90m'
}
)}
end # configure monitor haproxy listen

View File

@ -89,6 +89,7 @@ describe 'cloud::network::controller' do
:auth_port => '5000',
:auth_protocol => 'https',
:database_connection => 'mysql://neutron:secrete@10.0.0.1/neutron?charset=utf8',
:database_idle_timeout => '5000',
:api_workers => '2',
:agent_down_time => '60'
)

View File

@ -66,7 +66,8 @@ describe 'cloud::orchestration::api' do
:keystone_password => 'secrete',
:auth_uri => 'http://10.0.0.1:5000/v2.0',
:keystone_ec2_uri => 'http://10.0.0.1:5000/v2.0/ec2tokens',
:sql_connection => 'mysql://heat:secrete@10.0.0.1/heat?charset=utf8',
:database_connection => 'mysql://heat:secrete@10.0.0.1/heat?charset=utf8',
:database_idle_timeout => '5000',
:log_dir => false
)
end

View File

@ -70,7 +70,8 @@ describe 'cloud::orchestration::engine' do
:keystone_password => 'secrete',
:auth_uri => 'http://10.0.0.1:5000/v2.0',
:keystone_ec2_uri => 'http://10.0.0.1:5000/v2.0/ec2tokens',
:sql_connection => 'mysql://heat:secrete@10.0.0.1/heat?charset=utf8',
:database_connection => 'mysql://heat:secrete@10.0.0.1/heat?charset=utf8',
:database_idle_timeout => '5000',
:log_dir => false
)
is_expected.to contain_heat_config('clients/endpoint_type').with('value' => 'internalURL')

View File

@ -27,6 +27,7 @@ describe 'cloud::volume::api' do
cinder_db_host => '10.0.0.1',
cinder_db_user => 'cinder',
cinder_db_password => 'secrete',
cinder_db_idle_timeout => 5000,
rabbit_hosts => ['10.0.0.1'],
rabbit_password => 'secrete',
verbose => true,
@ -53,15 +54,17 @@ describe 'cloud::volume::api' do
it 'configure cinder common' do
is_expected.to contain_class('cinder').with(
:verbose => true,
:debug => true,
:database_connection => 'mysql://cinder:secrete@10.0.0.1/cinder?charset=utf8',
:database_idle_timeout => '5000',
:rabbit_userid => 'cinder',
:rabbit_hosts => ['10.0.0.1'],
:rabbit_password => 'secrete',
:rabbit_virtual_host => '/',
:verbose => true,
:debug => true,
:log_facility => 'LOG_LOCAL0',
:use_syslog => true,
:log_dir => false,
:use_syslog => true,
:storage_availability_zone => 'nova'
)
is_expected.to contain_class('cinder::ceilometer')

View File

@ -27,6 +27,7 @@ describe 'cloud::volume::scheduler' do
cinder_db_host => '10.0.0.1',
cinder_db_user => 'cinder',
cinder_db_password => 'secrete',
cinder_db_idle_timeout => 5000,
rabbit_hosts => ['10.0.0.1'],
rabbit_password => 'secrete',
verbose => true,
@ -43,15 +44,17 @@ describe 'cloud::volume::scheduler' do
it 'configure cinder common' do
is_expected.to contain_class('cinder').with(
:verbose => true,
:debug => true,
:database_connection => 'mysql://cinder:secrete@10.0.0.1/cinder?charset=utf8',
:database_idle_timeout => '5000',
:rabbit_userid => 'cinder',
:rabbit_hosts => ['10.0.0.1'],
:rabbit_password => 'secrete',
:rabbit_virtual_host => '/',
:verbose => true,
:debug => true,
:log_facility => 'LOG_LOCAL0',
:use_syslog => true,
:log_dir => false,
:use_syslog => true,
:storage_availability_zone => 'nova'
)
is_expected.to contain_class('cinder::ceilometer')

View File

@ -27,6 +27,7 @@ describe 'cloud::volume::storage' do
cinder_db_host => '10.0.0.1',
cinder_db_user => 'cinder',
cinder_db_password => 'secret',
cinder_db_idle_timeout => 5000,
rabbit_hosts => ['10.0.0.1'],
rabbit_password => 'secret',
verbose => true,
@ -104,15 +105,17 @@ describe 'cloud::volume::storage' do
it 'configure cinder common' do
is_expected.to contain_class('cinder').with(
:verbose => true,
:debug => true,
:database_connection => 'mysql://cinder:secret@10.0.0.1/cinder?charset=utf8',
:database_idle_timeout => '5000',
:rabbit_userid => 'cinder',
:rabbit_hosts => ['10.0.0.1'],
:rabbit_password => 'secret',
:rabbit_virtual_host => '/',
:verbose => true,
:debug => true,
:log_facility => 'LOG_LOCAL0',
:use_syslog => true,
:log_dir => false,
:use_syslog => true,
:storage_availability_zone => 'nova'
)