From f1ac643ee13573a26c34837fb1f4fd3aebf2730e Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Thu, 17 Oct 2019 16:06:26 +0200 Subject: [PATCH] Add configurable monitor timeouts for ovn dbs Under pressure, the default monitor timeout value of 20 seconds is not enough to prevent unnecessary failovers of the ovn-dbs pacemaker resource. While spawning a few VMs in the same time this could lead to unnecessary movements of master DB, then re-connections of ovn-controllers (slaves are read-only), further peaks of load on DBs, and at the end it could lead to snowball effect. Now this value can be configurable by OVNDBSPacemakerTimeout which will configure tripleo::profile::pacemaker::ovn_dbs_bundle (default is set to 60s) Depends-On: https://review.opendev.org/#/c/692123/ Signed-off-by: Kamil Sambor Change-Id: I1afb5f2ef31ec61b3b224e5e1672fb9f12bcb110 (cherry picked from commit ad1ef91aa4158c96356b13291605d8f2044f3069) (cherry picked from commit ca335af7cba23a8159cf0d875fb87f37549ced53) (cherry picked from commit a429b0cf0b5065b925db26ed6558ff2128d0cbe3) (cherry picked from commit 298788bc781bf83b747907202d6482d15f9a01f0) --- docker/services/pacemaker/ovn-dbs.yaml | 5 +++++ .../setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml diff --git a/docker/services/pacemaker/ovn-dbs.yaml b/docker/services/pacemaker/ovn-dbs.yaml index c57c4f3af3..245ff1defd 100644 --- a/docker/services/pacemaker/ovn-dbs.yaml +++ b/docker/services/pacemaker/ovn-dbs.yaml @@ -54,6 +54,10 @@ parameters: description: > Setting this to a unique value will re-run any deployment tasks which perform configuration on a Heat stack-update. + OVNDBSPacemakerTimeout: + description: timeout for monitor of ovn dbs resource in seconds + type: number + default: 60 conditions: puppet_debug_enabled: {get_param: ConfigDebug} @@ -92,6 +96,7 @@ outputs: - 'pcmklatest' - tripleo::profile::pacemaker::ovn_dbs_bundle::nb_db_port: {get_param: OVNNorthboundServerPort} - tripleo::profile::pacemaker::ovn_dbs_bundle::sb_db_port: {get_param: OVNSouthboundServerPort} + - tripleo::profile::pacemaker::ovn_dbs_bundle::dbs_timeout: {get_param: OVNDBSPacemakerTimeout} logging_source: {get_attr: [OVNDbsBase, role_data, logging_source]} logging_groups: {get_attr: [OVNDbsBase, role_data, logging_groups]} service_config_settings: {get_attr: [OVNDbsBase, role_data, service_config_settings]} diff --git a/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml b/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml new file mode 100644 index 0000000000..b50d089e28 --- /dev/null +++ b/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Under pressure, the default monitor timeout value of 20 seconds is not + enough to prevent unnecessary failovers of the ovn-dbs pacemaker resource. + While spawning a few VMs in the same time this could lead to unnecessary + movements of master DB, then re-connections of ovn-controllers (slaves are + read-only), further peaks of load on DBs, and at the end it could lead to + snowball effect. Now this value can be configurable by + OVNDBSPacemakerTimeout which will configure + tripleo::profile::pacemaker::ovn_dbs_bundle (default is set to 60s). \ No newline at end of file