From ad1ef91aa4158c96356b13291605d8f2044f3069 Mon Sep 17 00:00:00 2001 From: Kamil Sambor Date: Thu, 17 Oct 2019 16:06:26 +0200 Subject: [PATCH] Add configurable monitor timeouts for ovn dbs Under pressure, the default monitor timeout value of 20 seconds is not enough to prevent unnecessary failovers of the ovn-dbs pacemaker resource. While spawning a few VMs in the same time this could lead to unnecessary movements of master DB, then re-connections of ovn-controllers (slaves are read-only), further peaks of load on DBs, and at the end it could lead to snowball effect. Now this value can be configurable by OVNDBSPacemakerTimeout which will configure tripleo::profile::pacemaker::ovn_dbs_bundle (default is set to 60s) Depends-On: https://review.opendev.org/#/c/689166/ Signed-off-by: Kamil Sambor Change-Id: I1afb5f2ef31ec61b3b224e5e1672fb9f12bcb110 --- deployment/ovn/ovn-dbs-pacemaker-puppet.yaml | 5 +++++ .../setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml | 11 +++++++++++ 2 files changed, 16 insertions(+) create mode 100644 releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml diff --git a/deployment/ovn/ovn-dbs-pacemaker-puppet.yaml b/deployment/ovn/ovn-dbs-pacemaker-puppet.yaml index e2f25e74b5..0a545583a2 100644 --- a/deployment/ovn/ovn-dbs-pacemaker-puppet.yaml +++ b/deployment/ovn/ovn-dbs-pacemaker-puppet.yaml @@ -68,6 +68,10 @@ parameters: type: string description: Specifies the default CA cert to use if TLS is used for services in the internal network. + OVNDBSPacemakerTimeout: + description: timeout for monitor of ovn dbs resource in seconds + type: number + default: 60 conditions: @@ -110,6 +114,7 @@ outputs: - tripleo::profile::pacemaker::ovn_dbs_bundle::nb_db_port: {get_param: OVNNorthboundServerPort} - tripleo::profile::pacemaker::ovn_dbs_bundle::sb_db_port: {get_param: OVNSouthboundServerPort} - tripleo::profile::pacemaker::ovn_dbs_bundle::container_backend: {get_param: ContainerCli} + - tripleo::profile::pacemaker::ovn_dbs_bundle::dbs_timeout: {get_param: OVNDBSPacemakerTimeout} - tripleo::haproxy::ovn_dbs_manage_lb: false - tripleo::ovn_dbs::firewall_rules: '121 OVN DB server ports': diff --git a/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml b/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml new file mode 100644 index 0000000000..b50d089e28 --- /dev/null +++ b/releasenotes/notes/setup_timeouts_ovn_dbs-141be475dd2cd7ae.yaml @@ -0,0 +1,11 @@ +--- +features: + - | + Under pressure, the default monitor timeout value of 20 seconds is not + enough to prevent unnecessary failovers of the ovn-dbs pacemaker resource. + While spawning a few VMs in the same time this could lead to unnecessary + movements of master DB, then re-connections of ovn-controllers (slaves are + read-only), further peaks of load on DBs, and at the end it could lead to + snowball effect. Now this value can be configurable by + OVNDBSPacemakerTimeout which will configure + tripleo::profile::pacemaker::ovn_dbs_bundle (default is set to 60s). \ No newline at end of file