From 461dbc3206f3504ce56f1486431bcb85a3b9d627 Mon Sep 17 00:00:00 2001
From: Rajesh Tailor <ratailor@redhat.com>
Date: Fri, 20 Jul 2018 17:25:54 +0530
Subject: [PATCH] Add sleep in nova cron jobs

If many DB cron jobs are executed at the same time, it will
trigger lock contention or DB spike.

Added sleep to induce random delay before running cronjobs to avoid
running all cron jobs at the same time on all hosts where this
job is configured.

Change-Id: I093a713a4f0ba48686de615aa8cb22b17a56917b
(cherry picked from commit 316836a08cb76a8995121e9f1e1859295e15c820)
---
 manifests/cron/archive_deleted_rows.pp        | 16 +++++++++++++-
 .../nova_cron_archive_deleted_rows_spec.rb    | 22 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/manifests/cron/archive_deleted_rows.pp b/manifests/cron/archive_deleted_rows.pp
index 207c45f44..3d381eb4d 100644
--- a/manifests/cron/archive_deleted_rows.pp
+++ b/manifests/cron/archive_deleted_rows.pp
@@ -54,6 +54,13 @@
 #    (optional) Adds --until-complete to the archive command
 #    Defaults to false.
 #
+#  [*maxdelay*]
+#    (optional) In Seconds. Should be a positive integer.
+#    Induces a random delay before running the cronjob to avoid running
+#    all cron jobs at the same time on all hosts this job is configured.
+#    Defaults to 0.
+#
+
 class nova::cron::archive_deleted_rows (
   $minute      = 1,
   $hour        = 0,
@@ -64,6 +71,7 @@ class nova::cron::archive_deleted_rows (
   $user        = undef,
   $destination = '/var/log/nova/nova-rowsflush.log',
   $until_complete = false,
+  $maxdelay       = 0,
 ) {
 
   include ::nova::deps
@@ -73,8 +81,14 @@ class nova::cron::archive_deleted_rows (
     $until_complete_real = '--until-complete'
   }
 
+  if $maxdelay == 0 {
+    $sleep = ''
+  } else {
+    $sleep = "sleep `expr \${RANDOM} \\% ${maxdelay}`; "
+  }
+
   cron { 'nova-manage db archive_deleted_rows':
-    command     => "nova-manage db archive_deleted_rows --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
+    command     => "${sleep}nova-manage db archive_deleted_rows --max_rows ${max_rows} ${until_complete_real} >>${destination} 2>&1",
     environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
     user        => pick($user, $::nova::params::nova_user),
     minute      => $minute,
diff --git a/spec/classes/nova_cron_archive_deleted_rows_spec.rb b/spec/classes/nova_cron_archive_deleted_rows_spec.rb
index ad09d66da..d779d8b09 100644
--- a/spec/classes/nova_cron_archive_deleted_rows_spec.rb
+++ b/spec/classes/nova_cron_archive_deleted_rows_spec.rb
@@ -57,4 +57,26 @@ describe 'nova::cron::archive_deleted_rows' do
       )
     end
   end
+
+  context 'cron with maxdelay' do
+    before :each do
+      params.merge!(
+        :maxdelay => 600
+      )
+    end
+
+    it 'configures a cron with maxdelay' do
+      is_expected.to contain_cron('nova-manage db archive_deleted_rows').with(
+        :command     => "sleep `expr ${RANDOM} \\% #{params[:maxdelay]}`; nova-manage db archive_deleted_rows --max_rows #{params[:max_rows]}  >>#{params[:destination]} 2>&1",
+        :environment => 'PATH=/bin:/usr/bin:/usr/sbin SHELL=/bin/sh',
+        :user        => params[:user],
+        :minute      => params[:minute],
+        :hour        => params[:hour],
+        :monthday    => params[:monthday],
+        :month       => params[:month],
+        :weekday     => params[:weekday],
+        :require     => 'Anchor[nova::dbsync::end]',
+      )
+    end
+  end
 end