From b02c4111666d367791556c2d167b5f1415df2d8c Mon Sep 17 00:00:00 2001
From: Monty Taylor <mordred@inaugust.com>
Date: Wed, 31 Aug 2016 09:33:56 -0500
Subject: [PATCH] Run puppet on infracloud in a different cron

It takes too long to run puppet on infracloud and it's blocking our
other servers.

Change-Id: I7202617acc5a04e18672b217db53510167d597bd
---
 .../manifests/puppetmaster.pp                 | 25 ++++++++++++++
 run_all.sh                                    |  1 -
 run_infracloud.sh                             | 33 +++++++++++++++++++
 3 files changed, 58 insertions(+), 1 deletion(-)
 create mode 100755 run_infracloud.sh

diff --git a/modules/openstack_project/manifests/puppetmaster.pp b/modules/openstack_project/manifests/puppetmaster.pp
index c1acbc1ff1..d4ef838f68 100644
--- a/modules/openstack_project/manifests/puppetmaster.pp
+++ b/modules/openstack_project/manifests/puppetmaster.pp
@@ -39,6 +39,17 @@ class openstack_project::puppetmaster (
     environment => 'PATH=/var/lib/gems/1.8/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
   }
 
+  cron { 'updateinfracloud':
+    user        => 'root',
+    minute      => $puppetmaster_update_cron_interval[min],
+    hour        => $puppetmaster_update_cron_interval[hour],
+    monthday    => $puppetmaster_update_cron_interval[day],
+    month       => $puppetmaster_update_cron_interval[month],
+    weekday     => $puppetmaster_update_cron_interval[weekday],
+    command     => 'flock -n /var/run/puppet/puppet_run_infracloud.lock bash /opt/system-config/production/run_infracloud.sh >> /var/log/puppet_run_infracloud_cron.log 2>&1',
+    environment => 'PATH=/var/lib/gems/1.8/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
+  }
+
   logrotate::file { 'updatepuppetmaster':
     ensure  => present,
     log     => '/var/log/puppet_run_all.log',
@@ -67,6 +78,20 @@ class openstack_project::puppetmaster (
     require => Cron['updatepuppetmaster'],
   }
 
+  logrotate::file { 'updateinfracloudcron':
+    ensure  => present,
+    log     => '/var/log/puppet_run_infracloud_cron.log',
+    options => ['compress',
+      'copytruncate',
+      'delaycompress',
+      'missingok',
+      'rotate 7',
+      'daily',
+      'notifempty',
+    ],
+    require => Cron['updateinfracloud'],
+  }
+
   cron { 'deleteoldreports':
     user        => 'root',
     hour        => '3',
diff --git a/run_all.sh b/run_all.sh
index 9d1cd86775..f6f8b19cc5 100755
--- a/run_all.sh
+++ b/run_all.sh
@@ -37,6 +37,5 @@ timeout -k 2m 120m ansible-playbook -f 10 ${ANSIBLE_PLAYBOOKS}/remote_puppet_git
 # Run AFS changes separately so we can make sure to only do one at a time
 # (turns out quorum is nice to have)
 timeout -k 2m 120m ansible-playbook -f 1 ${ANSIBLE_PLAYBOOKS}/remote_puppet_afs.yaml
-timeout -k 2m 120m ansible-playbook -f 10 ${ANSIBLE_PLAYBOOKS}/remote_puppet_infracloud.yaml
 # Run everything else. We do not care if the other things worked
 timeout -k 2m 120m ansible-playbook -f 10 ${ANSIBLE_PLAYBOOKS}/remote_puppet_else.yaml
diff --git a/run_infracloud.sh b/run_infracloud.sh
new file mode 100755
index 0000000000..7626684a79
--- /dev/null
+++ b/run_infracloud.sh
@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Copyright 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# If updating the puppet system-config repo or installing puppet modules
+# fails then abort the puppet run as we will not get the results we
+# expect.
+set -e
+export ANSIBLE_LOG_PATH=/var/log/puppet_run_all_infracloud.log
+SYSTEM_CONFIG=/opt/system-config/production
+ANSIBLE_PLAYBOOKS=$SYSTEM_CONFIG/playbooks
+
+# It's possible for connectivity to a server or manifest application to break
+# for indeterminate periods of time, so the playbooks should be run without
+# errexit
+set +e
+
+# Run all the ansible playbooks under timeout to prevent them from getting
+# stuck if they are oomkilled
+
+timeout -k 2m 120m ansible-playbook -f 10 ${ANSIBLE_PLAYBOOKS}/remote_puppet_infracloud.yaml