Run puppet even if git servers fail

The current set of runs make the fleet depend on all git changes
working. The only thing we actually care about is that gerrit doesn't
get updated if git fails.

Change-Id: Id488e14c7dbaddfbffece7b1d8ef65f06b3688d8

modules/openstack_project/files/ansible/remote_puppet_else.yaml

@@ -0,0 +1,5 @@
+---
+- hosts: "!review.openstack.org:!git0*:!afs*"
+  gather_facts: false
+  roles:
+    - { role: puppet, puppetmaster: puppetmaster.openstack.org }

modules/openstack_project/files/ansible/remote_puppet_git.yaml

@@ -8,7 +8,3 @@
   gather_facts: false
   roles:
     - { role: puppet, puppetmaster: puppetmaster.openstack.org }
-- hosts: "!review.openstack.org:!git0*:!afs*"
-  gather_facts: false
-  roles:
-    - { role: puppet, puppetmaster: puppetmaster.openstack.org }

modules/openstack_project/manifests/puppetmaster.pp

@@ -18,7 +18,9 @@ class openstack_project::puppetmaster (
     ca_server                 => $ca_server,
   }
 
-  $ansible_remote_puppet_source = 'puppet:///modules/openstack_project/ansible/remote_puppet.yaml'
+  $ansible_remote_puppet_else_source = 'puppet:///modules/openstack_project/ansible/remote_puppet_else.yaml'
+  $ansible_remote_puppet_afs_source = 'puppet:///modules/openstack_project/ansible/remote_puppet_afs.yaml'
+  $ansible_remote_puppet_git_source = 'puppet:///modules/openstack_project/ansible/remote_puppet_git.yaml'
 
   file {'/etc/puppet/environments':
     ensure => directory,
@@ -164,9 +166,21 @@ class openstack_project::puppetmaster (
 
 # Playbooks
 #
-  file { '/etc/ansible/remote_puppet.yaml':
+  file { '/etc/ansible/remote_puppet_else.yaml':
     ensure  => present,
-    source  => $ansible_remote_puppet_source,
+    source  => $ansible_remote_puppet_else_source,
+    require => Class[ansible],
+  }
+
+  file { '/etc/ansible/remote_puppet_afs.yaml':
+    ensure  => present,
+    source  => $ansible_remote_puppet_afs_source,
+    require => Class[ansible],
+  }
+
+  file { '/etc/ansible/remote_puppet_git.yaml':
+    ensure  => present,
+    source  => $ansible_remote_puppet_git_source,
     require => Class[ansible],
   }
 

run_all.sh

@@ -23,8 +23,11 @@ git fetch -a && git reset -q --hard @{u}
 # some times
 touch manifests/site.pp
 
-# Run this as an external script so that the above pull will get new changes
+# First run the git/gerrit sequence, since it's important that they all work
-ansible-playbook /etc/ansible/remote_puppet.yaml >> /var/log/puppet_run_all.log 2>&1
+# together
+ansible-playbook /etc/ansible/remote_puppet_git.yaml >> /var/log/puppet_run_all.log 2>&1
 # Run AFS changes separately so we can make sure to only do one at a time
 # (turns out quorum is nice to have)
 ansible-playbook -f 1 /etc/ansible/remote_puppet_afs.yaml >> /var/log/puppet_run_all.log 2>&1
+# Run everything else. We do not care if the other things worked
+ansible-playbook /etc/ansible/remote_puppet_else.yaml >> /var/log/puppet_run_all.log 2>&1