Use ansible instead of direct ssh calls

Instead of a shell script looping over ssh calls, use a simple
ansible playbook. The benefit this gets is that we can then also
script ad-hoc admin tasks either via playbooks or on the command
line. We can also then get rid of the almost entirely unused
salt infrastructure.

Change-Id: I53112bd1f61d94c0521a32016c8a47c8cf9e50f7

manifests/site.pp

@@ -154,14 +154,6 @@ node 'community.openstack.org' {
 node 'ci-puppetmaster.openstack.org' {
   class { 'openstack_project::puppetmaster':
     root_rsa_key    => hiera('puppetmaster_root_rsa_key', 'XXX'),
-    override_list   => [
-      'git01.openstack.org',
-      'git02.openstack.org',
-      'git03.openstack.org',
-      'git04.openstack.org',
-      'git05.openstack.org',
-      'review.openstack.org',
-    ],
     sysadmins       => hiera('sysadmins', ['admin']),
   }
 }

modules/ansible/files/ansible.cfg

@@ -0,0 +1,3 @@
+[defaults]
+hostfile=/usr/local/bin/puppet-inventory
+log_path=/var/log/ansible.log

modules/ansible/files/puppet-inventory

@@ -0,0 +1,30 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import subprocess
+
+output = [
+    x.split()[1][1:-1] for x in subprocess.check_output(
+        ["puppet","cert","list","-a"]).split('\n')
+    if x.startswith('+')
+]
+
+data = {
+    '_meta': {'hostvars': dict()},
+    'ungrouped': output,
+}
+print json.dumps(data, sort_keys=True, indent=2)

modules/ansible/files/roles/puppet/tasks/main.yml

@@ -0,0 +1,6 @@
+---
+- name: run puppet
+  command: timeout -s 9 30m puppet agent --onetime --ignorecache --no-daemonize --no-usecacheonfailure --no-splay --detailed-exitcodes --verbose
+  register: result
+  failed_when: "result.rc != 0 and result.rc != 2"
+  changed_when: "result.rc == 4 or result.rc == 6"

modules/ansible/manifests/init.pp

@@ -0,0 +1,53 @@
+# == Class: ansible
+#
+class ansible {
+
+  include logrotate
+  include pip
+
+  package { 'ansible':
+    ensure   => latest,
+    provider => pip,
+  }
+
+  if ! defined(File['/etc/ansible']) {
+    file { '/etc/ansible':
+      ensure  => directory,
+    }
+  }
+
+  file { '/etc/ansible/ansible.cfg':
+    ensure  => present,
+    source  => 'puppet:///modules/ansible/ansible.cfg',
+    require => File['/etc/ansible'],
+  }
+
+  file { '/usr/local/bin/puppet-inventory':
+    ensure => present,
+    mode   => '0755',
+    owner  => 'root',
+    group  => 'root',
+    source => 'puppet:///modules/ansible/puppet-inventory',
+  }
+
+  file { '/etc/ansible/roles':
+    ensure  => directory,
+    recurse => true,
+    source  => 'puppet:///modules/ansible/roles',
+    require => File['/etc/ansible'],
+  }
+
+  include logrotate
+  logrotate::file { 'ansible':
+    log     => '/var/log/ansible.log',
+    options => [
+      'compress',
+      'copytruncate',
+      'missingok',
+      'rotate 7',
+      'daily',
+      'notifempty',
+    ],
+  }
+
+}

modules/openstack_project/files/ansible/remote_puppet.yaml

@@ -0,0 +1,14 @@
+---
+- hosts: git0*
+  gather_facts: false
+  max_fail_percentage: 1
+  roles:
+    - puppet
+- hosts: review.openstack.org
+  gather_facts: false
+  roles:
+    - puppet
+- hosts: "!review.openstack.org:!git0*"
+  gather_facts: false
+  roles:
+    - puppet

modules/openstack_project/manifests/base.pp

@@ -82,12 +82,16 @@ class openstack_project::base(
   }
 
   ssh_authorized_key { 'puppet-remote-2014-04-17':
+    ensure  => absent,
+    user    => 'root',
+  }
+
+  ssh_authorized_key { 'puppet-remote-2014-05-24':
     ensure  => present,
     user    => 'root',
     type    => 'ssh-rsa',
     key     => 'AAAAB3NzaC1yc2EAAAADAQABAAABAQDSLlN41ftgxkNeUi/kATYPwMPjJdMaSbgokSb9PSkRPZE7GeNai60BCfhu+ky8h5eMe70Bpwb7mQ7GAtHGXPNU1SRBPhMuVN9EYrQbt5KSiwuiTXtQHsWyYrSKtB+XGbl2PhpMQ/TPVtFoL5usxu/MYaakVkCEbt5IbPYNg88/NKPixicJuhi0qsd+l1X1zoc1+Fn87PlwMoIgfLIktwaL8hw9mzqr+pPcDIjCFQQWnjqJVEObOcMstBT20XwKj/ymiH+6p123nnlIHilACJzXhmIZIZO+EGkNF7KyXpcBSfv9efPI+VCE2TOv/scJFdEHtDFkl2kdUBYPC0wQ92rp',
     options => [
-      "command=\"${::openstack_project::params::allowed_ssh_command}\"",
       'from="ci-puppetmaster.openstack.org"',
     ],
     require => File['/root/.ssh'],

modules/openstack_project/manifests/params.pp

@@ -18,5 +18,4 @@ class openstack_project::params {
       fail("Unsupported osfamily: ${::osfamily} The 'openstack_project' module only supports osfamily Debian or RedHat (slaves only).")
     }
   }
-  $allowed_ssh_command = 'timeout -s 9 30m puppet agent --onetime --ignorecache --no-daemonize --no-usecacheonfailure --no-splay'
 }

modules/openstack_project/manifests/puppetmaster.pp

@@ -2,13 +2,13 @@
 #
 class openstack_project::puppetmaster (
   $root_rsa_key,
-  $override_list = [],
   $salt = true,
   $update_slave = true,
   $sysadmins = [],
   $version   = '2.7.',
   $ca_server = undef,
 ) {
+  include ansible
   include logrotate
   include openstack_project::params
 
@@ -77,12 +77,6 @@ class openstack_project::puppetmaster (
     mode   => '0750',
     }
 
-  file { '/usr/local/bin/run_remote_puppet':
-    ensure  => present,
-    mode    => '0700',
-    content => template('openstack_project/run_remote_puppet.sh.erb'),
-  }
-
   if ! defined(File['/root/.ssh']) {
     file { '/root/.ssh':
       ensure => directory,
@@ -121,4 +115,12 @@ class openstack_project::puppetmaster (
     puppetdb_soft_write_failure  => true,
   }
 
+# Playbooks
+#
+  file { '/etc/ansible/remote_puppet.yaml':
+    ensure  => present,
+    source  => 'puppet:///modules/openstack_project/ansible/remote_puppet.yaml',
+    require => Class[ansible],
+  }
+
 }

modules/openstack_project/templates/run_remote_puppet.sh.erb

@@ -1,62 +0,0 @@
-#!/bin/bash
-
-# Copyright 2014 Hewlett-Packard Development Company, L.P.
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-
-# This function will properly fail if puppet is disabled on the target host
-function run_ssh {
-    echo ssh root@$1 <%= scope.lookupvar('openstack_project::params::allowed_ssh_command') %>
-    ssh -o StrictHostKeyChecking=no root@$1 <%= scope.lookupvar('openstack_project::params::allowed_ssh_command') %> >/dev/null
-    ret=$?
-    if [ $ret == 124 ] ; then
-        echo "puppet on $1 timed out"
-    elif [ $ret != 0 ] ; then
-        echo "puppet on $1 failed with return code: $ret"
-    fi
-    return $ret
-}
-export -f run_ssh
-
-function brazenly_run_ssh {
-    run_ssh $1
-    return 0
-}
-export -f brazenly_run_ssh
-
-FULL_LIST=$(puppet cert list -a  | grep '^\+' | awk '{print $2}' | sed 's/"//g')
-OVERRIDE_LIST="
-<% @override_list.each do |host| -%>
-<%= host %>
-<% end -%>
-"
-FILTERED_LIST=""
-for host in $FULL_LIST; do
-    if ! echo $OVERRIDE_LIST | grep $host >/dev/null 2>&1 ; then
-        FILTERED_LIST="$FILTERED_LIST $host"
-    fi
-done
-
-cd /opt/config/production
-
-# Run things that need to be ordered
-for host in $OVERRIDE_LIST; do
-    run_ssh $host
-    ret=$?
-    if [ $ret != 0 ] ; then
-        break
-    fi
-done
-
-# Now, run everyone else
-echo $FILTERED_LIST | xargs -d' ' -P 10 -n 1 -I{} bash -c "brazenly_run_ssh {}"

run_all.sh

@@ -23,7 +23,5 @@ git fetch -a && git reset -q --hard @{u}
 # some times
 touch manifests/site.pp
 
-# Put in a logging header
-echo "****\n\n$(date)\n\n****" >> /var/log/puppet_run_all.log 2>&1
 # Run this as an external script so that the above pull will get new changes
-/usr/local/bin/run_remote_puppet >> /var/log/puppet_run_all.log 2>&1
+ansible-playbook -v /etc/ansible/remote_puppet.yaml