Add shade and openstack inventory to system

We're not ready to move from puppet inventory to openstack inventory
just yet, so don't actually swap the dynamic inventory plugin. But, add
it to the system so that running manual tests of all of the pieces is
possible.

Add the currently administratively disabled hosts to the disabled group
so that we can verify this works.

Change-Id: I73931332b2917b71a008f9213365f7594f69c41e

doc/source/puppet.rst

@@ -142,22 +142,7 @@ Disabling Puppet on Nodes
 -------------------------
 
 In the case of needing to disable the running of puppet on a node, it's a
-simple matter of disabling the agent:
+simple matter of adding an entry to the ansible inventory "disabled" group.
-
-.. code-block:: bash
-
-  sudo puppet agent --disable
-
-This will prevent any subsequent runs of the agent, including ones triggered
-globally by the run_all script. If, as an admin, you need to run puppet on
-a node where it has been disabled, you need to specify an alternate disable
-lock file which will allow your local run of puppet without allowing the
-globally orchestrated runs to occur:
-
-.. code-block:: bash
-
-  sudo puppet agent --test --agent_disabled_lockfile=/tmp/alt-lock-file
-
 
 Important Notes
 ---------------

doc/source/sysadmin.rst

@@ -285,14 +285,31 @@ Disable/Enable Puppet
 You should normally not make manual changes to servers, but instead,
 make changes through puppet.  However, under some circumstances, you
 may need to temporarily make a manual change to a puppet-managed
-resource on a server.  In that case, run the following command on that
+resource on a server.
-server to disable puppet::
 
-  sudo puppet agent --disable
+In the case of needing to disable the running of puppet on a node, it's a
+simple matter of adding an entry to the ansible inventory "disabled" group.
+There are two inventory files available for this, `/etc/ansible/hosts/static`
+and `/etc/ansible/hosts/emergency`. `/etc/ansible/hosts/static` is intended
+to be managed via git from the system-config repo in
+`modules/openstack_project/files/puppetmaster/static-inventory`.
+`/etc/ansible/hosts/emergency` is a file that should normally be empty, but
+the contents are not managed by puppet. It's purpose is to allow for disabling
+puppet at times when landing a change to the puppet repo would be either
+unreasonable or impossible.
 
-When you are ready for puppet to run again, use::
+There are two sections in each file, `disabled` and `disabled:children`. Due
+to te multi-cloud nature of the ansible inventory, a hostname cannot be counted
+on to be unique, so each cloud instance is listed in the inventory by its
+UUID with a group created for its hostname. If you want to disable a cloud
+instance by name, you need to put its name in `disabled:children`. If you want
+to refer to a single instance by UUID, or if there are statically defined
+hosts that need to be disabled, you should put those in `disabled`.
 
-  sudo puppet agent --enable
+Disabling puppet via ansible inventory does not disable puppet from being
+run directly on the host, it merely prevents the puppetmaster from causing
+puppet to be run. If you choose to run puppet manually on a host, take care
+to ensure that it has not been disabled at the puppetmaster level first.
 
 .. _cinder:
 

modules/openstack_project/files/puppetmaster/static-inventory

@@ -0,0 +1,11 @@
+[disabled]
+ci-backup-rs-ord.openstack.org
+jenkins-dev.openstack.org
+hound.openstack.org
+pypi.bhs1.openstack.org
+pypi.region-b.geo-1.openstack.org
+pypi.sjc1.openstack.org
+review-dev.openstack.org
+subunit-worker01.openstack.org
+
+[disabled:children]

modules/openstack_project/manifests/puppetmaster.pp

@@ -11,7 +11,9 @@ class openstack_project::puppetmaster (
   include logrotate
   include openstack_project::params
 
-  include ansible
+  class { '::ansible':
+    ansible_hostfile => '/etc/ansible/hosts',
+  }
 
   file { '/etc/ansible/hostfile':
     ensure  => present,
@@ -119,19 +121,19 @@ class openstack_project::puppetmaster (
   }
 
 # For launch/launch-node.py.
-  package { ['python-cinderclient', 'python-novaclient']:
+  package { 'shade':
     ensure   => latest,
     provider => pip,
-    require  => [Package['python-lxml'], Package['libxslt1-dev']],
   }
   package { 'python-paramiko':
     ensure => present,
   }
+  # No longer needed with latest client libs
   package { 'python-lxml':
-    ensure => present,
+    ensure => absent,
   }
   package { 'libxslt1-dev':
-    ensure => present,
+    ensure => absent,
   }
 
 # Enable puppetdb
@@ -170,25 +172,37 @@ class openstack_project::puppetmaster (
     require => Cron['restartjenkinsmasters'],
   }
 
-# Playbooks
+  vcsrepo { '/opt/ansible':
-#
+    ensure   => latest,
-  file { '/etc/ansible/playbooks':
+    provider => git,
-    ensure  => absent,
+    revision => 'stable-2.0',
+    source   => 'https://github.com/ansible/ansible',
   }
 
-  file { '/etc/ansible/remote_puppet.yaml':
+  file { '/etc/ansible/hosts':
-    ensure => absent,
+    ensure  => directory,
   }
-  file { '/etc/ansible/remote_puppet_afs.yaml':
+
-    ensure => absent,
+  file { '/etc/ansible/hosts/puppet':
+    owner       => 'root',
+    group       => 'root',
+    mode        => '0755',
+    subscribe   => Class['::ansible'],
+    source      => '/usr/local/bin/puppet.py',
+    replace     => true,
   }
-  file { '/etc/ansible/remote_puppet_else.yaml':
+
-    ensure => absent,
+  file { '/etc/ansible/hosts/static':
+    owner   => 'root',
+    group   => 'root',
+    mode    => '0444',
+    source  => 'puppet:///modules/openstack_project/puppetmaster/static-inventory',
   }
-  file { '/etc/ansible/remote_puppet_git.yaml':
+
-    ensure => absent,
+  file { '/etc/ansible/hosts/emergency':
-  }
+    ensure  => present,
-  file { '/etc/ansible/clean_workspaces.yaml':
+    owner   => 'root',
-    ensure => absent,
+    group   => 'root',
+    mode    => '0444',
   }
 }

modules/openstack_project/templates/puppetmaster/ansible-clouds.yaml.erb

@@ -1,3 +1,6 @@
+cache:
+  expiration_time: 86400
+  path: /var/cache/ansible-inventory.json
 clouds:
   openstackci2-hpcloud:
     profile: hp

playbooks/remote_puppet_afs.yaml

@@ -1,5 +1,5 @@
 ---
-- hosts: "afs*"
+- hosts: "afs*:!disabled"
   gather_facts: true
   roles:
   - role: puppet

playbooks/remote_puppet_else.yaml

@@ -1,4 +1,4 @@
-- hosts: '!review.openstack.org:!git0*:!afs*:!puppetmaster*'
+- hosts: '!review.openstack.org:!git0*:!afs*:!puppetmaster*!disabled'
   gather_facts: true
   roles:
   - role: puppet

playbooks/remote_puppet_git.yaml

@@ -1,5 +1,5 @@
 ---
-- hosts: localhost
+- hosts: "localhost:!disabled"
   gather_facts: true
   connection: local
   tasks:
@@ -10,14 +10,14 @@
   roles:
     - role: puppet
       copy_hieradata: False
-- hosts: git0*
+- hosts: "git0*:!disabled"
   gather_facts: true
   max_fail_percentage: 1
   roles:
     - role: puppet
       facts:
         project_config_ref: "{{ hostvars.localhost.gitinfo.after }}"
-- hosts: review.openstack.org
+- hosts: "review.openstack.org:!disabled"
   gather_facts: true
   roles:
     - role: puppet

playbooks/update_puppet.yaml

@@ -1,5 +1,5 @@
 ---
-- hosts: '!puppetmaster.openstack.org'
+- hosts: '!puppetmaster.openstack.org:!disabled'
   gather_facts: false
   tasks:
   - synchronize: