Adds second hiera data directory

This allows hiera data to be split into two buckets: sensitive and
non-sensitive. Sensitive data is put where it always has been. Non-
sensitive data, such as users ssh keys and lists of packages can be
put into the data directory contained within the
openstack-infra/config git repository.

This also introduces a more complex hierarchy: data is now split on
osfamily and lsbdistcondename. Operatingsystem is left for
compatibility.

Change-Id: I3b3faeab837aa5732da01e96bbe0c19aa936a58d

data/Debian/common.yaml

@@ -0,0 +1 @@
+---

data/RedHat/common.yaml

@@ -0,0 +1 @@
+---

data/common.yaml

@@ -0,0 +1 @@
+---

doc/source/puppet.rst

@@ -49,7 +49,7 @@ compatible, so be sure to use an older release - e.g. Ubuntu Precise.
    sudo su -
    git clone https://git.openstack.org/openstack-infra/config /opt/config/production
    /opt/config/production/install_puppet.sh
-   apt-get install puppetmaster-passenger hiera hiera-puppet
+   apt-get install puppetmaster-passenger
 
 Finally, install the modules, fix your hostname and use ``puppet apply`` to
 finish configuration:
@@ -59,7 +59,7 @@ finish configuration:
    bash /opt/config/production/install_modules.sh
    echo $REAL_HOSTNAME > /etc/hostname
    service hostname restart
-   puppet apply --modulepath='/opt/config/production/modules:/etc/puppet/modules' -e 'include openstack_project::puppetmaster'
+   puppet apply --confdir=$(pwd) --modulepath='/opt/config/production/modules:/etc/puppet/modules' -e 'include openstack_project::puppetmaster'
 
 Note: Hiera uses a systemwide configuration file in ``/etc/puppet/hiera.yaml``
 and this setup supports multiple configurations. The two sets of environments

doc/source/sysadmin.rst

@@ -88,7 +88,7 @@ repository::
 
   ./install_puppet.sh
   ./install_modules.sh
-  puppet apply -l /tmp/manifest.log --modulepath=modules:/etc/puppet/modules manifests/local.pp
+  puppet apply -l /tmp/manifest.log --confdir=`pwd` --modulepath=modules:/etc/puppet/modules manifests/local.pp
 
 That should turn the system you are logged into into an etherpad
 server with the same configuration as that used by the OpenStack

hiera.yaml

@@ -0,0 +1 @@
+modules/openstack_project/files/puppetmaster/hiera.yaml

install_jenkins_slave.sh

@@ -24,7 +24,7 @@ if cat /etc/*release | grep -e "CentOS" -e "Red Hat" &> /dev/null; then
 	# yum-config-manager --enable rhel-6-server-optional-rpms
 
         # NOTE: we preinstall lsb_release to ensure facter sets lsbdistcodename
-	yum install -y redhat-lsb-core git puppet
+	yum install -y redhat-lsb-core git puppet heira heira-puppet
 
 else #defaults to Ubuntu
 
@@ -41,11 +41,11 @@ else #defaults to Ubuntu
 
 	apt-get update
 	apt-get dist-upgrade
-	apt-get install -y puppet git rubygems
+	apt-get install -y puppet git rubygems heira heira-puppet
 
 fi
 
 git clone https://git.openstack.org/openstack-infra/config
 bash config/install_modules.sh
 
-puppet apply --modulepath=`pwd`/config/modules:/etc/puppet/modules -e 'node default {class { "openstack_project::bare_slave": install_users => false }}'
+puppet apply --confdir=$(pwd) --modulepath=$(pwd)/config/modules:/etc/puppet/modules -e 'node default {class { "openstack_project::bare_slave": install_users => false }}'

install_puppet.sh

@@ -63,7 +63,7 @@ EOF
     # yum-config-manager --enable rhel-6-server-optional-rpms
 
     # NOTE: we preinstall lsb_release to ensure facter sets lsbdistcodename
-    yum install -y redhat-lsb-core git puppet
+    yum install -y redhat-lsb-core git puppet heira hiera-puppet
 else
     #defaults to Ubuntu
     # NB: keep in sync with openstack_project/files/00-puppet.pref
@@ -83,5 +83,5 @@ EOF
     DEBIAN_FRONTEND=noninteractive apt-get --option 'Dpkg::Options::=--force-confold' \
         --assume-yes dist-upgrade
     DEBIAN_FRONTEND=noninteractive apt-get --option 'Dpkg::Options::=--force-confold' \
-        --assume-yes install -y --force-yes puppet git rubygems
+        --assume-yes install -y --force-yes puppet git rubygems hiera hiera-puppet
 fi

modules/openstack_project/files/nodepool/scripts/prepare_node.sh

@@ -45,10 +45,10 @@ sudo git clone --depth=1 git://git.openstack.org/openstack-infra/config.git \
     /root/config
 sudo /bin/bash /root/config/install_modules.sh
 if [ -z "$NODEPOOL_SSH_KEY" ] ; then
-    sudo puppet apply --modulepath=/root/config/modules:/etc/puppet/modules \
+    sudo puppet apply --confdir=/root/config --modulepath=/root/config/modules:/etc/puppet/modules \
 	-e "class {'openstack_project::single_use_slave': sudo => $SUDO, bare => $BARE, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, }"
 else
-    sudo puppet apply --modulepath=/root/config/modules:/etc/puppet/modules \
+    sudo puppet apply --confdir=/root/config --modulepath=/root/config/modules:/etc/puppet/modules \
 	-e "class {'openstack_project::single_use_slave': install_users => false, sudo => $SUDO, bare => $BARE, python3 => $PYTHON3, include_pypy => $PYPY, all_mysql_privs => $ALL_MYSQL_PRIVS, ssh_key => '$NODEPOOL_SSH_KEY', }"
 fi
 

modules/openstack_project/files/puppetmaster/hiera.yaml

@@ -1,8 +1,13 @@
 ---
 :hierarchy:
+  - fqdn/%{::fqdn}
+  - %{::osfamily}/%{::lsbdistcodename}
+  - %{::osfamily}/common
   - %{operatingsystem}
   - common
 :backends:
   - yaml
 :yaml:
   :datadir: '/etc/puppet/hieradata/%{environment}'
+  :datadir: '%{confdir}/data'
+  :datadir: '/opt/config/%{environment}/data'

run_puppet.sh

@@ -13,4 +13,4 @@ MANIFEST_LOG=/var/log/manifest.log
 cd $BASE_DIR
 /usr/bin/git pull -q && \
   /bin/bash install_modules.sh && \
-  /usr/bin/puppet apply -l $MANIFEST_LOG --modulepath=$MODULE_PATH manifests/site.pp
+  /usr/bin/puppet apply -l $MANIFEST_LOG --confdir=$BASE_DIR --modulepath=$MODULE_PATH manifests/site.pp

test.sh

@@ -13,5 +13,5 @@ sed -i -e 's/hiera..sysadmins../["admin"]/' applytest/puppetapplytest*
 sed -i -e 's/hiera..listadmins../["admin"]/' applytest/puppetapplytest*
 sed -i -e 's/hiera.*/PASSWORD,/' applytest/puppetapplytest*
 for f in `find applytest -name 'puppetapplytest*' -print` ; do
-    puppet apply --modulepath=./modules:/etc/puppet/modules -v --noop --debug $f >/dev/null
+    puppet apply --confdir=$(pwd) --modulepath=./modules:/etc/puppet/modules -v --noop --debug $f >/dev/null
 done