Instance HA support

This commit is the puppet-tripleo part that implements the instance HA
spec. It only adds a couple of pacemaker resources and stonith devices
when instance HA is activated via an environment variable.

The compute nodes that take part in instance HA need to have the
following services added to them:
OS::TripleO::Services::ComputeInstanceHA
OS::TripleO::Services::PacemakerRemote

More detailed info will be added to the docs.

blueprint instance-ha

Change-Id: I4d1908242e9513a225d2b1da06ed4ee769ee10f7

Puppetfile_extras

@@ -65,3 +65,6 @@ mod 'sensu',
   :git => 'https://github.com/sensu/sensu-puppet',
   :ref => 'master'
 
+mod 'pacemaker',
+  :git => 'https://github.com/openstack/puppet-pacemaker',
+  :ref => 'master'

manifests/fencing.pp

@@ -42,15 +42,28 @@
 #  Delay (in seconds) between attempts when creating fence devices
 #  and constraints.
 #  Defaults to 3
+#
+# [*enable_instanceha*]
+#  (Optional) Boolean driving the Instance HA controlplane configuration
+#  Defaults to false
+#
 class tripleo::fencing(
-  $config = {},
-  $tries = 10,
-  $try_sleep = 3,
+  $config            = {},
+  $tries             = 10,
+  $try_sleep         = 3,
+  $enable_instanceha = hiera('tripleo::instanceha', false),
 ) {
   $common_params = {
     'tries' => $tries,
     'try_sleep' => $try_sleep,
   }
+  # We will create stonith levels *only* if the node is a compute instanceha one
+  if member(hiera('compute_instanceha_short_node_names'), downcase($::hostname)) {
+    $is_compute_instanceha_node = true
+  } else {
+    $is_compute_instanceha_node = false
+  }
+
 
   $all_devices = $config['devices']
 
@@ -59,6 +72,24 @@ class tripleo::fencing(
 
   $ipmilan_devices = local_fence_devices('fence_ipmilan', $all_devices)
   create_resources('pacemaker::stonith::fence_ipmilan', $ipmilan_devices, $common_params)
+  if ($enable_instanceha and $is_compute_instanceha_node) {
+    if length($ipmilan_devices) != 1 {
+      fail('Multiple (or zero) IPmilan devices for a single host are not supported with instance HA')
+    }
+    # Get the first (and only) key which is the mac-address
+    $mac = keys($ipmilan_devices)[0]
+    $safe_mac = regsubst($mac, ':', '', 'G')
+    $stonith_resources = [ "stonith-fence_ipmilan-${safe_mac}", 'stonith-fence_compute-fence-nova' ]
+    Pcmk_stonith <||> -> Pcmk_stonith_level <||>
+    pacemaker::stonith::level{ "stonith-level-${safe_mac}":
+      level             => 1,
+      target            => '$(/usr/sbin/crm_node -n)',
+      stonith_resources => $stonith_resources,
+      tries             => $tries,
+      try_sleep         => $try_sleep,
+    }
+  }
+
 
   $ironic_devices = local_fence_devices('fence_ironic', $all_devices)
   create_resources('pacemaker::stonith::fence_ironic', $ironic_devices, $common_params)

manifests/profile/base/pacemaker.pp

@@ -67,6 +67,10 @@
 #   (Optional) Whether or not to enable encryption of the pacemaker traffic
 #   Defaults to true
 #
+# [*enable_instanceha*]
+#  (Optional) Boolean driving the Instance HA controlplane configuration
+#  Defaults to false
+#
 class tripleo::profile::base::pacemaker (
   $step                      = Integer(hiera('step')),
   $pcs_tries                 = hiera('pcs_tries', 20),
@@ -79,6 +83,7 @@ class tripleo::profile::base::pacemaker (
   $remote_try_sleep          = hiera('pacemaker_remote_try_sleep', 60),
   $cluster_recheck_interval  = hiera('pacemaker_cluster_recheck_interval', undef),
   $encryption                = true,
+  $enable_instanceha         = hiera('tripleo::instanceha', false),
 ) {
 
   if count($remote_short_node_names) != count($remote_node_ips) {
@@ -176,26 +181,29 @@ class tripleo::profile::base::pacemaker (
     }
   }
 
-  if $step >= 2 {
-    if $pacemaker_master {
+  if $enable_instanceha and $pacemaker_master {
+    include ::tripleo::profile::base::pacemaker::instance_ha
+  }
+
+  if ($step >= 2 and $pacemaker_master) {
+    if ! $enable_instanceha {
       include ::pacemaker::resource_defaults
-      # When we have a non-zero number of pacemaker remote nodes we
-      # want to set the cluster-recheck-interval property to something
-      # lower (unless the operator has explicitely set a value)
-      if count($remote_short_node_names) > 0 and $cluster_recheck_interval == undef {
-        pacemaker::property{ 'cluster-recheck-interval-property':
-          property => 'cluster-recheck-interval',
-          value    => '60s',
-          tries    => $pcs_tries,
-        }
-      } elsif $cluster_recheck_interval != undef {
-        pacemaker::property{ 'cluster-recheck-interval-property':
-          property => 'cluster-recheck-interval',
-          value    => $cluster_recheck_interval,
-          tries    => $pcs_tries,
-        }
+    }
+    # When we have a non-zero number of pacemaker remote nodes we
+    # want to set the cluster-recheck-interval property to something
+    # lower (unless the operator has explicitely set a value)
+    if count($remote_short_node_names) > 0 and $cluster_recheck_interval == undef {
+      pacemaker::property{ 'cluster-recheck-interval-property':
+        property => 'cluster-recheck-interval',
+        value    => '60s',
+        tries    => $pcs_tries,
+      }
+    } elsif $cluster_recheck_interval != undef {
+      pacemaker::property{ 'cluster-recheck-interval-property':
+        property => 'cluster-recheck-interval',
+        value    => $cluster_recheck_interval,
+        tries    => $pcs_tries,
       }
     }
   }
-
 }

manifests/profile/base/pacemaker/instance_ha.pp

@@ -0,0 +1,121 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: tripleo::profile::base::pacemaker::instance_ha
+#
+# Pacemaker profile for configuring instance HA on the control plane in tripleo
+# Note that this class is included under the condition $pacemaker_master and $enable_instanceha
+#
+# === Parameters
+#
+# [*step*]
+#   (Optional) The current step in deployment. See tripleo-heat-templates
+#   for more details.
+#   Defaults to hiera('step')
+#
+# [*pcs_tries*]
+#   (Optional) The number of times pcs commands should be retried.
+#   Defaults to hiera('pcs_tries', 20)
+#
+# [*keystone_endpoint_url*]
+#   The keystone public endpoint url
+#   Defaults to hiera('keystone::endpoint::public_url')
+#
+# [*keystone_password*]
+#   The keystone admin password
+#   Defaults to hiera('keystone::admin_password')
+#
+# [*keystone_admin*]
+#   The keystone admin username
+#   Defaults to hiera('keystone::roles::admin::admin_tenant', 'admin')
+#
+# [*keystone_domain*]
+#   The keystone domain
+#   Defaults to hiera('tripleo::clouddomain', 'localdomain')
+#
+# [*user_domain*]
+#   The keystone user domain for nova
+#   Defaults to hiera('nova::keystone::authtoken::user_domain_name', 'Default')
+#
+# [*project_domain*]
+#   The keystone project domain for nova
+#   Defaults to hiera('nova::keystone::authtoken::project_domain_name', 'Default')
+#
+class tripleo::profile::base::pacemaker::instance_ha (
+  $step                  = Integer(hiera('step')),
+  $pcs_tries             = hiera('pcs_tries', 20),
+  $keystone_endpoint_url = hiera('keystone::endpoint::public_url'),
+  $keystone_password     = hiera('keystone::admin_password'),
+  $keystone_admin        = hiera('keystone::roles::admin::admin_tenant', 'admin'),
+  $keystone_domain       = hiera('tripleo::clouddomain', 'localdomain'),
+  $user_domain           = hiera('nova::keystone::authtoken::user_domain_name', 'Default'),
+  $project_domain        = hiera('nova::keystone::authtoken::project_domain_name', 'Default'),
+) {
+  if $step >= 2 {
+    class { '::pacemaker::resource_defaults':
+      tries    => $pcs_tries,
+      defaults => {
+        'fencing-default' => {
+          name  => 'requires',
+          value => 'fencing',
+        },
+      },
+    }
+  }
+  # We need the guarantee that keystone is configured before creating the next resources
+  if $step >= 4 {
+    pacemaker::stonith::fence_compute { 'fence-nova':
+      auth_url       => $keystone_endpoint_url,
+      login          => $keystone_admin,
+      passwd         => $keystone_password,
+      tenant_name    => $keystone_admin,
+      project_domain => $project_domain,
+      user_domain    => $user_domain,
+      domain         => $keystone_domain,
+      record_only    => 1,
+      meta_attr      => 'provides=unfencing',
+      pcmk_host_list => '',
+      tries          => $pcs_tries,
+    }
+
+    pacemaker::resource::ocf { 'compute-unfence-trigger':
+      ocf_agent_name => 'pacemaker:Dummy',
+      op_params      => 'start requires=unfencing',
+      clone_params   => true,
+      tries          => $pcs_tries,
+      location_rule  => {
+        resource_discovery => 'never',
+        score              => '-INFINITY',
+        expression         => ['compute-instanceha-role ne true'],
+      }
+    }
+    if hiera('tripleo::instanceha::no_shared_storage', true) {
+      $iha_no_shared_storage = 'no_shared_storage=true'
+    } else {
+      $iha_no_shared_storage = 'no_shared_storage=false'
+    }
+    pacemaker::resource::ocf { 'nova-evacuate':
+      ocf_agent_name  => 'openstack:NovaEvacuate',
+      # lint:ignore:140chars
+      resource_params => "auth_url=${keystone_endpoint_url} username=${keystone_admin} password=${keystone_password} user_domain=${user_domain} project_domain=${project_domain} tenant_name=${keystone_admin} ${iha_no_shared_storage}",
+      # lint:endignore
+      tries           => $pcs_tries,
+      location_rule   => {
+        resource_discovery => 'never',
+        score              => '-INFINITY',
+        expression         => ['compute-instanceha-role eq true'],
+      }
+    }
+  }
+}

manifests/profile/pacemaker/compute_instanceha.pp

@@ -0,0 +1,33 @@
+# == Class: tripleo::profile::pacemaker::compute_instanceha
+#
+# Configures Compute nodes for Instance HA
+#
+# === Parameters:
+#
+# [*step*]
+#   (Optional) The current step in deployment. See tripleo-heat-templates
+#   for more details.
+#   Defaults to hiera('step')
+#
+# [*pcs_tries*]
+#   (Optional) The number of times pcs commands should be retried.
+#   Defaults to hiera('pcs_tries', 20)
+#
+# [*enable_instanceha*]
+#  (Optional) Boolean driving the Instance HA controlplane configuration
+#  Defaults to false
+#
+class tripleo::profile::pacemaker::compute_instanceha (
+  $step              = Integer(hiera('step')),
+  $pcs_tries         = hiera('pcs_tries', 20),
+  $enable_instanceha = hiera('tripleo::instanceha', false),
+) {
+  if $step >= 2 and $enable_instanceha {
+    pacemaker::property { 'compute-instanceha-role-node-property':
+      property => 'compute-instanceha-role',
+      value    => true,
+      tries    => $pcs_tries,
+      node     => $::hostname,
+    }
+  }
+}

spec/classes/tripleo_profile_base_pacemaker_spec.rb

@@ -0,0 +1,71 @@
+#
+# Copyright (C) 2017 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+
+require 'spec_helper'
+
+describe 'tripleo::profile::base::pacemaker' do
+  shared_examples_for 'tripleo::profile::base::pacemaker' do
+    before :each do
+      facts.merge!({
+        :step => params[:step],
+      })
+    end
+
+    context 'with step 4 with defaults (instanceha disabled)' do
+      let(:params) { {
+        :step => 4,
+      } }
+
+      it {
+        is_expected.to_not contain_class('tripleo::profile::base::pacemaker::instance_ha')
+        is_expected.to_not contain_class('pacemaker::stonith::fence_compute')
+      }
+    end
+
+    context 'with step 4 with instanceha enabled' do
+      let(:params) { {
+        :step              => 4,
+        :enable_instanceha => true,
+      } }
+
+      it {
+        is_expected.to contain_class('tripleo::profile::base::pacemaker::instance_ha')
+        is_expected.to contain_class('pacemaker::resource_defaults')
+        is_expected.to contain_pcmk_stonith('stonith-fence_compute-fence-nova').with({
+          :stonith_type    => "fence_compute",
+        })
+        is_expected.to contain_pcmk_resource('compute-unfence-trigger').with({
+          :resource_type   => "ocf:pacemaker:Dummy",
+          :op_params       => "start requires=unfencing",
+        })
+        is_expected.to contain_pcmk_resource('nova-evacuate').with({
+          :resource_type   => "ocf:openstack:NovaEvacuate",
+          :resource_params => "auth_url=localhost:5000 username=admin password=password user_domain=Default project_domain=Default tenant_name=admin no_shared_storage=true",
+        })
+      }
+    end
+  end
+
+  on_supported_os.each do |os, facts|
+    context "on #{os}" do
+      let(:facts) do
+        facts.merge({ :hostname => 'node.example.com' })
+      end
+
+      it_behaves_like 'tripleo::profile::base::pacemaker'
+    end
+  end
+end

spec/fixtures/hieradata/default.yaml

@@ -50,3 +50,16 @@ octavia::rabbit_password: 'password'
 horizon::secret_key: 'secrete'
 #Neutron related
 neutron::rabbit_password: 'password'
+# Pacemaker related
+pacemaker_short_bootstrap_node_name: 'node.example.com'
+pacemaker_short_node_names:
+ - 'node.example.com'
+hacluster_pwd: 'password'
+pacemaker::resource_defaults::defaults:
+ test-default:
+   name: 'requires'
+   value: 'noop'
+# pcmk instance ha
+keystone::endpoint::public_url: 'localhost:5000'
+keystone::admin_password: 'password'
+