Improve cluster deployment

- Add firewall rules for corosync communication - Use dedicated cluster.pp manifest to allow the deployment of coexistant clusters for LMA plugins - Use primary role property Implements: blueprint elasticsearch-clustering Change-Id: Ibf4c1c4e62f214725875869621b40a3ef4c20e53
2016-01-04 10:06:31 +01:00 · 2016-01-04 10:06:31 +01:00 · ab90489df0
parent 082b7c4988
commit ab90489df0
10 changed files with 142 additions and 44 deletions
--- a/deployment_scripts/puppet/manifests/cluster.pp
+++ b/deployment_scripts/puppet/manifests/cluster.pp
@ -0,0 +1,60 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+# Compared to the osnailyfacter/modular/cluster/cluster.pp manifest, this task
+# supports the use case where the Pacemaker cluster is made of several
+# unrelated roles.
+prepare_network_config(hiera_hash('network_scheme'))
+
+$corosync_nodes = corosync_nodes(
+    get_nodes_hash_by_roles(
+      hiera_hash('network_metadata'),
+      hiera_array('lma::corosync_roles')
+    ),
+    'mgmt/corosync'
+)
+
+class { 'cluster':
+  internal_address => get_network_role_property('mgmt/corosync', 'ipaddr'),
+  corosync_nodes   => $corosync_nodes,
+}
+
+pcmk_nodes { 'pacemaker' :
+  nodes               => $corosync_nodes,
+  add_pacemaker_nodes => false,
+}
+
+Service <| title == 'corosync' |> {
+  subscribe => File['/etc/corosync/service.d'],
+  require   => File['/etc/corosync/corosync.conf'],
+}
+
+Service['corosync'] -> Pcmk_nodes<||>
+Pcmk_nodes<||> -> Service<| provider == 'pacemaker' |>
+
+# Sometimes during first start pacemaker can not connect to corosync
+# via IPC due to pacemaker and corosync processes are run under different users
+if($::operatingsystem == 'Ubuntu') {
+  $pacemaker_run_uid = 'hacluster'
+  $pacemaker_run_gid = 'haclient'
+
+  file {'/etc/corosync/uidgid.d/pacemaker':
+    content =>"uidgid {
+   uid: ${pacemaker_run_uid}
+   gid: ${pacemaker_run_gid}
+}"
+  }
+
+  File['/etc/corosync/corosync.conf'] -> File['/etc/corosync/uidgid.d/pacemaker'] -> Service <| title == 'corosync' |>
+}
--- a/deployment_scripts/puppet/manifests/elasticsearch.pp
+++ b/deployment_scripts/puppet/manifests/elasticsearch.pp
@ -16,7 +16,7 @@ prepare_network_config(hiera('network_scheme', {}))
 $mgmt_address = get_network_role_property('management', 'ipaddr')
 $elasticsearch_kibana = hiera_hash('elasticsearch_kibana')
 $network_metadata = hiera('network_metadata')
-$es_nodes = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana'])
+$es_nodes = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana', 'primary-elasticsearch_kibana'])
 $es_address_map = get_node_to_ipaddr_map_by_network_role($es_nodes, 'management')
 $es_nodes_ips = values($es_address_map)

--- a/deployment_scripts/puppet/manifests/firewall.pp
+++ b/deployment_scripts/puppet/manifests/firewall.pp
@ -37,6 +37,17 @@ firewall {'020 ssh':
  proto  => 'tcp',
  action => 'accept',
 }
+firewall { '113 corosync-input':
+  port   => 5405,
+  proto  => 'udp',
+  action => 'accept',
+}
+
+firewall { '114 corosync-output':
+  port   => 5404,
+  proto  => 'udp',
+  action => 'accept',
+}

 firewall { '100 elasticsearch REST':
  port   => 9200,
--- a/deployment_scripts/puppet/manifests/haproxy.pp
+++ b/deployment_scripts/puppet/manifests/haproxy.pp
@ -12,19 +12,12 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 #
-$plugin_name = 'elasticsearch_kibana'
+$role_name = 'elasticsearch_kibana'
 $es_port = '9200'
 $nginx_port = '80'
-$vip_name = 'es_vip_mgmt'
-
-$network_metadata = hiera_hash('network_metadata')
-if ! $network_metadata['vips'][$vip_name] {
-  fail('Elasticsearch VIP is not defined')
-}
-$vip = $network_metadata['vips'][$vip_name]['ipaddr']
-notice($vip)
-$roles = [ $plugin_name ]
-$es_nodes = get_nodes_hash_by_roles($network_metadata, $roles)
+$vip = hiera('lma::elasticsearch::vip')
+$roles = [ $role_name , "primary-${role_name}"]
+$es_nodes = get_nodes_hash_by_roles(hiera_hash('network_metadata'), $roles)
 $es_address_map = get_node_to_ipaddr_map_by_network_role($es_nodes, 'elasticsearch')
 $es_nodes_ips = values($es_address_map)
 $es_nodes_names = keys($es_address_map)
--- a/deployment_scripts/puppet/manifests/hiera_override.pp
+++ b/deployment_scripts/puppet/manifests/hiera_override.pp
@ -15,11 +15,16 @@
 $hiera_dir            = '/etc/hiera/plugins'
 $plugin_name          = 'elasticsearch_kibana'
 $plugin_yaml          = "${plugin_name}.yaml"
-$corosync_roles       = [$plugin_name]
+$corosync_roles       = [$plugin_name, "primary-${plugin_name}"]
 $elasticsearch_kibana = hiera_hash('elasticsearch_kibana')
 $network_metadata     = hiera('network_metadata')
-$es_nodes             = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana'])
+$es_nodes             = get_nodes_hash_by_roles($network_metadata, ['elasticsearch_kibana', 'primary-elasticsearch_kibana'])
 $es_nodes_count       = count($es_nodes)
+$vip_name             = 'es_vip_mgmt'
+if ! $network_metadata['vips'][$vip_name] {
+  fail('Elasticsearch VIP is not defined')
+}
+$vip = $network_metadata['vips'][$vip_name]['ipaddr']

 if is_integer($elasticsearch_kibana['number_of_replicas']) and $elasticsearch_kibana['number_of_replicas'] < $es_nodes_count {
  $number_of_replicas = 0 + $elasticsearch_kibana['number_of_replicas']
@ -52,12 +57,13 @@ if is_integer($elasticsearch_kibana['recover_after_nodes']) and $elasticsearch_k
 }

 $calculated_content = inline_template('
-corosync_roles:
+lma::corosync_roles:
 <%
@corosync_roles.each do |crole|
 %>  - <%= crole %>
 <% end -%>

+lma::elasticsearch::vip: <%= @vip%>
 lma::elasticsearch::number_of_replicas: <%= @number_of_replicas %>
 lma::elasticsearch::minimum_master_nodes: <%= @minimum_master_nodes %>
 lma::elasticsearch::recover_after_time: <%= @recover_after_time %>
--- a/deployment_tasks.yaml
+++ b/deployment_tasks.yaml
@ -1,3 +1,21 @@
+- id: primary-elasticsearch_kibana
+  type: group
+  role: [primary-elasticsearch_kibana]
+  tasks:
+    - fuel_pkgs
+    - hiera
+    - globals
+    - tools
+    - logging
+    - netconfig
+    - hosts
+    - cluster-haproxy
+  required_for: [deploy_end]
+  requires: [deploy_start]
+  parameters:
+    strategy:
+      type: one_by_one
+
 - id: elasticsearch_kibana
  type: group
  role: [elasticsearch_kibana]
@ -9,30 +27,46 @@
    - logging
    - netconfig
    - hosts
-    - firewall
-    - deploy_start
-    - cluster
    - cluster-haproxy
-    - openstack-haproxy-stats
  required_for: [deploy_end]
-  requires: [deploy_start]
+  requires: [deploy_start, primary-elasticsearch_kibana]
  parameters:
    strategy:
-      type: one_by_one
+      type: parallel

- id: es-virtual-ip
+- id: lma-es-firewall
  type: puppet
-  groups: [elasticsearch_kibana]
+  groups: [primary-elasticsearch_kibana, elasticsearch_kibana]
  required_for: [deploy_end]
-  requires: [cluster]
+  requires: [netconfig]
+  parameters:
+    puppet_manifest: "puppet/manifests/firewall.pp"
+    puppet_modules: "puppet/modules"
+    timeout: 3600
+
+- id: lma-es-cluster
+  type: puppet
+  groups: [primary-elasticsearch_kibana, elasticsearch_kibana]
+  required_for: [cluster-haproxy]
+  requires: [lma-es-firewall]
+  parameters:
+    puppet_manifest: "puppet/manifests/cluster.pp"
+    puppet_modules: "puppet/modules"
+    timeout: 3600
+
+- id: lma-es-virtual-ip
+  type: puppet
+  groups: [primary-elasticsearch_kibana, elasticsearch_kibana]
+  required_for: [deploy_end]
+  requires: [lma-es-cluster]
  parameters:
    puppet_manifest: "puppet/modules/osnailyfacter/modular/virtual_ips/virtual_ips.pp"
    puppet_modules: "puppet/modules"
    timeout: 3600

- id: es-hiera-override
+- id: lma-es-hiera
  type: puppet
-  groups: [elasticsearch_kibana]
+  groups: [primary-elasticsearch_kibana, elasticsearch_kibana]
  requires: [globals]
  required_for: [logging]
  parameters:
@ -40,10 +74,10 @@
    puppet_modules: "puppet/modules"
    timeout: 120

- id: es-kibana-haproxy
+- id: lma-es-kibana-haproxy
  type: puppet
-  groups: [elasticsearch_kibana]
-  requires: [cluster-haproxy, es-virtual-ip]
+  groups: [primary-elasticsearch_kibana, elasticsearch_kibana]
+  requires: [cluster-haproxy, lma-es-virtual-ip]
  required_for: [deploy_end]
  parameters:
    puppet_manifest: "puppet/manifests/haproxy.pp"
--- a/network_roles.yaml
+++ b/network_roles.yaml
@ -9,3 +9,4 @@
         alias: "elasticsearch"
         node_roles:
           - "elasticsearch_kibana"
+           - "primary-elasticsearch_kibana"
--- a/node_roles.yaml
+++ b/node_roles.yaml
@ -1,7 +1,7 @@
 elasticsearch_kibana:
  name: 'Elasticsearch Kibana'
  description: 'Install Elasticsearch and Kibana'
-  has_primary: false
+  has_primary: true
  public_ip_required: false
  weight: 100
  limits:
--- a/6
+++ b/6
@ -8,6 +8,7 @@ CONCAT_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-concat-1
 STDLIB_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-stdlib-4.7.0.tar.gz"
 NGINX_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/jfryman-nginx-0.2.2.tar.gz"
 ELASTICSEARCH_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/elasticsearch-elasticsearch-0.9.1.tar.gz"
+FIREWALL_TARBALL_URL="https://forgeapi.puppetlabs.com/v3/files/puppetlabs-firewall-1.7.2.tar.gz"
 FUEL_LIB_COMMIT="stable/8.0"
 FUEL_LIB_TARBALL_URL="https://github.com/openstack/fuel-library/archive/${FUEL_LIB_COMMIT}.tar.gz"

@ -36,6 +37,7 @@ download_puppet_module "concat" "$CONCAT_TARBALL_URL"
 download_puppet_module "stdlib" "$STDLIB_TARBALL_URL"
 download_puppet_module "nginx" "$NGINX_TARBALL_URL"
 download_puppet_module "elasticsearch" "$ELASTICSEARCH_TARBALL_URL"
+download_puppet_module "firewall" "$FIREWALL_TARBALL_URL"

 # Untar kibana
 KIBANA_FOLDER="${MODULES_DIR}/lma_logging_analytics/files/kibana/src"
@ -43,10 +45,10 @@ mkdir -p "${KIBANA_FOLDER}"
 wget -qO- "${KIBANA_TARBALL_URL}" | tar -C "${KIBANA_FOLDER}" --strip-components=1 -xz

 # Extract dependent manifests from fuel-library
-rm -rf "${MODULES_DIR:?}"/{l23network,osnailyfacter,cluster,pacemaker_wrappers,pacemaker,openstack,haproxy}
+rm -rf "${MODULES_DIR:?}"/{l23network,osnailyfacter,cluster,pacemaker_wrappers,pacemaker,openstack,haproxy,corosync}
 wget -qO- "${FUEL_LIB_TARBALL_URL}" | \
    tar -C "${MODULES_DIR}" --strip-components=3 -zxvf - \
-    fuel-library-${FUEL_LIB_COMMIT/\//-}/deployment/puppet/{l23network,osnailyfacter,cluster,pacemaker_wrappers,pacemaker,openstack,haproxy}
+    fuel-library-${FUEL_LIB_COMMIT/\//-}/deployment/puppet/{l23network,osnailyfacter,cluster,pacemaker_wrappers,pacemaker,openstack,haproxy,corosync}

 # Remove the dead link : osnailyfacter/modular/master -> ../nailgun/examples
 rm -rf ${MODULES_DIR}/osnailyfacter/modular/master
--- a/tasks.yaml
+++ b/tasks.yaml
@ -1,5 +1,5 @@
 # This task is executed before any real deployment task
- role: ['elasticsearch_kibana']
+- role: ['primary-elasticsearch_kibana']
  stage: post_deployment/8000
  type: puppet
  parameters:
@ -7,16 +7,7 @@
    puppet_modules: puppet/modules
    timeout: 300

-# The following tasks are executed in the order they are declared
- role: ['elasticsearch_kibana']
-  stage: post_deployment/8100
-  type: puppet
-  parameters:
-    puppet_manifest: puppet/manifests/firewall.pp
-    puppet_modules: /etc/puppet/modules
-    timeout: 300
-
- role: ['elasticsearch_kibana']
+- role: ['elasticsearch_kibana', 'primary-elasticsearch_kibana']
  stage: post_deployment/8100
  type: puppet
  parameters:
@ -24,7 +15,7 @@
    puppet_modules: puppet/modules
    timeout: 600

- role: ['elasticsearch_kibana']
+- role: ['elasticsearch_kibana', 'primary-elasticsearch_kibana']
  stage: post_deployment/8100
  type: puppet
  parameters: