Add support for cephfs+ganesha backend in manila

Change-Id: I1262faef895df0023a225ab10dfe66dfb3d3efba
Partially-Implements: bluprint nfs-ganesha

manifests/profile/base/manila/api.pp

@@ -84,7 +84,7 @@ class tripleo::profile::base::manila::api (
       $cifs_protocol = undef
     }
     if $backend_cephfs_enabled {
-      $cephfs_protocol = 'CEPHFS'
+      $cephfs_protocol = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', 'CEPHFS')
     } else {
       $cephfs_protocol = undef
     }

manifests/profile/base/manila/share.pp

@@ -94,12 +94,37 @@ class tripleo::profile::base::manila::share (
       include ::manila::volume::cinder
     }
 
-    # manila cephfsnative:
+    # manila cephfs:
     if $backend_cephfs_enabled {
+      $manila_cephfs_protocol_helper_type = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', false)
+      if $manila_cephfs_protocol_helper_type {
+        # new manila ceph driver is renamed from 'cephfsnative' to 'cephfs'
+        # and supports both direct cephfs access or access through
+        # nfs-ganesha depending whether 'cephfs_protocol_helper_type' is
+        # set to 'CEPHFS' or 'NFS'
+        $manila_cephfs_backend = hiera('manila::backend::cephfs::title')
+        $cephfs_auth_id = hiera('manila::backend::cephfs::cephfs_auth_id')
+        manila::backend::cephfs { $manila_cephfs_backend :
+          driver_handles_share_servers => hiera('manila::backend::cephfs::driver_handles_share_servers', false),
+          share_backend_name           => hiera('manila::backend::cephfs::share_backend_name'),
+          cephfs_conf_path             => hiera('manila::backend::cephfs::cephfs_conf_path'),
+          cephfs_auth_id               => $cephfs_auth_id,
+          cephfs_cluster_name          => hiera('manila::backend::cephfs::cephfs_cluster_name'),
+          cephfs_enable_snapshots      => hiera('manila::backend::cephfs::cephfs_enable_snapshots'),
+          cephfs_protocol_helper_type  => $manila_cephfs_protocol_helper_type,
+          cephfs_ganesha_server_ip     => hiera('ganesha_vip', undef),
+        }
+        if $manila_cephfs_protocol_helper_type == 'NFS' {
+          manila_config {
+            "${manila_cephfs_backend}/ganesha_rados_store_enable":    value => true;
+            "${manila_cephfs_backend}/ganesha_rados_store_pool_name": value => 'manila_data';
+          }
+        }
+      } else {
+        # for backward compatibility with older heat templates which used
+        # 'cephfsnative' driver
         $manila_cephfsnative_backend = hiera('manila::backend::cephfsnative::title')
         $cephfs_auth_id = hiera('manila::backend::cephfsnative::cephfs_auth_id')
-      $keyring_path = "/etc/ceph/ceph.client.${cephfs_auth_id}.keyring"
-
         manila::backend::cephfsnative { $manila_cephfsnative_backend :
           driver_handles_share_servers => hiera('manila::backend::cephfsnative::driver_handles_share_servers', false),
           share_backend_name           => hiera('manila::backend::cephfsnative::share_backend_name'),
@@ -108,7 +133,9 @@ class tripleo::profile::base::manila::share (
           cephfs_cluster_name          => hiera('manila::backend::cephfsnative::cephfs_cluster_name'),
           cephfs_enable_snapshots      => hiera('manila::backend::cephfsnative::cephfs_enable_snapshots'),
         }
+      }
 
+      $keyring_path = "/etc/ceph/ceph.client.${cephfs_auth_id}.keyring"
       ceph_config {
         "client.${cephfs_auth_id}/keyring": value => $keyring_path;
         "client.${cephfs_auth_id}/client mount uid": value => 0;
@@ -207,6 +234,7 @@ class tripleo::profile::base::manila::share (
       [
         $manila_generic_backend,
         $manila_cephfsnative_backend,
+        $manila_cephfs_backend,
         $manila_netapp_backend,
         $manila_vmax_backend,
         $manila_isilon_backend,

manifests/profile/pacemaker/ceph_nfs.pp

@@ -0,0 +1,124 @@
+# Copyright 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# == Class: tripleo::profile::pacemaker::ceph_nfs
+#
+# Ganesha Pacemaker HA profile for tripleo
+#
+# === Parameters
+#
+# [*bootstrap_node*]
+#   (Optional) The hostname of the node responsible for bootstrapping tasks
+#   Defaults to hiera('manila_share_short_bootstrap_node_name')
+#
+# [*step*]
+#   (Optional) The current step in deployment. See tripleo-heat-templates
+#   for more details.
+#   Defaults to hiera('step')
+#
+# [*pcs_tries*]
+#   (Optional) The number of times pcs commands should be retried.
+#   Defaults to hiera('pcs_tries', 20)
+#
+class tripleo::profile::pacemaker::ceph_nfs (
+  $bootstrap_node          = hiera('manila_share_short_bootstrap_node_name'),
+  $step                    = hiera('step'),
+  $pcs_tries               = hiera('pcs_tries', 20),
+) {
+  if $::hostname == downcase($bootstrap_node) {
+    $pacemaker_master = true
+  } else {
+    $pacemaker_master = false
+  }
+
+  $ganesha_vip = hiera('ganesha_vip')
+  # NB: Until the IPaddr2 RA has a fix for https://bugzilla.redhat.com/show_bug.cgi?id=1445628
+  # we need to specify the nic when creating the ipv6 vip.
+  if is_ipv6_address($ganesha_vip) {
+    $netmask        = '128'
+    $nic            = interface_for_ip($ganesha_vip)
+    $ipv6_addrlabel = '99'
+  } else {
+    $netmask        = '32'
+    $nic            = ''
+    $ipv6_addrlabel = ''
+  }
+
+
+  Service <| tag == 'ceph-nfs' |> {
+    hasrestart => true,
+    restart    => '/bin/true',
+    start      => '/bin/true',
+    stop       => '/bin/true',
+  }
+
+  if $step >= 2 {
+    pacemaker::property { 'ceph-nfs-role-node-property':
+      property => 'ceph-nfs-role',
+      value    => true,
+      tries    => $pcs_tries,
+      node     => $::hostname,
+    }
+    if $pacemaker_master {
+      pacemaker::resource::ip { 'ganesha_vip':
+        ip_address    => $ganesha_vip,
+        cidr_netmask  => $netmask,
+        nic           => $nic,
+        #ipv6_addrlabel => $ipv6_addrlabel,
+        #ipv6_addrlabel => '',
+        tries         => $pcs_tries,
+        location_rule => {
+          resource_discovery => 'exclusive',
+          score              => 0,
+          expression         => ['ceph-nfs-role eq true'],
+        },
+      }
+    }
+  }
+
+  if $step >= 5 and $pacemaker_master {
+    pacemaker::resource::service { 'ceph-nfs' :
+      service_name  => 'ceph-nfs@pacemaker',
+      op_params     => 'start timeout=200s stop timeout=200s',
+      tries         => $pcs_tries,
+      location_rule => {
+        resource_discovery => 'exclusive',
+        score              => 0,
+        expression         => ['ceph-nfs-role eq true'],
+      },
+    }
+
+    pacemaker::constraint::colocation { 'ganesha_vip-with-ganesha':
+      source => "ip-${ganesha_vip}",
+      target => 'ceph-nfs',
+      score  => 'INFINITY',
+      tries  => $pcs_tries,
+    }
+
+    pacemaker::constraint::order { 'gansha_vip-then-ganesha':
+      first_resource    => "ip-${ganesha_vip}",
+      second_resource   => 'ceph-nfs',
+      first_action      => 'start',
+      second_action     => 'start',
+      constraint_params => 'kind=Optional',
+      tries             => $pcs_tries,
+      tag               => 'pacemaker_constraint',
+    }
+
+    Pacemaker::Resource::Ip['ganesha_vip']
+      -> Pacemaker::Resource::Service['ceph-nfs']
+        -> Pacemaker::Constraint::Order['gansha_vip-then-ganesha']
+          -> Pacemaker::Constraint::Colocation['ganesha_vip-with-ganesha']
+  }
+}

manifests/profile/pacemaker/manila/share_bundle.pp

@@ -18,6 +18,10 @@
 #
 # === Parameters
 #
+# [*backend_cephfs_enabled*]
+#   (Optional) Whether or not the cephfs backend is enabled
+#   Defaults to hiera('manila_backend_cephfs_enabled', false)
+#
 # [*manila_share_docker_image*]
 #   (Optional) The docker image to use for creating the pacemaker bundle
 #   Defaults to hiera('tripleo::profile::pacemaker::manila::share_bundle::manila_docker_image', undef)
@@ -39,6 +43,7 @@
 class tripleo::profile::pacemaker::manila::share_bundle (
   $bootstrap_node             = hiera('manila_share_short_bootstrap_node_name'),
   $manila_share_docker_image  = hiera('tripleo::profile::pacemaker::manila::share_bundle::manila_share_docker_image', undef),
+  $backend_cephfs_enabled     = hiera('manila_backend_cephfs_enabled', false),
   $pcs_tries                  = hiera('pcs_tries', 20),
   $step                       = Integer(hiera('step')),
 ) {
@@ -67,18 +72,7 @@ class tripleo::profile::pacemaker::manila::share_bundle (
     if $pacemaker_master {
       $manila_share_nodes_count = count(hiera('manila_share_short_node_names', []))
 
-      pacemaker::resource::bundle { $::manila::params::share_service:
+      $default_storage_maps = {
-        image             => $manila_share_docker_image,
-        replicas          => 1,
-        location_rule     => {
-          resource_discovery => 'exclusive',
-          score              => 0,
-          expression         => ['manila-share-role eq true'],
-        },
-        container_options => 'network=host',
-        options           => '--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS',
-        run_command       => '/bin/bash /usr/local/bin/kolla_start',
-        storage_maps      => {
         'manila-share-cfg-files'      => {
           'source-dir' => '/var/lib/kolla/config_files/manila_share.json',
           'target-dir' => '/var/lib/kolla/config_files/config.json',
@@ -134,7 +128,69 @@ class tripleo::profile::pacemaker::manila::share_bundle (
           'target-dir' => '/etc/ceph',
           'options'    => 'ro',
         },
+      }
+
+      # if ceph-nfs backend is used, then DBus is used for dynamic
+      # creation of NFS exports and DBus socket has to be mounted
+      # both to manila-share and ganesha containers so they can talk
+      # to each other
+      $manila_cephfs_protocol_helper_type = hiera('manila::backend::cephfs::cephfs_protocol_helper_type', '')
+      $nfs_ganesha = ($backend_cephfs_enabled and $manila_cephfs_protocol_helper_type == 'NFS')
+      if $nfs_ganesha {
+        $extra_storage_maps = {
+          'dbus-docker'                => {
+            'source-dir' => '/var/run/dbus/system_bus_socket',
+            'target-dir' => '/var/run/dbus/system_bus_socket',
+            'options'    => 'rw',
           },
+          'etc-ganesha'                => {
+            'source-dir' => '/etc/ganesha',
+            'target-dir' => '/etc/ganesha',
+            'options'    => 'rw',
+          },
+        }
+      } else {
+        $extra_storage_maps = {}
+      }
+
+      $storage_maps = merge($default_storage_maps, $extra_storage_maps)
+
+      pacemaker::resource::bundle { $::manila::params::share_service:
+        image             => $manila_share_docker_image,
+        replicas          => 1,
+        location_rule     => {
+          resource_discovery => 'exclusive',
+          score              => 0,
+          expression         => ['manila-share-role eq true'],
+        },
+        container_options => 'network=host',
+        options           => '--ipc=host --privileged=true --user=root --log-driver=journald -e KOLLA_CONFIG_STRATEGY=COPY_ALWAYS',
+        run_command       => '/bin/bash /usr/local/bin/kolla_start',
+        storage_maps      => $storage_maps,
+      }
+
+      if $nfs_ganesha {
+        pacemaker::constraint::order { 'ceph-nfs-then-manila-share':
+          first_resource    => 'ceph-nfs',
+          second_resource   => 'openstack-manila-share',
+          first_action      => 'start',
+          second_action     => 'start',
+          constraint_params => 'kind=Optional',
+          tries             => $pcs_tries,
+          tag               => 'pacemaker_constraint',
+        }
+
+        pacemaker::constraint::colocation { 'openstack-manila-share-with-ceph-nfs':
+          source => 'openstack-manila-share',
+          target => 'ceph-nfs',
+          score  => 'INFINITY',
+          tries  => $pcs_tries,
+          tag    => 'pacemaker_constraint',
+        }
+
+        Pacemaker::Resource::Bundle['openstack-manila-share']
+          -> Pacemaker::Constraint::Colocation['openstack-manila-share-with-ceph-nfs']
+            -> Pacemaker::Constraint::Order['ceph-nfs-then-manila-share']
       }
     }
   }