Remove /etc/ceph dependency and add tripleo_ceph_client role

This change introduces a new CephConfigPath parameter that can be used
by all the OpenStack clients when looking for Ceph client related info
(ceph.conf and keyrings).
By overriding this parameter we can make the containers able to pull
data from different path than /etc/ceph wich was hardcoded.
On top of this change, a new bool is added to prevent the ceph-ansible
client role being executed.
When this boolean is true, the 'ceph_client' tag is added to the list
of tags that should be skipped in ceph-ansible.
By doing this, ceph-ansible won't run the client role [1] and the new,
tripleo_ceph_client role is imported and executed.

[1] https://github.com/ceph/ceph-ansible/blob/master/site-container.yml.sample#L269

Depends-On: Iaabb66cd26f0246defe391a4e34f4eab3c3c5fee
Depends-On: Ia60bc6d5d1a04bd560f2fcb05a4b64078015ae9d

Change-Id: I36673367411cc8d68ffb9ec4a2fbff64ebf12f29

deployment/ceph-ansible/ceph-base.yaml

@@ -333,6 +333,17 @@ parameters:
   EnableInternalTLS:
     type: boolean
     default: false
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
+  CephAnsibleSkipClient:
+    description: |
+      This boolean (when true) prevents the ceph-ansible client role execution
+      by adding the ceph-ansible tag 'ceph_client' to the --skip-tags list.
+    type: boolean
+    default: false
 
 parameter_groups:
 - label: deprecated
@@ -382,6 +393,7 @@ conditions:
       - {get_param: [ServiceData, net_ip_version_map, {get_param: [ServiceNetMap, CephMonNetwork]}]}
       - 6
   internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
+  ceph_ansible_client_disabled: {equals: [{get_param: CephAnsibleSkipClient}, true]}
 
 resources:
   ContainerImageUrlParts:
@@ -665,13 +677,22 @@ outputs:
           block:
             - name: set ceph-ansible facts
               set_fact:
-                ceph_ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
-                ceph_ansible_playbooks_param: {get_param: CephAnsiblePlaybook}
-                ceph_ansible_environment_variables:
+                map_merge:
+                  - if:
+                    - ceph_ansible_client_disabled
+                    -
+                      ceph_ansible_skip_tags:
+                        list_join:
+                          - ','
+                          - - get_param: CephAnsibleSkipTags
+                          - - 'ceph_client'
+                    - ceph_ansible_skip_tags: {get_param: CephAnsibleSkipTags}
+                  - ceph_ansible_playbook_verbosity: {get_param: CephAnsiblePlaybookVerbosity}
+                  - ceph_ansible_playbooks_param: {get_param: CephAnsiblePlaybook}
+                  - ceph_ansible_environment_variables:
                       yaql:
                         data: {get_param: CephAnsibleEnvironmentVariables}
                         expression: $.data.items().select($.join('='))
-                ceph_ansible_skip_tags: {get_param:  CephAnsibleSkipTags}
             - name: Refresh ceph-ansible group_vars variables
               include_role:
                 name: tripleo_ceph_work_dir
@@ -687,6 +708,12 @@ outputs:
             - name: run ceph-ansible
               include_role:
                 name: tripleo_ceph_run_ansible
+            - name: configure ceph clients
+              include_role:
+                name: tripleo_ceph_client
+              vars:
+                tripleo_ceph_client_config_home: {get_param: CephConfigPath}
+              when: {get_param: CephAnsibleSkipClient}
             - name: ensure ceph health is OK before proceeding
               import_role:
                 role: ceph

deployment/cinder/cinder-backup-pacemaker-puppet.yaml

@@ -83,6 +83,11 @@ parameters:
     default:
       tag: openstack.cinder.backup
       file: /var/log/containers/cinder/cinder-backup.log
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   docker_enabled: {equals: [{get_param: ContainerCli}, 'docker']}
@@ -138,6 +143,7 @@ outputs:
             tripleo::profile::pacemaker::cinder::backup_bundle::docker_volumes: {get_attr: [CinderCommon, cinder_backup_volumes]}
             tripleo::profile::pacemaker::cinder::backup_bundle::docker_environment: {get_attr: [CinderCommon, cinder_backup_environment]}
             tripleo::profile::pacemaker::cinder::backup_bundle::container_backend: {get_param: ContainerCli}
+            tripleo::profile::pacemaker::cinder::backup_bundle::ceph_conf_path: {get_param: CephConfigPath}
             cinder::backup::manage_service: false
             cinder::backup::enabled: false
       service_config_settings:

deployment/cinder/cinder-common-container-puppet.yaml

@@ -73,6 +73,11 @@ parameters:
                  API network is using TLS.
     type: boolean
     default: false
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
 
@@ -103,7 +108,7 @@ outputs:
           - { 'path': /var/lib/cinder, 'setype': container_file_t }
       - name: ensure ceph configurations exist
         file:
-          path: /etc/ceph
+          path: {get_param: CephConfigPath}
           state: directory
 
   cinder_common_volumes:
@@ -209,7 +214,11 @@ outputs:
         -
           - /var/lib/kolla/config_files/cinder_volume.json:/var/lib/kolla/config_files/config.json:ro
           - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
-          - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+          - list_join:
+            - ':'
+            - - {get_param: CephConfigPath}
+            - - '/var/lib/kolla/config_files/src-ceph'
+            - - 'ro'
           - /lib/modules:/lib/modules:ro
           - /dev/:/dev/
           - /run/:/run/
@@ -243,7 +252,11 @@ outputs:
         -
           - /var/lib/kolla/config_files/cinder_backup.json:/var/lib/kolla/config_files/config.json:ro
           - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
-          - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+          - list_join:
+            - ':'
+            - - {get_param: CephConfigPath}
+            - - '/var/lib/kolla/config_files/src-ceph'
+            - - 'ro'
           - /dev/:/dev/
           - /run/:/run/
           - /sys:/sys

deployment/cinder/cinder-volume-container-puppet.yaml

@@ -176,6 +176,11 @@ parameters:
                  API network is using TLS.
     type: boolean
     default: false
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
 
@@ -240,6 +245,7 @@ outputs:
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_extra_pools: {get_param: CinderRbdExtraPools}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_secret_uuid: {get_param: CephClusterFSID}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_user_name: {get_param: CephClientUserName}
+            tripleo::profile::base::cinder::volume::cinder_rbd_ceph_conf_path: {get_param: CephConfigPath}
             tripleo::profile::base::cinder::volume::rbd::cinder_rbd_flatten_volume_from_snapshot: {get_param: CinderRbdFlattenVolumeFromSnapshot}
             # NOTE: bind IP is found in hiera replacing the network name with the local node IP
             # for the given network; replacement examples (eg. for internal_api):

deployment/cinder/cinder-volume-pacemaker-puppet.yaml

@@ -71,6 +71,11 @@ parameters:
     default:
       tag: openstack.cinder.volume
       file: /var/log/containers/cinder/cinder-volume.log
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   docker_enabled: {equals: [{get_param: ContainerCli}, 'docker']}
@@ -125,6 +130,7 @@ outputs:
             tripleo::profile::pacemaker::cinder::volume_bundle::docker_volumes: {get_attr: [CinderCommon, cinder_volume_volumes]}
             tripleo::profile::pacemaker::cinder::volume_bundle::docker_environment: {get_attr: [CinderCommon, cinder_volume_environment]}
             tripleo::profile::pacemaker::cinder::volume_bundle::container_backend: {get_param: ContainerCli}
+            tripleo::profile::pacemaker::cinder::volume_bundle::ceph_conf_path: {get_param: CephConfigPath}
             cinder::volume::manage_service: false
             cinder::volume::enabled: false
             cinder::backend_host: hostgroup

deployment/glance/glance-api-container-puppet.yaml

@@ -276,6 +276,11 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
   MultipathdEnable:
     default: false
     description: Whether to enable the multipath daemon
@@ -505,6 +510,7 @@ outputs:
             tripleo::profile::base::glance::api::glance_backend_id: {get_param: GlanceBackendID}
             tripleo::profile::base::glance::api::glance_store_description: {get_param: GlanceStoreDescription}
             tripleo::profile::base::glance::api::multistore_config: {get_param: GlanceMultistoreConfig}
+            tripleo::profile::base::glance::backend::rbd::glance_rbd_ceph_conf_path: {get_param: CephConfigPath}
             glance::notify::rabbitmq::notification_driver: {get_param: NotificationDriver}
           -
             if:
@@ -645,7 +651,11 @@ outputs:
                 -
                   - /var/lib/kolla/config_files/glance_api.json:/var/lib/kolla/config_files/config.json
                   - /var/lib/config-data/puppet-generated/glance_api:/var/lib/kolla/config_files/src:ro
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
                   - /var/lib/glance:/var/lib/glance:slave
                 -
                   if:
@@ -731,7 +741,7 @@ outputs:
             when: glance_staging_nfs_share != ''
           - name: ensure ceph configurations exist
             file:
-              path: /etc/ceph
+              path: {get_param: CephConfigPath}
               state: directory
           - name: ensure /var/lib/glance exists
             file:

deployment/gnocchi/gnocchi-api-container-puppet.yaml

@@ -122,6 +122,11 @@ parameters:
     default: ''
     description: Indicate whether this resource may be shared with the domain received in the request
                  "origin" header.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   cors_allowed_origin_unset: {equals : [{get_param: GnocchiCorsAllowedOrigin}, '']}
@@ -215,6 +220,7 @@ outputs:
                   $NETWORK: {get_param: [ServiceNetMap, GnocchiApiNetwork]}
             tripleo::profile::base::gnocchi::api::gnocchi_backend: {get_param: GnocchiBackend}
             tripleo::profile::base::gnocchi::api::incoming_storage_driver: {get_param: GnocchiIncomingStorageDriver}
+            tripleo::profile::base::gnocchi::api::gnocchi_rbd_ceph_conf_path: {get_param: CephConfigPath}
             # NOTE: bind IP is found in hiera replacing the network name with the
             # local node IP for the given network; replacement examples
             # (eg. for internal_api):
@@ -360,7 +366,11 @@ outputs:
                             - 'shared,z'
                   - /var/log/containers/gnocchi:/var/log/gnocchi:z
                   - /var/log/containers/httpd/gnocchi-api:/var/log/httpd:z
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
             environment:
               KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
               TRIPLEO_DEPLOY_IDENTIFIER: {get_param: DeployIdentifier}
@@ -389,7 +399,11 @@ outputs:
                   - /var/lib/config-data/puppet-generated/gnocchi:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi:z
                   - /var/log/containers/httpd/gnocchi-api:/var/log/httpd:z
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
                 - if:
                     - internal_tls_enabled
                     - - /etc/pki/tls/certs/httpd:/etc/pki/tls/certs/httpd:ro
@@ -430,7 +444,7 @@ outputs:
             setype: container_file_t
         - name: ensure ceph configurations exist
           file:
-            path: /etc/ceph
+            path: {get_param: CephConfigPath}
             state: directory
       upgrade_tasks: []
       metadata_settings:

deployment/gnocchi/gnocchi-metricd-container-puppet.yaml

@@ -69,6 +69,11 @@ parameters:
     default: 30
     description: Delay between processing metrics.
     type: number
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   nfs_backend_enabled: {equals: [{get_param: GnocchiNfsEnabled}, true]}
@@ -154,7 +159,11 @@ outputs:
                   - /var/lib/kolla/config_files/gnocchi_metricd.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/gnocchi:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi:z
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
                   - str_replace:
                       template: GNOCCHI_FILE_BASE_PATH:GNOCCHI_FILE_BASE_PATH:SE_FLAGS
                       params:
@@ -182,7 +191,7 @@ outputs:
             setype: container_file_t
         - name: ensure ceph configurations exist
           file:
-            path: /etc/ceph
+            path: {get_param: CephConfigPath}
             state: directory
       upgrade_tasks: []
       external_upgrade_tasks:

deployment/gnocchi/gnocchi-statsd-container-puppet.yaml

@@ -61,6 +61,11 @@ parameters:
   MonitoringSubscriptionGnocchiStatsd:
     default: 'overcloud-gnocchi-statsd'
     type: string
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   nfs_backend_enabled: {equals: [{get_param: GnocchiNfsEnabled}, true]}
@@ -146,7 +151,11 @@ outputs:
                   - /var/lib/kolla/config_files/gnocchi_statsd.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/gnocchi:/var/lib/kolla/config_files/src:ro
                   - /var/log/containers/gnocchi:/var/log/gnocchi:z
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
                   - str_replace:
                       template: GNOCCHI_FILE_BASE_PATH:GNOCCHI_FILE_BASE_PATH:SE_FLAGS
                       params:
@@ -174,7 +183,7 @@ outputs:
             setype: container_file_t
         - name: ensure ceph configurations exist
           file:
-            path: /etc/ceph
+            path: {get_param: CephConfigPath}
             state: directory
       upgrade_tasks: []
       external_upgrade_tasks:

deployment/manila/manila-backend-cephfs.yaml

@@ -39,6 +39,11 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
   # CephFS backend params:
   ManilaCephFSDataPoolName:
     default: manila_data
@@ -89,6 +94,7 @@ outputs:
         manila::backend::cephfs::driver_handles_share_servers: {get_param: ManilaCephFSDriverHandlesShareServers}
         manila::backend::cephfs::pool_name: {get_param: ManilaCephFSDataPoolName}
         manila::backend::cephfs::share_backend_name: {get_param: ManilaCephFSShareBackendName}
+        manila_cephfs_ceph_conf_path: {get_param: CephConfigPath}
         manila::backend::cephfs::cephfs_conf_path:
           list_join:
           - ''

deployment/manila/manila-share-common.yaml

@@ -38,6 +38,11 @@ parameters:
     type: string
     constraints:
     - allowed_values: ['CEPHFS', 'NFS']
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
 
@@ -57,7 +62,11 @@ outputs:
         -
           - /var/lib/kolla/config_files/manila_share.json:/var/lib/kolla/config_files/config.json:ro
           - /var/lib/config-data/puppet-generated/manila:/var/lib/kolla/config_files/src:ro
-          - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+          - list_join:
+            - ':'
+            - - {get_param: CephConfigPath}
+            - - '/var/lib/kolla/config_files/src-ceph'
+            - - 'ro'
           - /dev/:/dev/
           - /run/:/run/
           - /sys:/sys

deployment/manila/manila-share-container-puppet.yaml

@@ -48,6 +48,11 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
   MonitoringSubscriptionManilaShare:
     default: 'overcloud-manila-share'
     type: string
@@ -188,6 +193,6 @@ outputs:
             - { 'path': /var/lib/manila, 'setype': container_file_t }
         - name: ensure ceph configurations exist
           file:
-            path: /etc/ceph
+            path: {get_param:CephConfigPath}
             state: directory
       upgrade_tasks: []

deployment/manila/manila-share-pacemaker-puppet.yaml

@@ -66,6 +66,11 @@ parameters:
     description: >
       Setting this to a unique value will re-run any deployment tasks which
       perform configuration on a Heat stack-update.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
   docker_enabled: {equals: [{get_param: ContainerCli}, 'docker']}
@@ -117,6 +122,7 @@ outputs:
             tripleo::profile::pacemaker::manila::share_bundle::docker_volumes: {get_attr: [ManilaCommon, manila_share_volumes]}
             tripleo::profile::pacemaker::manila::share_bundle::docker_environment: {get_attr: [ManilaCommon, manila_share_environment]}
             tripleo::profile::pacemaker::manila::share_bundle::container_backend: {get_param: ContainerCli}
+            tripleo::profile::pacemaker::manila::share_bundle::ceph_conf_path: { get_param: CephConfigPath}
             manila::share::manage_service: false
             manila::share::enabled: false
             manila::host: hostgroup
@@ -171,7 +177,7 @@ outputs:
             - { 'path': /var/lib/manila, 'setype': container_file_t }
         - name: ensure ceph configurations exist
           file:
-            path: /etc/ceph
+            path: {get_param: CephConfigPath}
             state: directory
       deploy_steps_tasks:
         - name: Manila Share tag container image for pacemaker

deployment/nova/nova-compute-container-puppet.yaml

@@ -57,6 +57,11 @@ parameters:
       description: >
         The Ceph cluster name must be at least 1 character and contain only
         letters and numbers.
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
   NovaComputeOptVolumes:
     default: []
     description: list of optional volumes to be mounted
@@ -946,7 +951,11 @@ outputs:
                       - /var/lib/kolla/config_files/nova_compute.json:/var/lib/kolla/config_files/config.json:ro
                       - /var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro
                       - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro
-                      - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                      - list_join:
+                        - ':'
+                        - - {get_param: CephConfigPath}
+                        - - '/var/lib/kolla/config_files/src-ceph'
+                        - - 'ro'
                       - /dev:/dev
                       - /lib/modules:/lib/modules:ro
                       - /run:/run
@@ -1094,7 +1103,7 @@ outputs:
                   daemon_reload: yes
           - name: ensure ceph configurations exist
             file:
-              path: /etc/ceph
+              path: {get_param: CephConfigPath}
               state: directory
           - name: is Instance HA enabled
             set_fact:

deployment/nova/nova-libvirt-container-puppet.yaml

@@ -236,6 +236,11 @@ parameters:
     type: comma_delimited_list
     tags:
       - role_specific
+  CephConfigPath:
+    type: string
+    default: "/etc/ceph"
+    description: |
+      The path where the Ceph Cluster config files are stored on the host.
 
 conditions:
 
@@ -382,6 +387,7 @@ outputs:
                 - {get_param: CephClusterName}
                 - '.conf'
             nova::compute::rbd::libvirt_rbd_user: {get_param: CephClientUserName}
+            tripleo::profile::base::nova::compute_libvirt_shared::nova_rbd_ceph_conf_path: {get_param: CephConfigPath}
             nova::compute::rbd::rbd_keyring:
               list_join:
               - '.'
@@ -672,7 +678,11 @@ outputs:
                   - /var/lib/kolla/config_files/nova_libvirt.json:/var/lib/kolla/config_files/config.json:ro
                   - /var/lib/config-data/puppet-generated/nova_libvirt:/var/lib/kolla/config_files/src:ro
                   - /var/lib/container-config-scripts/nova_libvirt_launcher.sh:/nova_libvirt_launcher.sh:ro
-                  - /etc/ceph:/var/lib/kolla/config_files/src-ceph:ro
+                  - list_join:
+                    - ':'
+                    - - {get_param: CephConfigPath}
+                    - - '/var/lib/kolla/config_files/src-ceph'
+                    - - 'ro'
                   - /lib/modules:/lib/modules:ro
                   - /dev:/dev
                   - /run:/run
@@ -827,7 +837,7 @@ outputs:
                 seuser: system_u
             - name: ensure ceph configurations exist
               file:
-                path: /etc/ceph
+                path: {get_param: CephConfigPath}
                 state: directory
             - name: check if libvirt is installed
               command: /usr/bin/rpm -q libvirt-daemon