Sahara Integration

Integration of OpenStack data processing service (sahara) with
TripleO.

- Deploys sahara in distributed mode (separate api and engine
  processes on each controller node)
- Load balancing w/haproxy
- RabbitMQ/MySQL supported per current TripleO standard
- Minimal configurability at this time

Change-Id: I77a6a69ed5691e3b1ba34e9ebb4d88c80019642c
Partially-implements: blueprint sahara-integration
Depends-On: I0f0a1dc2eaa57d8226bad8cfb250110296ab9614
Depends-On: Ib84cc59667616ec94e7edce2715cbd7dd944f4ae
Depends-On: I9fe321fd4284f7bfd55bd2e69dcfe623ed6f8a2a

network/endpoints/endpoint_map.yaml

@@ -40,6 +40,9 @@ parameters:
   SwiftProxyVirtualIP:
     type: string
     default: ''
+  SaharaApiVirtualIP:
+    type: string
+    default: ''
   EndpointMap:
     type: json
     default:
@@ -74,6 +77,9 @@ parameters:
       SwiftAdmin: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
       SwiftInternal: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
       SwiftPublic: {protocol: 'http', port: '8080', host: 'IP_ADDRESS'}
+      SaharaAdmin: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
+      SaharaInternal: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
+      SaharaPublic: {protocol: 'http', port: '8386', host: 'IP_ADDRESS'}
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
   CloudName:
@@ -404,6 +410,31 @@ resources:
       IP: {get_param: SwiftProxyVirtualIP}
       CloudName: {get_param: CloudName}
 
+  SaharaInternal:
+    type: OS::TripleO::Endpoint
+    properties:
+      EndpointName: SaharaInternal
+      EndpointMap: { get_param: EndpointMap }
+      IP: {get_param: SaharaApiVirtualIP}
+      CloudName: {get_param: CloudName}
+      UriSuffix: '/v1.1/%(tenant_id)s'
+  SaharaPublic:
+    type: OS::TripleO::Endpoint
+    properties:
+      EndpointName: SaharaPublic
+      EndpointMap: { get_param: EndpointMap }
+      IP: {get_param: SaharaApiVirtualIP}
+      CloudName: {get_param: CloudName}
+      UriSuffix: '/v1.1/%(tenant_id)s'
+  SaharaAdmin:
+    type: OS::TripleO::Endpoint
+    properties:
+      EndpointName: SaharaAdmin
+      EndpointMap: { get_param: EndpointMap }
+      IP: {get_param: SaharaApiVirtualIP}
+      CloudName: {get_param: CloudName}
+      UriSuffix: '/v1.1/%(tenant_id)s'
+
 outputs:
   endpoint_map:
     value:
@@ -448,3 +479,6 @@ outputs:
       SwiftS3Internal: {get_attr: [ SwiftS3Internal, endpoint] }
       SwiftS3Public: {get_attr: [ SwiftS3Public, endpoint] }
       SwiftS3Admin: {get_attr: [ SwiftS3Admin, endpoint] }
+      SaharaInternal: {get_attr: [ SaharaInternal, endpoint] }
+      SaharaPublic: {get_attr: [ SaharaPublic, endpoint] }
+      SaharaAdmin: {get_attr: [ SaharaAdmin, endpoint] }

overcloud.yaml

@@ -529,6 +529,12 @@ parameters:
     type: number
     default: 3
     description: How many replicas to use in the swift rings.
+  SaharaPassword:
+    description: The password for the sahara service account.
+    # TODO(egafford): Remove default on merge of https://review.openstack.org/#/c/221418/ (added to avoid circular dep)
+    default: unset
+    type: string
+    hidden: true
 
 # Compute-specific params
   CeilometerComputeAgent:
@@ -619,6 +625,7 @@ parameters:
       NovaVncProxyNetwork: internal_api
       SwiftMgmtNetwork: storage_mgmt
       SwiftProxyNetwork: storage
+      SaharaApiNetwork: internal_api
       HorizonNetwork: internal_api
       MemcachedNetwork: internal_api
       RabbitMqNetwork: internal_api
@@ -812,6 +819,7 @@ resources:
       MysqlVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
       NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
       NovaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+      SaharaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
       SwiftProxyVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
       PublicVirtualIP: {get_attr: [VipMap, net_ip_map, external]}
 
@@ -912,6 +920,7 @@ resources:
           RabbitClientUseSSL: {get_param: RabbitClientUseSSL}
           RabbitClientPort: {get_param: RabbitClientPort}
           RabbitFDLimit: {get_param: RabbitFDLimit}
+          SaharaPassword: {get_param: SaharaPassword}
           SnmpdReadonlyUserName: {get_param: SnmpdReadonlyUserName}
           SnmpdReadonlyUserPassword: {get_param: SnmpdReadonlyUserPassword}
           RedisVirtualIP: {get_attr: [RedisVirtualIP, ip_address]}
@@ -937,6 +946,7 @@ resources:
           KeystonePublicApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
           NeutronApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
           NovaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+          SaharaApiVirtualIP: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
           UpdateIdentifier: {get_param: UpdateIdentifier}
           Hostname:
             str_replace:
@@ -1158,6 +1168,7 @@ resources:
       neutron_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, NeutronApiNetwork]}]}
       keystone_public_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, KeystonePublicApiNetwork]}]}
       keystone_admin_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, KeystoneAdminApiNetwork]}]}
+      sahara_api_node_ips: {get_attr: [ControllerIpListMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
       DeployIdentifier: {get_param: DeployIdentifier}
       UpdateIdentifier: {get_param: UpdateIdentifier}
 
@@ -1270,6 +1281,7 @@ resources:
         control_virtual_ip: {get_attr: [VipMap, net_ip_map, ctlplane]}
         public_virtual_ip: {get_attr: [VipMap, net_ip_map, external]}
         internal_api_virtual_ip: {get_attr: [VipMap, net_ip_map, internal_api]}
+        sahara_api_vip: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
         storage_virtual_ip: {get_attr: [VipMap, net_ip_map, storage]}
         storage_mgmt_virtual_ip: {get_attr: [VipMap, net_ip_map, storage_mgmt]}
 
@@ -1527,6 +1539,9 @@ outputs:
   NovaInternalVip:
     description: VIP for Nova API internal endpoint
     value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, NovaApiNetwork]}]}
+  SaharaInternalVip:
+    description: VIP for Sahara API internal endpoint
+    value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
   SwiftInternalVip:
     description: VIP for Swift Proxy internal endpoint
     value: {get_attr: [VipMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}

puppet/all-nodes-config.yaml

@@ -50,6 +50,8 @@ parameters:
     type: comma_delimited_list
   keystone_admin_api_node_ips:
     type: comma_delimited_list
+  sahara_api_node_ips:
+    type: comma_delimited_list
 
   DeployIdentifier:
     type: string
@@ -241,6 +243,14 @@ resources:
                         list_join:
                         - "','"
                         - {get_param: keystone_admin_api_node_ips}
+                sahara_api_node_ips:
+                  str_replace:
+                    template: "['SERVERS_LIST']"
+                    params:
+                      SERVERS_LIST:
+                        list_join:
+                        - "','"
+                        - {get_param: sahara_api_node_ips}
 
                 # NOTE(gfidente): interpolation with %{} in the
                 # hieradata file can't be used as it returns string
@@ -250,6 +260,7 @@ resources:
                 neutron::rabbit_hosts: *rabbit_nodes_array
                 nova::rabbit_hosts: *rabbit_nodes_array
                 keystone::rabbit_hosts: *rabbit_nodes_array
+                sahara::rabbit_hosts: *rabbit_nodes_array
 
                 deploy_identifier: {get_param: DeployIdentifier}
                 update_identifier: {get_param: UpdateIdentifier}

puppet/controller.yaml

@@ -315,6 +315,14 @@ parameters:
     default: 0
     description: Number of workers for Keystone service.
     type: number
+  SaharaApiVirtualIP:
+    type: string
+    default: ''
+  SaharaPassword:
+    default: unset
+    description: The password for the sahara service account, used by sahara-api.
+    type: string
+    hidden: true
   MysqlClusterUniquePart:
     description: A unique identifier of the MySQL cluster the controller is in.
     type: string
@@ -822,6 +830,7 @@ resources:
       server: {get_resource: Controller}
       NodeIndex: {get_param: NodeIndex}
 
+
   ControllerDeployment:
     type: OS::TripleO::SoftwareDeployment
     depends_on: NetworkDeployment
@@ -1089,6 +1098,15 @@ resources:
         swift_mount_check: {get_param: SwiftMountCheck}
         enable_package_install: {get_param: EnablePackageInstall}
         enable_package_upgrade: {get_attr: [UpdateDeployment, update_managed_packages]}
+        sahara_password: {get_param: SaharaPassword}
+        sahara_dsn:
+          list_join:
+            - ''
+            - - 'mysql://sahara:'
+              - {get_param: SaharaPassword}
+              - '@'
+              - {get_param: MysqlVirtualIP}
+              - '/sahara'
         swift_proxy_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftProxyNetwork]}]}
         swift_management_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SwiftMgmtNetwork]}]}
         cinder_iscsi_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, CinderIscsiNetwork]}]}
@@ -1111,6 +1129,7 @@ resources:
         rabbitmq_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RabbitMqNetwork]}]}
         redis_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, RedisNetwork]}]}
         redis_vip: {get_param: RedisVirtualIP}
+        sahara_api_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, SaharaApiNetwork]}]}
         memcached_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MemcachedNetwork]}]}
         mysql_network: {get_attr: [NetIpMap, net_ip_map, {get_param: [ServiceNetMap, MysqlNetwork]}]}
         mysql_virtual_ip: {get_param: MysqlVirtualIP}
@@ -1424,6 +1443,29 @@ resources:
                 horizon::bind_address: {get_input: horizon_network}
                 horizon::keystone_url: {get_input: keystone_auth_uri}
 
+                # Sahara
+                sahara::host: {get_input: sahara_api_network}
+                sahara::plugins:
+                  - cdh
+                  - hdp
+                  - mapr
+                  - vanilla
+                  - spark
+                  - storm
+                sahara::admin_password: {get_input: sahara_password}
+                sahara::auth_uri: {get_input: keystone_auth_uri}
+                sahara::admin_user: sahara
+                sahara::identity_uri: {get_input: keystone_identity_uri}
+                sahara::use_neutron: true
+                sahara::database_connection: {get_input: sahara_dsn}
+                sahara::debug: {get_input: debug}
+                sahara::rpc_backend: rabbit
+                sahara::rabbit_userid: {get_input: rabbit_username}
+                sahara::rabbit_password: {get_input: rabbit_password}
+                sahara::rabbit_use_ssl: {get_input: rabbit_client_use_ssl}
+                sahara::rabbit_port: {get_input: rabbit_client_port}
+                sahara::db::mysql::password: {get_input: sahara_password}
+
                 # Rabbit
                 rabbitmq::node_ip_address: {get_input: rabbitmq_network}
                 rabbitmq::erlang_cookie: {get_input: rabbit_cookie}

puppet/hieradata/controller.yaml

@@ -38,6 +38,7 @@ cinder::api::keystone_tenant: 'service'
 swift::proxy::authtoken::admin_tenant_name: 'service'
 ceilometer::api::keystone_tenant: 'service'
 heat::keystone_tenant: 'service'
+sahara::admin_tenant_name: 'service'
 
 # keystone
 keystone::cron::token_flush::maxdelay: 3600
@@ -120,6 +121,7 @@ tripleo::loadbalancer::nova_metadata: true
 tripleo::loadbalancer::nova_novncproxy: true
 tripleo::loadbalancer::mysql: true
 tripleo::loadbalancer::redis: true
+tripleo::loadbalancer::sahara: true
 tripleo::loadbalancer::swift_proxy_server: true
 tripleo::loadbalancer::ceilometer: true
 tripleo::loadbalancer::heat_api: true

puppet/hieradata/database.yaml

@@ -53,3 +53,10 @@ ceilometer::db::mysql::dbname: ceilometer
 ceilometer::db::mysql::allowed_hosts:
   - '%'
   - "%{hiera('mysql_bind_host')}"
+
+sahara::db::mysql::user: sahara
+sahara::db::mysql::host: "%{hiera('mysql_virtual_ip')}"
+sahara::db::mysql::dbname: sahara
+sahara::db::mysql::allowed_hosts:
+  - '%'
+  - "%{hiera('mysql_bind_host')}"

puppet/manifests/overcloud_controller.pp

@@ -101,6 +101,7 @@ if hiera('step') >= 2 {
   include ::neutron::db::mysql
   include ::cinder::db::mysql
   include ::heat::db::mysql
+  include ::sahara::db::mysql
   if downcase(hiera('ceilometer_backend')) == 'mysql' {
     include ::ceilometer::db::mysql
   }
@@ -541,6 +542,11 @@ if hiera('step') >= 3 {
   include ::heat::api_cloudwatch
   include ::heat::engine
 
+  # Sahara
+  include ::sahara
+  include ::sahara::service::api
+  include ::sahara::service::engine
+
   # Horizon
   if 'cisco_n1kv' in hiera('neutron::plugins::ml2::mechanism_drivers') {
     $_profile_support = 'cisco'

puppet/manifests/overcloud_controller_pacemaker.pp

@@ -443,6 +443,10 @@ MYSQL_HOST=localhost\n",
         require => Exec['galera-ready'],
       }
     }
+
+    class { '::sahara::db::mysql':
+      require       => Exec['galera-ready'],
+    }
   }
 
   # pre-install swift here so we can build rings
@@ -852,6 +856,18 @@ if hiera('step') >= 3 {
     enabled_backends => $cinder_enabled_backends,
   }
 
+  class { '::sahara':
+    sync_db => $sync_db,
+  }
+  class { '::sahara::service::api':
+    manage_service => false,
+    enabled        => false,
+  }
+  class { '::sahara::service::engine':
+    manage_service => false,
+    enabled        => false,
+  }
+
   # swift proxy
   class { '::swift::proxy' :
     manage_service => $non_pcmk_start,
@@ -1097,6 +1113,24 @@ if hiera('step') >= 4 {
                   Pacemaker::Resource::Service[$::cinder::params::volume_service]],
     }
 
+    # Sahara
+    pacemaker::resource::service { $::sahara::params::api_service_name :
+      clone_params => 'interleave=true',
+      require      => Pacemaker::Resource::Service[$::keystone::params::service_name],
+    }
+    pacemaker::resource::service { $::sahara::params::engine_service_name :
+      clone_params => 'interleave=true',
+    }
+    pacemaker::constraint::base { 'keystone-then-sahara-api-constraint':
+      constraint_type => 'order',
+      first_resource  => "${::keystone::params::service_name}-clone",
+      second_resource => "${::sahara::params::api_service_name}-clone",
+      first_action    => 'start',
+      second_action   => 'start',
+      require         => [Pacemaker::Resource::Service[$::sahara::params::api_service_name],
+                          Pacemaker::Resource::Service[$::keystone::params::service_name]],
+    }
+
     # Glance
     pacemaker::resource::service { $::glance::params::registry_service_name :
       clone_params => 'interleave=true',

puppet/vip-config.yaml

@@ -19,6 +19,7 @@ resources:
                 cinder_api_vip: {get_input: cinder_api_vip}
                 glance_api_vip: {get_input: glance_api_vip}
                 glance_registry_vip: {get_input: glance_registry_vip}
+                sahara_api_vip: {get_input: sahara_api_vip}
                 swift_proxy_vip: {get_input: swift_proxy_vip}
                 nova_api_vip: {get_input: nova_api_vip}
                 nova_metadata_vip: {get_input: nova_metadata_vip}