logging: use service_config_settings for fluentd

The initial fluentd client implementation predates the introduction of
service_config_settings, and necessitated some invasive changes to
what is now common/serivces.yaml. This commit modifies existing
services to use the service_config_settings based configuration
mechanism supported by more recent versions of the fluentd support in
puppet-tripleo.

Partial-bug: #1715187
Depends-On: I3149902401d68d6fd236073a73a20f982d4b952a
Depends-On: I2b057190ec0e4e75ee4ee47ebe0164c2644e5ab7
Depends-On: Ie7df4b8b94cb0ae38096ab95800f211ef1cd8455
Change-Id: I28028ffa00df2da8e0478a551d3de89c3ee46e1f

ci/environments/scenario001-multinode-containers.yaml

@@ -20,6 +20,7 @@ resource_registry:
   OS::TripleO::Tasks::ControllerPreConfig: OS::Heat::None
   OS::TripleO::Tasks::ControllerPostConfig: OS::Heat::None
   OS::TripleO::Tasks::ControllerPostPuppetRestart: ../../extraconfig/tasks/post_puppet_pacemaker_restart.yaml
+  #Needs to run scenario001
   OS::TripleO::Services::Fluentd: ../../docker/services/fluentd.yaml
   OS::TripleO::Services::SensuClient: ../../docker/services/sensu-client.yaml
   # Some infra instances don't pass the ping test but are otherwise working.

puppet/services/ceilometer-agent-central.yaml

@@ -60,16 +60,20 @@ outputs:
     value:
       service_name: ceilometer_agent_central
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerCentral}
-      logging_source: {get_param: CeilometerAgentCentralLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer_redis_password: {get_param: RedisPassword}
             central_namespace: true
       service_config_settings:
-        get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+        map_merge:
+          - get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+          - fluentd:
+              tripleo_fluentd_groups_ceilometer_agent_central:
+                - ceilometer
+              tripleo_fluentd_sources_ceilometer_agent_central:
+                - {get_param: CeilometerAgentCentralLoggingSource}
+
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:

puppet/services/ceilometer-agent-ipmi.yaml

@@ -60,14 +60,17 @@ outputs:
     value:
       service_name: ceilometer_agent_ipmi
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerIpmi}
-      logging_source: {get_param: CeilometerAgentIpmiLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         map_merge:
           - get_attr: [CeilometerServiceBase, role_data, config_settings]
           - ceilometer_redis_password: {get_param: RedisPassword}
             ipmi_namespace: true
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_ceilometer_agent_ipmi:
+            - ceilometer
+          tripleo_fluentd_sources_ceilometer_agent_ipmi:
+            - {get_param: CeilometerAgentIpmiLoggingSource}
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::polling
       upgrade_tasks:

puppet/services/ceilometer-agent-notification.yaml

@@ -39,7 +39,6 @@ parameters:
       tag: openstack.ceilometer.agent.notification
       path: /var/log/ceilometer/agent-notification.log
 
-
 resources:
   CeilometerServiceBase:
     type: ./ceilometer-base.yaml
@@ -57,13 +56,16 @@ outputs:
     value:
       service_name: ceilometer_agent_notification
       monitoring_subscription: {get_param: MonitoringSubscriptionCeilometerNotification}
-      logging_source: {get_param: CeilometerAgentNotificationLoggingSource}
-      logging_groups:
-        - ceilometer
       config_settings:
         get_attr: [CeilometerServiceBase, role_data, config_settings]
       service_config_settings:
-        get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+        map_merge:
+          - get_attr: [CeilometerServiceBase, role_data, service_config_settings]
+          - fluentd:
+              tripleo_fluentd_groups_ceilometer_agent_notification:
+                - ceilometer
+              tripleo_fluentd_sources_ceilometer_agent_notification:
+                - {get_param: CeilometerAgentNotificationLoggingSource}
       step_config: |
         include ::tripleo::profile::base::ceilometer::agent::notification
       upgrade_tasks:

puppet/services/cinder-api.yaml

@@ -103,9 +103,6 @@ outputs:
     value:
       service_name: cinder_api
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderApi}
-      logging_source: {get_param: CinderApiLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
@@ -160,6 +157,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::cinder::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_api:
+            - cinder
+          tripleo_fluentd_sources_cinder_api:
+            - {get_param: CinderApiLoggingSource}
         keystone:
           cinder::keystone::auth::tenant: 'service'
           cinder::keystone::auth::public_url: {get_param: [EndpointMap, CinderPublic, uri]}

puppet/services/cinder-scheduler.yaml

@@ -57,13 +57,16 @@ outputs:
     value:
       service_name: cinder_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderScheduler}
-      logging_source: {get_param: CinderSchedulerLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
           - cinder::scheduler::scheduler_driver: cinder.scheduler.filter_scheduler.FilterScheduler
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_scheduler:
+            - cinder
+          tripleo_fluentd_sources_cinder_scheduler:
+            - {get_param: CinderSchedulerLoggingSource}
       step_config: |
         include ::tripleo::profile::base::cinder::scheduler
       upgrade_tasks:

puppet/services/cinder-volume.yaml

@@ -121,9 +121,6 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_param: MonitoringSubscriptionCinderVolume}
-      logging_source: {get_param: CinderVolumeLoggingSource}
-      logging_groups:
-        - cinder
       config_settings:
         map_merge:
           - get_attr: [CinderBase, role_data, config_settings]
@@ -149,6 +146,12 @@ outputs:
             # internal_api_uri -> [IP]
             # internal_api_subnet - > IP/CIDR
             tripleo::profile::base::cinder::volume::iscsi::cinder_iscsi_address: {get_param: [ServiceNetMap, CinderIscsiNetwork]}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_cinder_volume:
+            - cinder
+          tripleo_fluentd_sources_cinder_volume:
+            - {get_param: CinderVolumeLoggingSource}
       step_config: |
         include ::tripleo::profile::base::cinder::volume
       upgrade_tasks:

puppet/services/database/mongodb.yaml

@@ -31,10 +31,6 @@ parameters:
     description: Mapping of service endpoint -> protocol. Typically set
                  via parameter_defaults in the resource registry.
     type: json
-  MongodbMemoryLimit:
-    default: '20G'
-    description: Limit the amount of memory mongodb uses with systemd.
-    type: string
   MongoDbLoggingSource:
     type: json
     description: Fluentd logging configuration for mongodb.
@@ -44,6 +40,10 @@ parameters:
       format: >-
         /(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d+\+\d{4})
         (?<message>.*)$/
+  MongodbMemoryLimit:
+    default: '20G'
+    description: Limit the amount of memory mongodb uses with systemd.
+    type: string
   EnableInternalTLS:
     type: boolean
     default: false
@@ -73,9 +73,6 @@ outputs:
     description: Service mongodb using composable services.
     value:
       service_name: mongodb
-      logging_groups:
-        - mongodb
-      logging_source: {get_param: MongoDbLoggingSource}
       config_settings:
         map_merge:
           - get_attr: [MongoDbBase, role_data, config_settings]
@@ -123,6 +120,12 @@ outputs:
                       params:
                         NETWORK: {get_param: [ServiceNetMap, MongodbNetwork]}
               - {}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_mongodb:
+            - mongodb
+          tripleo_fluentd_sources_mongodb:
+            - {get_param: MongoDbLoggingSource}
       step_config: |
         include ::tripleo::profile::base::database::mongodb
       upgrade_tasks:

puppet/services/ec2-api.yaml

@@ -96,9 +96,6 @@ outputs:
     value:
       service_name: ec2_api
       monitoring_subscription: {get_param: MonitoringSubscriptionEc2Api}
-      logging_source: {get_param: Ec2ApiLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [TLSProxyBase, role_data, config_settings]
@@ -179,6 +176,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::ec2api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_ec2_api:
+            - nova
+          tripleo_fluentd_sources_ec2_api:
+            - {get_param: Ec2ApiLoggingSource}
         keystone:
           ec2api::keystone::auth::tenant: 'service'
           ec2api::keystone::auth::public_url: {get_param: [EndpointMap, Ec2ApiPublic, uri]}

puppet/services/glance-api.yaml

@@ -179,9 +179,6 @@ outputs:
     value:
       service_name: glance_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGlanceApi}
-      logging_source: {get_param: GlanceApiLoggingSource}
-      logging_groups:
-        - glance
       config_settings:
         map_merge:
           - get_attr: [TLSProxyBase, role_data, config_settings]
@@ -268,6 +265,11 @@ outputs:
             - {}
             - glance::api::workers: {get_param: GlanceWorkers}
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_glance_api:
+            - glance
+          tripleo_fluentd_sources_glance_api:
+            - {get_param: GlanceApiLoggingSource}
         keystone:
           glance::keystone::auth::public_url: {get_param: [EndpointMap, GlancePublic, uri]}
           glance::keystone::auth::internal_url: {get_param: [EndpointMap, GlanceInternal, uri]}

puppet/services/gnocchi-api.yaml

@@ -92,9 +92,6 @@ outputs:
     value:
       service_name: gnocchi_api
       monitoring_subscription: {get_param: MonitoringSubscriptionGnocchiApi}
-      logging_source: {get_param: GnocchiApiLoggingSource}
-      logging_groups:
-        - gnocchi
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]
@@ -133,6 +130,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::gnocchi::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_gnocchi_api:
+            - gnocchi
+          tripleo_fluentd_sources_gnocchi_api:
+            - {get_param: GnocchiApiLoggingSource}
         keystone:
           gnocchi::keystone::auth::admin_url: { get_param: [ EndpointMap, GnocchiAdmin, uri ] }
           gnocchi::keystone::auth::internal_url: {get_param: [EndpointMap, GnocchiInternal, uri]}

puppet/services/heat-api-cfn.yaml

@@ -86,9 +86,6 @@ outputs:
     value:
       service_name: heat_api_cfn
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApiCnf}
-      logging_source: {get_param: HeatApiCfnLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -121,6 +118,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::heat::api_cfn
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_api_cfn:
+            - heat
+          tripleo_fluentd_sources_heat_api_cfn:
+            - {get_param: HeatApiCfnLoggingSource}
         keystone:
           map_merge:
             - get_attr: [HeatBase, role_data, service_config_settings, keystone]

puppet/services/heat-api.yaml

@@ -92,9 +92,6 @@ outputs:
     value:
       service_name: heat_api
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatApi}
-      logging_source: {get_param: HeatApiLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -128,6 +125,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::heat::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_api:
+            - heat
+          tripleo_fluentd_sources_heat_api:
+            - {get_param: HeatApiLoggingSource}
         keystone:
           map_merge:
             - get_attr: [HeatBase, role_data, service_config_settings, keystone]

puppet/services/heat-engine.yaml

@@ -89,9 +89,6 @@ outputs:
     value:
       service_name: heat_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionHeatEngine}
-      logging_source: {get_param: HeatEngineLoggingSource}
-      logging_groups:
-        - heat
       config_settings:
         map_merge:
           - get_attr: [HeatBase, role_data, config_settings]
@@ -144,6 +141,11 @@ outputs:
         include ::tripleo::profile::base::heat::engine
 
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_heat_engine:
+            - heat
+          tripleo_fluentd_sources_heat_engine:
+            - {get_param: HeatEngineLoggingSource}
         mysql:
           heat::db::mysql::password: {get_param: HeatPassword}
           heat::db::mysql::user: heat

puppet/services/keystone.yaml

@@ -325,9 +325,6 @@ outputs:
     value:
       service_name: keystone
       monitoring_subscription: {get_param: MonitoringSubscriptionKeystone}
-      logging_source: {get_param: KeystoneLoggingSource}
-      logging_groups:
-        - keystone
       config_settings:
         map_merge:
           - get_attr: [ApacheServiceBase, role_data, config_settings]
@@ -503,6 +500,11 @@ outputs:
       step_config: |
         include ::tripleo::profile::base::keystone
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_keystone:
+            - keystone
+          tripleo_fluentd_sources_keystone:
+            - {get_param: KeystoneLoggingSource}
         mysql:
           keystone::db::mysql::password: {get_param: AdminToken}
           keystone::db::mysql::user: keystone

puppet/services/neutron-api.yaml

@@ -135,9 +135,6 @@ outputs:
     value:
       service_name: neutron_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronServer}
-      logging_source: {get_param: NeutronApiLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -205,6 +202,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::neutron::server
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_api:
+            - neutron
+          tripleo_fluentd_sources_neutron_api:
+            - {get_param: NeutronApiLoggingSource}
         keystone:
           neutron::keystone::auth::tenant: 'service'
           neutron::keystone::auth::public_url: {get_param: [EndpointMap, NeutronPublic, uri]}

puppet/services/neutron-dhcp.yaml

@@ -91,9 +91,6 @@ outputs:
     value:
       service_name: neutron_dhcp
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronDhcp}
-      logging_source: {get_param: NeutronDhcpAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -115,6 +112,12 @@ outputs:
                 proto: 'udp'
                 chain: 'OUTPUT'
                 dport: 68
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_dhcp:
+            - neutron
+          tripleo_fluentd_sources_neutron_dhcp:
+            - {get_param: NeutronDhcpAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::dhcp
       upgrade_tasks:

puppet/services/neutron-l2gw-agent.yaml

@@ -83,9 +83,6 @@ outputs:
     value:
       service_name: neutron_l2gw_agent
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL2gwAgent}
-      logging_source: {get_param: NeutronL2gwAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - neutron::agents::l2gw::ovsdb_hosts: {get_param: L2gwAgentOvsdbHosts}
@@ -108,6 +105,12 @@ outputs:
                     dport: {get_param: L2gwAgentManagerTableListeningPort}
               - null
 
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_l2gw_agent:
+            - neutron
+          tripleo_fluentd_sources_neutron_l2gw_agent:
+            - {get_param: NeutronL2gwAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::agents::l2gw
       upgrade_tasks:

puppet/services/neutron-l3-compute-dvr.yaml

@@ -75,9 +75,6 @@ outputs:
     value:
       service_name: neutron_l3_compute_dvr
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3Dvr}
-      logging_source: {get_param: NeutronL3ComputeAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -87,5 +84,11 @@ outputs:
             - external_network_bridge_empty
             - {}
             - neutron::agents::l3::external_network_bridge: {get_param: NeutronExternalNetworkBridge}
+        service_config_settings:
+          fluentd:
+            tripleo_fluentd_groups_neutron_l3_compute_dvr:
+              - neutron
+            tripleo_fluentd_sources_neutron_l3_compute_dvr:
+              - {get_param: NeutronL3ComputeAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::l3

puppet/services/neutron-l3.yaml

@@ -89,9 +89,6 @@ outputs:
     value:
       service_name: neutron_l3
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronL3}
-      logging_source: {get_param: NeutronL3AgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
         - get_attr: [NeutronBase, role_data, config_settings]
@@ -109,6 +106,12 @@ outputs:
           - external_network_bridge_empty
           - {}
           - neutron::agents::l3::external_network_bridge: {get_param: NeutronExternalNetworkBridge}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_l3:
+            - neutron
+          tripleo_fluentd_sources_neutron_l3:
+            - {get_param: NeutronL3AgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::l3
       upgrade_tasks:

puppet/services/neutron-metadata.yaml

@@ -94,9 +94,6 @@ outputs:
     value:
       service_name: neutron_metadata
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronMetadata}
-      logging_source: {get_param: NeutronMetadataAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -126,6 +123,12 @@ outputs:
             - neutron_workers_unset
             - {}
             - neutron::agents::metadata::metadata_workers: {get_param: NeutronWorkers}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_metadata:
+            - neutron
+          tripleo_fluentd_sources_neutron_metadata:
+            - {get_param: NeutronMetadataAgentLoggingSource}
       step_config: |
         include tripleo::profile::base::neutron::metadata
       upgrade_tasks:

puppet/services/neutron-ovs-agent.yaml

@@ -128,9 +128,6 @@ outputs:
     value:
       service_name: neutron_ovs_agent
       monitoring_subscription: {get_param: MonitoringSubscriptionNeutronOvs}
-      logging_source: {get_param: NeutronOpenVswitchAgentLoggingSource}
-      logging_groups:
-        - neutron
       config_settings:
         map_merge:
           - get_attr: [NeutronBase, role_data, config_settings]
@@ -158,6 +155,12 @@ outputs:
             - no_firewall_driver
             - {}
             - neutron::agents::ml2::ovs::firewall_driver: {get_param: NeutronOVSFirewallDriver}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_neutron_ovs_agent:
+            - neutron
+          tripleo_fluentd_sources_neutron_ovs_agent:
+            - {get_param: NeutronOpenVswitchAgentLoggingSource}
       step_config: |
         include ::tripleo::profile::base::neutron::ovs
       upgrade_tasks:

puppet/services/nova-api.yaml

@@ -111,9 +111,6 @@ outputs:
     value:
       service_name: nova_api
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaApi}
-      logging_source: {get_param: NovaApiLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -170,6 +167,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_api:
+            - nova
+          tripleo_fluentd_sources_nova_api:
+            - {get_param: NovaApiLoggingSource}
         mysql:
           map_merge:
           - {get_attr: [NovaBase, role_data, service_config_settings, mysql]}

puppet/services/nova-compute.yaml

@@ -153,9 +153,6 @@ outputs:
     value:
       service_name: nova_compute
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaCompute}
-      logging_source: {get_param: NovaComputeLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -217,6 +214,11 @@ outputs:
         # Until then, just include it in the default nova-compute role.
         include tripleo::profile::base::nova::compute::libvirt
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_compute:
+            - nova
+          tripleo_fluentd_sources_nova_compute:
+            - {get_param: NovaComputeLoggingSource}
         collectd:
           tripleo.collectd.plugins.nova_compute:
             - virt

puppet/services/nova-conductor.yaml

@@ -34,14 +34,14 @@ parameters:
     default: 0
     description: Number of workers for Nova services.
     type: number
-  MonitoringSubscriptionNovaConductor:
-    default: 'overcloud-nova-conductor'
-    type: string
   NovaConductorLoggingSource:
     type: json
     default:
       tag: openstack.nova.conductor
       path: /var/log/nova/nova-conductor.log
+  MonitoringSubscriptionNovaConductor:
+    default: 'overcloud-nova-conductor'
+    type: string
   UpgradeLevelNovaCompute:
     type: string
     description: Nova Compute upgrade level
@@ -67,9 +67,6 @@ outputs:
     value:
       service_name: nova_conductor
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConductor}
-      logging_source: {get_param: NovaConductorLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -78,6 +75,12 @@ outputs:
           - nova_workers_zero
           - {}
           - nova::conductor::workers: {get_param: NovaWorkers}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_conductor:
+            - nova
+          tripleo_fluentd_sources_nova_conductor:
+            - {get_param: NovaConductorLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::conductor
       upgrade_tasks:

puppet/services/nova-consoleauth.yaml

@@ -56,11 +56,14 @@ outputs:
     value:
       service_name: nova_consoleauth
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaConsoleauth}
-      logging_source: {get_param: NovaConsoleauthLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         get_attr: [NovaBase, role_data, config_settings]
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_consoleauth:
+            - nova
+          tripleo_fluentd_sources_nova_consoleauth:
+            - {get_param: NovaConsoleauthLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::consoleauth
       upgrade_tasks:

puppet/services/nova-placement.yaml

@@ -85,9 +85,6 @@ outputs:
     value:
       service_name: nova_placement
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaPlacement}
-      logging_source: {get_param: NovaPlacementLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
         - get_attr: [NovaBase, role_data, config_settings]
@@ -123,6 +120,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::nova::placement
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_placement:
+            - nova
+          tripleo_fluentd_sources_nova_placement:
+            - {get_param: NovaPlacementLoggingSource}
         keystone:
           nova::keystone::auth_placement::tenant: 'service'
           nova::keystone::auth_placement::public_url: {get_param: [EndpointMap, NovaPlacementPublic, uri]}

puppet/services/nova-scheduler.yaml

@@ -83,9 +83,6 @@ outputs:
     value:
       service_name: nova_scheduler
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaScheduler}
-      logging_source: {get_param: NovaSchedulerLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -94,6 +91,12 @@ outputs:
             nova::scheduler::filter::scheduler_default_filters: {get_param: NovaSchedulerDefaultFilters}
             nova::scheduler::filter::scheduler_max_attempts: {get_param: NovaSchedulerMaxAttempts}
             nova::scheduler::discover_hosts_in_cells_interval: {get_param: NovaSchedulerDiscoverHostsInCellsInterval}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_scheduler:
+            - nova
+          tripleo_fluentd_sources_nova_scheduler:
+            - {get_param: NovaSchedulerLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::scheduler
       upgrade_tasks:

puppet/services/nova-vnc-proxy.yaml

@@ -56,9 +56,6 @@ outputs:
     value:
       service_name: nova_vnc_proxy
       monitoring_subscription: {get_param: MonitoringSubscriptionNovaVNCProxy}
-      logging_source: {get_param: NovaVncproxyLoggingSource}
-      logging_groups:
-        - nova
       config_settings:
         map_merge:
           - get_attr: [NovaBase, role_data, config_settings]
@@ -77,6 +74,12 @@ outputs:
                 dport:
                   - 6080
                   - 13080
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_nova_vnc_proxy:
+            - nova
+          tripleo_fluentd_sources_nova_vnc_proxy:
+            - {get_param: NovaVncproxyLoggingSource}
       step_config: |
         include tripleo::profile::base::nova::vncproxy
       upgrade_tasks:

puppet/services/octavia-api.yaml

@@ -71,9 +71,6 @@ outputs:
     value:
       service_name: octavia_api
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaApi}
-      logging_source: {get_param: OctaviaApiLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -102,6 +99,11 @@ outputs:
       step_config: |
         include tripleo::profile::base::octavia::api
       service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_api:
+            - octavia
+          tripleo_fluentd_sources_octavia_api:
+            - {get_param: OctaviaApiLoggingSource}
         keystone:
           octavia::keystone::auth::tenant: 'service'
           octavia::keystone::auth::public_url: {get_param: [EndpointMap, OctaviaPublic, uri]}

puppet/services/octavia-health-manager.yaml

@@ -61,14 +61,17 @@ outputs:
     value:
       service_name: octavia_health_manager
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaHealthManager}
-      logging_source: {get_param: OctaviaHealthManagerLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
           - octavia::health_manager::heartbeat_key: {get_param: OctaviaHeartbeatKey}
             octavia::health_manager::event_streamer_driver: 'queue_event_streamer'
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_health_manager:
+            - octavia
+          tripleo_fluentd_sources_octavia_health_manager:
+            - {get_param: OctaviaHealthManagerLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::health_manager
 

puppet/services/octavia-housekeeping.yaml

@@ -68,9 +68,6 @@ outputs:
     value:
       service_name: octavia_housekeeping
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaHousekeeping}
-      logging_source: {get_param: OctaviaHousekeepingLoggingSource}
-      logging_groups:
-        - octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -79,6 +76,12 @@ outputs:
             - amphora_expiry_is_zero
             - {}
             - octavia::worker::amphora_expiry_age: {get_param: OctaviaAmphoraExpiryAge}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_housekeeping:
+            - octavia
+          tripleo_fluentd_sources_octavia_housekeeping:
+            - {get_param: OctaviaHousekeepingLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::housekeeping
 

puppet/services/octavia-worker.yaml

@@ -99,9 +99,6 @@ outputs:
     value:
       service_name: octavia_worker
       monitoring_subscription: {get_param: MonitoringSubscriptionOctaviaWorker}
-      logging_source: {get_param: OctaviaWorkerLoggingSource}
-      logging_groups:
-        -octavia
       config_settings:
         map_merge:
           - get_attr: [OctaviaBase, role_data, config_settings]
@@ -125,6 +122,12 @@ outputs:
             - {}
             - octavia::certificates::client_cert_data: {get_param: OctaviaClientCert}
 
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_octavia_worker:
+            - octavia
+          tripleo_fluentd_sources_octavia_worker:
+            - {get_param: OctaviaWorkerLoggingSource}
       step_config: |
         include tripleo::profile::base::octavia::worker
 

puppet/services/pacemaker.yaml

@@ -99,7 +99,6 @@ parameters:
         \[(?<pid>[^ ]*)\]
         (?<host>[^ ]*)
         (?<message>.*)$/
-
   EnableLoadBalancer:
     default: true
     description: Whether to deploy a LoadBalancer on the Controller
@@ -111,9 +110,6 @@ outputs:
     value:
       service_name: pacemaker
       monitoring_subscription: {get_param: MonitoringSubscriptionPacemaker}
-      logging_groups:
-        - haclient
-      logging_source: {get_param: PacemakerLoggingSource}
       config_settings:
         pacemaker::corosync::cluster_name: 'tripleo_cluster'
         pacemaker::corosync::manage_fw: false
@@ -142,6 +138,12 @@ outputs:
                 - {get_param: PcsdPassword}
                 - {get_param: [DefaultPasswords, pcsd_password]}
         tripleo::profile::base::pacemaker::remote_authkey: {get_param: PacemakerRemoteAuthkey}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_pacemaker:
+            - haclient
+          tripleo_fluentd_sources_pacemaker:
+            - {get_param: PacemakerLoggingSource}
       step_config: |
         include ::tripleo::profile::base::pacemaker
       upgrade_tasks:

puppet/services/pacemaker/cinder-volume.yaml

@@ -49,8 +49,6 @@ outputs:
     value:
       service_name: cinder_volume
       monitoring_subscription: {get_attr: [CinderVolumeBase, role_data, monitoring_subscription]}
-      logging_source: {get_attr: [CinderVolumeBase, role_data, logging_source]}
-      logging_groups: {get_attr: [CinderVolumeBase, role_data, logging_groups]}
       config_settings:
         map_merge:
           - get_attr: [CinderVolumeBase, role_data, config_settings]

puppet/services/pacemaker_remote.yaml

@@ -96,9 +96,6 @@ outputs:
     value:
       service_name: pacemaker_remote
       monitoring_subscription: {get_param: MonitoringSubscriptionPacemakerRemote}
-      logging_groups:
-        - haclient
-      logging_source: {get_param: PacemakerRemoteLoggingSource}
       config_settings:
         tripleo.pacemaker_remote.firewall_rules:
           '130 pacemaker_remote tcp':
@@ -116,5 +113,11 @@ outputs:
               passwords:
                 - {get_param: PcsdPassword}
                 - {get_param: [DefaultPasswords, pcsd_password]}
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_pacemaker_remote:
+            - haclient
+          tripleo_fluentd_sources_pacemaker_remote:
+            - {get_param: PacemakerRemoteLoggingSource}
       step_config: |
         include ::tripleo::profile::base::pacemaker_remote

puppet/services/sahara-api.yaml

@@ -74,9 +74,6 @@ outputs:
     value:
       service_name: sahara_api
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaApi}
-      logging_source: {get_param: SaharaApiLoggingSource}
-      logging_groups:
-        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]
@@ -94,6 +91,12 @@ outputs:
                 dport:
                   - 8386
                   - 13386
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_sahara_api:
+            - sahara
+          tripleo_fluentd_sources_sahara_api:
+            - {get_param: SaharaApiLoggingSource}
       step_config: |
         include ::tripleo::profile::base::sahara::api
       service_config_settings:

puppet/services/sahara-engine.yaml

@@ -56,12 +56,15 @@ outputs:
     value:
       service_name: sahara_engine
       monitoring_subscription: {get_param: MonitoringSubscriptionSaharaEngine}
-      logging_source: {get_param: SaharaEngineLoggingSource}
-      logging_groups:
-        - sahara
       config_settings:
         map_merge:
           - get_attr: [SaharaBase, role_data, config_settings]
+      service_config_settings:
+        fluentd:
+          tripleo_fluentd_groups_sahara_engine:
+            - sahara
+          tripleo_fluentd_sources_sahara_engine:
+            - {get_param: SaharaEngineLoggingSource}
       step_config: |
         include ::tripleo::profile::base::sahara::engine
       upgrade_tasks: