Add STF environment

Add yaml to enable the STF client side. The only manual configuration
necessary should be pointing the host to the STF server.

Remove tcpconns from the collectd default plugin list, as
there are no users of it, and it should not be in the default
plugin list when STF is enabled.

More discussion about settings at
https://github.com/infrawatch/telemetry-framework/pull/100

Note about the plugins:

- virt plugin is already automatically loaded on each role where
  nova-compute is deployed. The hieradata is always set in
  eployment/metrics/collectd-container-puppet but it's fine because the
  plugin isn't always enabled. No need to add extra complexity with
  parameters.
- cpu, df, load, intel_rdt, connectivity, procevent and ipmi are
  automatically loaded if STF is enabled.

Related: https://review.opendev.org/#/c/710376/

Signed-off-by: Ryan McCabe <rmccabe@redhat.com>
Signed-off-by: pleimer <pfbleimer@gmail.com>
Signed-off-by: Leif Madsen <lmadsen@redhat.com>
Co-Authored-By: Emilien Macchi <emilien@redhat.com>

Change-Id: Ifb1923bff9a344985b3e4de04ade4abd4916d2d6

ci/environments/scenario001-standalone.yaml

@@ -151,10 +151,26 @@ parameter_defaults:
     * evidence of criminal activity, system personnel may provide    *
     * the evidence from such monitoring to law enforcement officials.*
     ******************************************************************
+  EnableSTF: true
+  MetricsQdrAddresses:
+    - prefix: collectd
+      distribution: multicast
+  MetricsQdrSSLProfiles:
+    - name: sslProfile
   CollectdConnectionType: amqp1
+  CollectdAmqpInterval: 5
+  CollectdDefaultPollingInterval: 5
   CollectdExtraPlugins:
     - rrdtool
   CollectdEnableSensubility: true
+  CollectdAmqpInstances:
+    notify:
+      notify: true
+      format: JSON
+      presettle: false
+    telemetry:
+      format: JSON
+      presettle: true
   LoggingServers:
     - host: 127.0.0.1
       port: 24224

deployment/metrics/collectd-container-puppet.yaml

@@ -87,7 +87,6 @@ parameters:
       - load
       - memory
       - processes
-      - tcpconns
       - unixsock
       - uptime
     type: comma_delimited_list
@@ -382,6 +381,10 @@ parameters:
     type: number
     description: The number of seconds sensu-plugin-aware handlers should wait before taking second action.
     default: 90
+  EnableSTF:
+    type: boolean
+    description: Set to true to enable configuration for STF client.
+    default: false
 
 conditions:
   amqp_connection:
@@ -402,12 +405,13 @@ conditions:
       - equals: [{get_param: CollectdGnocchiKeystoneEndpoint}, nil]
   enable_sensubility:
     equals: [{get_param: CollectdEnableSensubility}, true]
+  enable_stf:
+    equals: [{get_param: EnableSTF}, true]
   enable_sqlalchemy_collectd: {equals : [{get_param: EnableSQLAlchemyCollectd}, true]}
   sensubility_needs_sudo:
     not:
       equals: [{get_param: CollectdSensubilityExecSudoRule}, '']
 
-
 resources:
   ContainersCommon:
     type: ../containers-common.yaml
@@ -426,17 +430,39 @@ outputs:
             collectd::recurse: true
             collectd::purge_config: true
             collectd::minimum_version: "5.7"
+            collectd::interval: {get_param: CollectdDefaultPollingInterval}
             collectd::plugin::unixsock::socketgroup: root
             collectd::plugin::unixsock::socketfile: /var/run/collectd-socket
             collectd::plugin::unixsock::deletesocket: true
-            collectd::interval: {get_param: CollectdDefaultPollingInterval}
+            collectd::plugin::cpu::reportbycpu: true
+            collectd::plugin::cpu::reportbystate: true
+            collectd::plugin::cpu::reportnumcpu: false
+            collectd::plugin::cpu::valuespercentage: true
+            collectd::plugin::df::ignoreselected: true
+            collectd::plugin::df::reportbydevice: true
+            collectd::plugin::df::fstypes: ['xfs']
+            collectd::plugin::load::reportrelative: true
+            collectd::plugin::virt::connection: "qemu:///system"
+            collectd::plugin::virt::extra_stats: "cpu_util disk disk_err pcpu job_stats_background perf vcpupin"
+            collectd::plugin::virt::hostname_format: "hostname"
             tripleo.collectd.plugins.collectd:
               yaql:
                 data:
                   default_plugins: {get_param: CollectdDefaultPlugins}
+                  stf_plugins:
+                    if:
+                    - enable_stf
+                    - - cpu
+                      - df
+                      - load
+                      - connectivity
+                      - intel_rdt
+                      - ipmi
+                      - procevent
+                    - []
                   extra_plugins: {get_param: CollectdExtraPlugins}
                 expression: >
-                  ($.data.default_plugins + $.data.extra_plugins)
+                  ($.data.default_plugins + $.data.stf_plugins + $.data.extra_plugins)
                   .flatten().distinct()
           - if: # Collectd connected to QDR
             - amqp_connection

deployment/metrics/qdr-container-puppet.yaml

@@ -138,9 +138,14 @@ parameters:
   EnableInternalTLS:
     type: boolean
     default: false
+  EnableSTF:
+    default: false
+    description: Set to true to enable configuration for STF client.
+    type: boolean
 
 conditions:
   internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]}
+  enable_stf: {equals: [{get_param: EnableSTF}, true]}
 
 
 resources:
@@ -248,6 +253,11 @@ outputs:
                       keyFile: '/etc/pki/tls/private/metrics_qdr.key'
                       caCertFile: {get_param: InternalTLSCAFile}
             - tripleo::profile::base::metrics::qdr::ssl_profiles: {get_param: MetricsQdrSSLProfiles}
+          - if:
+            - enable_stf
+            - tripleo::profile::base::metrics::qdr::interior_mesh_nodes: ''
+              tripleo::profile::base::metrics::qdr::router_mode: edge
+            - null
       metadata_settings:
         if:
           - internal_tls_enabled

environments/enable-stf.yaml

@@ -0,0 +1,39 @@
+# This heat environment can be used to enable STF client side
+resource_registry:
+  OS::TripleO::Services::Collectd: ../deployment/metrics/collectd-container-puppet.yaml
+  OS::TripleO::Services::MetricsQdr: ../deployment/metrics/qdr-container-puppet.yaml
+  OS::TripleO::Services::Redis: ../deployment/database/redis-pacemaker-puppet.yaml
+# We want these when smartgateway is able to process ceilometer messages
+  # OS::TripleO::Services::CeilometerAgentCentral: ../deployment/ceilometer/ceilometer-agent-central-container-puppet.yaml
+  # OS::TripleO::Services::CeilometerAgentNotification: ../deployment/ceilometer/ceilometer-agent-notification-container-puppet.yaml
+  # OS::TripleO::Services::ComputeCeilometerAgent: ../deployment/ceilometer/ceilometer-agent-compute-container-puppet.yaml
+
+parameter_defaults:
+  # Uncomment when smartgateway can handle ceilometer messages
+  # CeilometerQdrPublish: true
+  EnableSTF: true
+  CollectdConnectionType: amqp1
+  CollectdAmqpInterval: 5
+  CollectdDefaultPollingInterval: 5
+
+  CollectdAmqpInstances:
+    notify:
+      notify: true
+      format: JSON
+      presettle: false
+    telemetry:
+      format: JSON
+      presettle: true
+  MetricsQdrAddresses:
+    - prefix: collectd
+      distribution: multicast
+  MetricsQdrSSLProfiles:
+    - name: sslProfile
+# The snippet below should be added to a separate yaml file, edited, and
+# passed in at deploy time.
+  #MetricsQdrConnectors:
+  #  - host: qdr-normal-sa-telemetry.apps.remote.tld
+  #    port: 443
+  #    role: edge
+  #    sslProfile: sslProfile
+  #    verifyHostname: false