tripleo-heat-templates/deployment/metrics/collectd-container-puppet.yaml

heat_template_version: wallaby

description: >
  Containerized collectd service  

parameters:
  ContainerCollectdImage:
    description: image
    type: string
    tags:
      - role_specific
  ContainerCollectdConfigImage:
    description: The container image to use for the collectd config_volume
    type: string
    tags:
      - role_specific
  EndpointMap:
    default: {}
    description: Mapping of service endpoint -> protocol. Typically set
                 via parameter_defaults in the resource registry.
    type: json
  ServiceData:
    default: {}
    description: Dictionary packing service data
    type: json
  ServiceNetMap:
    default: {}
    description: Mapping of service_name -> network name. Typically set
                 via parameter_defaults in the resource registry. Use
                 parameter_merge_strategies to merge it with the defaults.
    type: json
  RoleName:
    default: ''
    description: Role name on which the service is applied
    type: string
  RoleParameters:
    default: {}
    description: Parameters specific to the role
    type: json
  KeystoneRegion:
    type: string
    description: Keystone region for endpoint
    default: 'regionOne'
  MetricsQdrPort:
    default: 5666
    description: Service name or port number on which the qdrouterd will accept
                 connections.
    type: number
  MetricsQdrUsername:
    default: 'guest'
    description: Username which should be used to authenticate to the deployed
                 qdrouterd.
    type: string
  MetricsQdrPassword:
    default: 'guest'
    description: Password which should be used to authenticate to the deployed
                 qdrouterd.
    type: string
    hidden: true
  MonitoringSubscriptionCollectd:
    default: 'overcloud-collectd'
    type: string
  CollectdConnectionType:
    default: 'amqp1'
    description: Define which write plugin should collectd use. Currently
                 supported are 'amqp1' and 'network'.
    type: string
  CollectdDefaultPollingInterval:
    default: 120
    type: number
    description: >
      Controls how often registered read functions are called and with that
      the resolution of the collected data. This value can be overridden per
      plugin(per role) by setting "::collectd::plugin::<plugin_name>::interval"
      key in ExtraConfig(<role_name>ExtraConfig) if using puppet, and by
      setting "collectd_plugin_<plugin_name>_interval" in
      <role_name>ExtraConfig/CollectdVars if using ansible.      
  CollectdDefaultPlugins:
    default:
      - cpu
      - df
      - disk
      - hugepages
      - interface
      - load
      - memory
      - unixsock
      - uptime
    type: comma_delimited_list
    description: >
      List of collectd plugins to activate on all overcloud hosts.  See
      the documentation for the puppet-collectd module for a list plugins
      supported by the module (https://github.com/voxpupuli/puppet-collectd).
      Set this key to override the default list of plugins.  Use
      CollectdExtraPlugins if you want to load additional plugins without
      overriding the defaults.      
  CollectdExtraPlugins:
    default: []
    type: comma_delimited_list
    description: >
      List of collectd plugins to activate on all overcloud hosts.  See
      the documentation for the puppet-collectd module for a list plugins
      supported by the module (https://github.com/voxpupuli/puppet-collectd).
      Set this key to load plugins in addition to those in
      CollectdDefaultPlugins.      
  CollectdServer:
    type: string
    description: >
      Address of remote collectd server to which we will send
      metrics.      
    default: ''
  CollectdServerPort:
    type: number
    default: 25826
    description: >
      Port on remote collectd server to which we will send
      metrics.      
  CollectdUsername:
    type: string
    description: >
      Username for authenticating to the remote collectd server. The default
      is to not configure any authentication.      
    default: ''
  CollectdPassword:
    type: string
    hidden: true
    description: >
      Password for authenticating to the remote collectd server. The
      default is to not configure any authentication.      
    default: ''
  CollectdSecurityLevel:
    type: string
    description: >
      Security level setting for remote collectd connection. If it is
      set to Sign or Encrypt the CollectdPassword and CollectdUsername
      parameters need to be set.      
    default: 'None'
    constraints:
      - allowed_values:
          - None
          - Sign
          - Encrypt
  EnableSQLAlchemyCollectd:
    type: boolean
    description: >
        Set to true to enable the SQLAlchemy-collectd server plugin        
    default: false
  CollectdSQLAlchemyLogMessages:
    type: string
    description: set to "debug" to enable message logging.
    default: 'info'
  CollectdSQLAlchemyBindHost:
    type: string
    description: >
      hostname for SQLAlchemy-collectd plugin to bind on.  defaults
      to localhost.      
    default: 'localhost'
  CollectdAmqpHost:
    type: string
    description: Hostname or IP address of the AMQP 1.0 intermediary.
    default: nil
  CollectdAmqpPort:
    type: number
    description: >
      Service name or port number on which the AMQP 1.0 intermediary accepts
      connections. This argument must be a string, even if the numeric form
      is used.      
    default: 5666
  CollectdAmqpUser:
    type: string
    description: >
      User part of credentials used to authenticate to the AMQP 1.0 intermediary.      
    default: guest
  CollectdAmqpPassword:
    type: string
    description: >
      Password part of credentials used to authenticate to the AMQP 1.0 intermediary.      
    default: guest
    hidden: true
  CollectdAmqpTransportName:
    type: string
    description: Name of the AMQP 1.0 transport.
    default: metrics
  CollectdAmqpAddress:
    type: string
    description: >
      This option specifies the prefix for the send-to value in the message.      
    default: collectd
  CollectdAmqpInstances:
    type: json
    description: >
      Hash of hashes. Each inner hash represent Instance block in plugin
      configuration file. Key of outer hash represents instance name.
      The 'address' value concatenated with the 'name' given will be used
      as the send-to address for communications over the messaging link.      
    default: {}
  CollectdAmqpRetryDelay:
    type: number
    description: >
      When the AMQP 1.0 connection is lost, defines the time in seconds to wait
      before attempting to reconnect.      
    default: 1
  CollectdAmqpInterval:
    type: number
    description: >
      Interval on which metrics should be sent to AMQP intermediary. If not set
      the default for all collectd plugins is used.      
    default: -666
  CollectdAmqpSendQueueLimit:
    type: number
    description: >
      Number of data sets to be kept in memory, older sets will be discarded,
      if set to -1, this feature is disabled.      
    default: -1
  CollectdEnableSensubility:
    type: boolean
    description: Set to true if sensubility should be executed by exec plugin.
    default: false
  CollectdSensubilityExecSudoRule:
    type: string
    description: >
      Given rule will be created in /etc/sudoers.d for sensubility to enable it calling
      restricted commands via sensubility executor.      
    default: ''
  CollectdSensubilityLogLevel:
    type: string
    description: Use for override the default logging level (WARNING).
    default: WARNING
  CollectdSensubilityConnection:
    type: string
    description: URL to Sensu sever side
    default: amqp://sensu:sensu@localhost:5672//sensu
  CollectdSensubilityKeepaliveInterval:
    type: number
    description: Interval in seconds for sending keepalive messages to Sensu server side.
    default: 20
  CollectdSensubilityTmpDir:
    type: string
    description: Path to temporary directory which is used for creation of check scripts.
    default: /var/tmp/collectd-sensubility-checks
  CollectdSensubilityShellPath:
    type: string
    description: Path to shell used for executing check scripts.
    default: /usr/bin/sh
  CollectdSensubilityWorkerCount:
    type: number
    description: Number of goroutines spawned for executing check scripts.
    default: 2
  CollectdSensubilityChecks:
    type: json
    description: JSON formatted definition of standalone checks to be scheduled on client side.
    default: {}
  CollectdSensubilityTransport:
    type: string
    description: Bus type for sent data. Options are 'sensu' (rabbitmq) and 'amqp1'
    default: sensu
  CollectdSensubilityResultsChannel:
    type: string
    description: AMQP1 channel address
    default: collectd/notify
  CollectdSensubilityScripts:
    type: json
    description: |
      Hash of scripts for download for sensubility usage. The hash has to be in following format:
      {"script-name": {"source": "<http URI to download script>", "checksum": "<md5sum of the script file>",
      "create_bin_link": true/false <creates /usr/bin/sensubility_script-name if true which is default>}}      
    default: {}
  CollectdEnableContainerHealthCheck:
    type: boolean
    description: >
      Set to false if container health check should not be defined and attached
      to CollectdEnableContainerHealthCheck.      
    default: true
  CollectdContainerHealthCheckCommand:
    type: string
    default: /scripts/collectd_check_health.py
  CollectdContainerHealthCheckInterval:
    type: number
    description: The frequency in seconds the docker health check is executed.
    default: 10
  CollectdContainerHealthCheckHandlers:
    default: []
    description: The Sensu event handler to use for events created by the docker health check.
    type: comma_delimited_list
  CollectdContainerHealthCheckOccurrences:
    type: number
    description: The number of event occurrences before sensu-plugin-aware handler should take action.
    default: 3
  CollectdContainerHealthCheckRefresh:
    type: number
    description: The number of seconds sensu-plugin-aware handlers should wait before taking second action.
    default: 90
  EnableSTF:
    type: boolean
    description: Set to true to enable configuration for STF client.
    default: false
  CollectdEnableMcelog:
    type: boolean
    description: Set to true to enable mcelog
    default: false
  CollectdEnableLibpodstats:
    type: boolean
    description: Set to true if collectd should run the libpodstats plugin
    default: false
  CollectdContainerAdditionalCapAdd:
    type: comma_delimited_list
    description: Additional container capabilities to add to the collectd container.
                 By default is appended to IPC_LOCK.
    default: []
  CollectdLoggingSource:
    type: json
    default:
      tag: collectd
      file: /var/log/containers/collectd/collectd.log


conditions:
  amqp_connection:
    equals: [{get_param: CollectdConnectionType}, 'amqp1']
  amqp_connection_set:
    not: {equals: [{get_param: CollectdAmqpHost}, nil]}
  amqp_interval_set:
    not: {equals: [{get_param: CollectdAmqpInterval}, -666]}
  amqp_send_queue_limit_set:
    not: {equals: [{get_param: CollectdAmqpSendQueueLimit}, -1]}
  collectd_connection:
    equals: [{get_param: CollectdConnectionType}, 'network']
  sensubility_needs_sudo:
    not:
      equals: [{get_param: CollectdSensubilityExecSudoRule}, '']
  enable_libpodstats:
    equals: [{get_param: CollectdEnableLibpodstats}, true]

resources:
  ContainersCommon:
    type: ../containers-common.yaml

  RoleParametersValue:
    type: OS::Heat::Value
    properties:
      type: json
      value:
        map_replace:
          - map_replace:
            - ContainerCollectdImage: ContainerCollectdImage
              ContainerCollectdConfigImage: ContainerCollectdConfigImage
            - values: {get_param: [RoleParameters]}
          - values:
              ContainerCollectdImage: {get_param: ContainerCollectdImage}
              ContainerCollectdConfigImage: {get_param: ContainerCollectdConfigImage}

outputs:
  role_data:
    description: Role data for the collectd role.
    value:
      service_name: collectd
      config_settings:
        map_merge:
          - tripleo::profile::base::metrics::collectd::enable_file_logging: true
            collectd::plugin::logfile::log_file: /var/log/collectd/collectd.log
            collectd::manage_repo: false
            collectd::purge: true
            collectd::recurse: true
            collectd::purge_config: true
            collectd::minimum_version: "5.7"
            collectd::interval: {get_param: CollectdDefaultPollingInterval}
            collectd::plugin::unixsock::socketgroup: root
            collectd::plugin::unixsock::socketfile: /run/collectd-socket
            collectd::plugin::unixsock::deletesocket: true
            collectd::plugin::cpu::reportbycpu: true
            collectd::plugin::cpu::reportbystate: true
            collectd::plugin::cpu::reportnumcpu: false
            collectd::plugin::cpu::valuespercentage: true
            collectd::plugin::df::ignoreselected: true
            collectd::plugin::df::reportbydevice: true
            collectd::plugin::df::fstypes: ['xfs']
            collectd::plugin::load::reportrelative: true
            collectd::plugin::virt::connection: "qemu:///system"
            collectd::plugin::virt::extra_stats:
              list_join:
              - ' '
              - - 'pcpu'
                - 'cpu_util'
                - 'vcpupin'
                - 'vcpu'
                - 'memory'
                - 'disk'
                - 'disk_err'
                - 'disk_allocation'
                - 'disk_capacity'
                - 'disk_physical'
                - 'domain_state'
                - 'job_stats_background'
                - 'perf'
            collectd::plugin::virt::hostname_format: "hostname"
            tripleo.collectd.plugins.collectd:
              list_concat_unique:
                - {get_param: CollectdDefaultPlugins}
                - if:
                    - {get_param: EnableSTF}
                    - - cpu
                      - df
                      - load
                      - connectivity
                      - intel_rdt
                      - ipmi
                      - procevent
                - {get_param: CollectdExtraPlugins}
          - if: # Collectd connected to QDR
            - amqp_connection
            - map_merge:
              - tripleo::profile::base::metrics::collectd::amqp_transport_name:
                  get_param: CollectdAmqpTransportName
                tripleo::profile::base::metrics::collectd::amqp_address:
                  get_param: CollectdAmqpAddress
                tripleo::profile::base::metrics::collectd::amqp_instances:
                  get_param: CollectdAmqpInstances
                tripleo::profile::base::metrics::collectd::amqp_retry_delay:
                  get_param: CollectdAmqpRetryDelay
                tripleo::profile::base::metrics::collectd::amqp_interval:
                  if:
                    - amqp_interval_set
                    - {get_param: CollectdAmqpInterval}
                tripleo::profile::base::metrics::collectd::amqp_default_send_queue_limit:
                  if:
                    - amqp_send_queue_limit_set
                    - {get_param: CollectdAmqpSendQueueLimit}
              - if:
                - amqp_connection_set
                - tripleo::profile::base::metrics::collectd::amqp_host:
                    get_param: CollectdAmqpHost
                  tripleo::profile::base::metrics::collectd::amqp_port:
                    get_param: CollectdAmqpPort
                  tripleo::profile::base::metrics::collectd::amqp_user:
                    get_param: CollectdAmqpUser
                  tripleo::profile::base::metrics::collectd::amqp_password:
                    get_param: CollectdAmqpPassword
                - tripleo::profile::base::metrics::collectd::amqp_host:
                    str_replace:
                      template:
                        "%{lookup('$NETWORK')}"
                      params:
                        $NETWORK:
                          get_param:
                            - ServiceNetMap
                            - str_replace:
                                template: "ROLENAMEMetricsQdrNetwork"
                                params:
                                  ROLENAME: {get_param: RoleName}
                  tripleo::profile::base::metrics::collectd::amqp_port:
                    get_param: MetricsQdrPort
                  tripleo::profile::base::metrics::collectd::amqp_user:
                    get_param: MetricsQdrUsername
                  tripleo::profile::base::metrics::collectd::amqp_password:
                    get_param: MetricsQdrPassword
          - if: # Collectd connected to external collectd instance
            - collectd_connection
            - tripleo::profile::base::metrics::collectd::collectd_server:
                get_param: CollectdServer
              tripleo::profile::base::metrics::collectd::collectd_port:
                get_param: CollectdServerPort
              tripleo::profile::base::metrics::collectd::collectd_username:
                get_param: CollectdUsername
              tripleo::profile::base::metrics::collectd::collectd_password:
                get_param: CollectdPassword
              tripleo::profile::base::metrics::collectd::collectd_securitylevel:
                get_param: CollectdSecurityLevel
          - if:
            - {get_param: EnableSQLAlchemyCollectd}
            - tripleo::profile::base::metrics::collectd::enable_sqlalchemy_collectd:
                  true
              tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_bind_host:
                  get_param: CollectdSQLAlchemyBindHost
              tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_log_messages:
                  get_param: CollectdSQLAlchemyLogMessages
          - if: # Collectd should run collectd-sensubility via collectd-exec
            - {get_param: CollectdEnableSensubility}
            - map_merge:
              - if:
                - sensubility_needs_sudo
                - tripleo::profile::base::metrics::collectd::sensubility::exec_sudo_rule:
                    get_param: CollectdSensubilityExecSudoRule
              - tripleo::profile::base::metrics::collectd::enable_sensubility:
                  get_param: CollectdEnableSensubility
                tripleo::profile::base::metrics::collectd::sensubility::connection:
                  get_param: CollectdSensubilityConnection
                tripleo::profile::base::metrics::collectd::sensubility::log_level:
                  get_param: CollectdSensubilityLogLevel
                tripleo::profile::base::metrics::collectd::sensubility::client_name:
                  str_replace:
                    template: "%{lookup('fqdn_NETWORK')}"
                    params:
                      NETWORK:
                        get_param:
                          - ServiceNetMap
                          - str_replace:
                              template: "ROLENAMEMetricsQdrNetwork"
                              params:
                                ROLENAME: {get_param: RoleName}
                tripleo::profile::base::metrics::collectd::sensubility::client_address:
                  str_replace:
                    template: "%{lookup('NETWORK')}"
                    params:
                      NETWORK:
                        get_param:
                          - ServiceNetMap
                          - str_replace:
                              template: "ROLENAMEMetricsQdrNetwork"
                              params:
                                ROLENAME: {get_param: RoleName}
                tripleo::profile::base::metrics::collectd::sensubility::keepalive_interval:
                  get_param: CollectdSensubilityKeepaliveInterval
                tripleo::profile::base::metrics::collectd::sensubility::tmp_base_dir:
                  get_param: CollectdSensubilityTmpDir
                tripleo::profile::base::metrics::collectd::sensubility::shell_path:
                  get_param: CollectdSensubilityShellPath
                tripleo::profile::base::metrics::collectd::sensubility::worker_count:
                  get_param: CollectdSensubilityWorkerCount
                tripleo::profile::base::metrics::collectd::sensubility::checks:
                  map_merge:
                    - {get_param: CollectdSensubilityChecks}
                    - check-container-health:
                        standalone: true
                        command: {get_param: CollectdContainerHealthCheckCommand}
                        interval: {get_param: CollectdContainerHealthCheckInterval}
                        handlers: {get_param: CollectdContainerHealthCheckHandlers}
                        occurrences: {get_param: CollectdContainerHealthCheckOccurrences}
                        refresh: {get_param: CollectdContainerHealthCheckRefresh}
                tripleo::profile::base::metrics::collectd::sensubility::results_channel:
                  get_param: CollectdSensubilityResultsChannel
                tripleo::profile::base::metrics::collectd::sensubility::transport:
                  get_param: CollectdSensubilityTransport
                tripleo::profile::base::metrics::collectd::sensubility::amqp_port:
                  get_param: CollectdAmqpPort
                tripleo::profile::base::metrics::collectd::sensubility::scripts:
                  get_param: CollectdSensubilityScripts
            - {}
          - if:
            - {get_param: CollectdEnableLibpodstats}
            - tripleo::profile::base::metrics::collectd::enable_libpodstats:
                get_param: CollectdEnableLibpodstats
      service_config_settings:
        rsyslog:
          tripleo_logging_sources_collectd:
            - {get_param: CollectdLoggingSource}
      # BEGIN DOCKER SETTINGS
      puppet_config:
        config_volume: collectd
        puppet_tags:  collectd_client_config,exec
        step_config: include tripleo::profile::base::metrics::collectd
        config_image: {get_attr: [RoleParametersValue, value, ContainerCollectdConfigImage]}
      kolla_config:
        /var/lib/kolla/config_files/collectd.json:
          command: /usr/sbin/collectd -f
          config_files:
            - source: "/var/lib/kolla/config_files/src/*"
              dest: "/"
              merge: true
              preserve_properties: true
            - source: "/var/lib/kolla/config_files/src/etc/collectd.d"
              dest: "/etc/"
              merge: false
              preserve_properties: true
          permissions:
            - path: /var/log/collectd
              owner: collectd:collectd
              recurse: true
      container_config_scripts:
        map_merge:
          - {get_attr: [ContainersCommon, container_config_scripts]}
          - collectd_check_health.py:
              mode: "0755"
              content: { get_file: ../../container_config_scripts/monitoring/collectd_check_health.py }
      docker_config:
        step_2:
          if:
            - {get_param: CollectdEnableSensubility}
            - collectd_init_perm:
                image: {get_attr: [RoleParametersValue, value, ContainerCollectdImage]}
                net: none
                user: root
                volumes:
                  - /run:/run:rw
                command: ['setfacl', '-R', '-m', 'u:collectd:rwx', '/run/podman']
        step_3:
          collectd:
            image: {get_attr: [RoleParametersValue, value, ContainerCollectdImage]}
            net: host
            pid: host
            user: root
            restart: always
            mem_limit: 512m
            cap_add:
              list_concat:
                - {get_param: CollectdContainerAdditionalCapAdd}
                - [IPC_LOCK]
            healthcheck:
              test: /openstack/healthcheck
            volumes:
              list_concat:
                - {get_attr: [ContainersCommon, volumes]}
                -
                  - /var/lib/kolla/config_files/collectd.json:/var/lib/kolla/config_files/config.json:ro
                  - /var/lib/containers/storage/overlay-containers:/var/lib/containers/storage/overlay-containers:ro
                  - /var/lib/config-data/puppet-generated/collectd:/var/lib/kolla/config_files/src:ro
                  - /var/log/containers/collectd:/var/log/collectd:rw,z
                  - /var/lib/container-config-scripts:/scripts:ro
                  - /run:/run:rw
                  - /sys/fs/cgroup:/sys/fs/cgroup:ro
            environment:
              KOLLA_CONFIG_STRATEGY: COPY_ALWAYS
      deploy_steps_tasks:
        - name: set enable_sensubility fact
          set_fact:
            enable_sensubility: {get_param: CollectdEnableSensubility}
        - name: Configure rsyslog for container healthchecks
          when:
            - step|int == 1
          block:
            - name: Check if rsyslog exists
              shell: systemctl list-unit-files --type=service | grep -q rsyslog
              register: rsyslog_config
              failed_when: rsyslog_config.rc == 2
            - name: Configure if we can
              when:
                - rsyslog_config is changed
                - rsyslog_config.rc == 0
              block:
                - name: Log healthchecks in dedicated file
                  when:
                    - enable_sensubility|bool
                  register: logconfig_add
                  copy:
                    dest: /etc/rsyslog.d/openstack-healthcheck.conf
                    content: |
                      if ($programname startswith 'podman' and ($msg contains 'container exec' or $msg contains 'healthy')) or ($programname startswith 'systemd' and $msg contains 'podman healthcheck run') then -/var/log/containers/collectd/healthchecks.stdout
                      & stop                      
                - name: Remove healthcheck log
                  when:
                    - not enable_sensubility|bool
                  register: logconfig_rm
                  file:
                    path: /etc/rsyslog.d/openstack-healthcheck.conf
                    state: absent
                - name: Reload rsyslogd if needed
                  when: logconfig_add is changed or logconfig_rm is changed
                  service:
                    name: rsyslog
                    state: restarted
      host_prep_tasks:
        - name: create persistent directories
          file:
            path: "{{ item.path }}"
            state: directory
            setype: "{{ item.setype }}"
            mode: "{{ item.mode }}"
          with_items:
            - { 'path': /var/log/containers/collectd, 'setype': container_file_t, 'mode': '0750' }
        - name: import provision_mcelog
          include_role:
            name: tripleo_provision_mcelog
          when: {get_param: CollectdEnableMcelog}
        - name: enable podman socket
          ansible.builtin.service:
            name: podman.socket
            state: started
            enabled: true
          when: {get_param: CollectdEnableSensubility}