From c5c558229eaedfac0dcba5fbbf3983649aa1182c Mon Sep 17 00:00:00 2001 From: ramishra Date: Wed, 28 Apr 2021 09:34:02 +0530 Subject: [PATCH] Simplify metrics and messaging service templates Change-Id: Ibd240fee7df197301ef17c8029d44d06d6007bc5 --- .../rpc-qdrouterd-container-puppet.yaml | 4 +- .../metrics/collectd-container-ansible.yaml | 7 +- .../metrics/collectd-container-puppet.yaml | 91 +++++++------------ deployment/metrics/qdr-container-ansible.yaml | 24 ++--- deployment/metrics/qdr-container-puppet.yaml | 41 +++------ .../multipathd-container-ansible.yaml | 3 +- 6 files changed, 60 insertions(+), 110 deletions(-) diff --git a/deployment/messaging/rpc-qdrouterd-container-puppet.yaml b/deployment/messaging/rpc-qdrouterd-container-puppet.yaml index f8ea27f92d..b9b68ca9ad 100644 --- a/deployment/messaging/rpc-qdrouterd-container-puppet.yaml +++ b/deployment/messaging/rpc-qdrouterd-container-puppet.yaml @@ -53,7 +53,6 @@ parameters: type: string resources: - ContainersCommon: type: ../containers-common.yaml @@ -132,8 +131,7 @@ outputs: volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - - - - /var/lib/kolla/config_files/qdrouterd.json:/var/lib/kolla/config_files/config.json:ro + - - /var/lib/kolla/config_files/qdrouterd.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/config-data/puppet-generated/qdrouterd:/var/lib/kolla/config_files/src:ro - /var/lib/qdrouterd:/var/lib/qdrouterd:z - /var/log/containers/qdrouterd:/var/log/qdrouterd:z diff --git a/deployment/metrics/collectd-container-ansible.yaml b/deployment/metrics/collectd-container-ansible.yaml index a6953065d2..872feb3cf3 100644 --- a/deployment/metrics/collectd-container-ansible.yaml +++ b/deployment/metrics/collectd-container-ansible.yaml @@ -349,10 +349,10 @@ resources: tripleo_collectd_enable_mcelog: {get_param: CollectdEnableMcelog} tripleo_collectd_enable_libpodstats: {get_param: CollectdEnableLibpodstats } # The last element should be the CollectdVars, which overides any previous deprecated metric. - - { get_param: CollectdVars } + - {get_param: CollectdVars} - if: - role_specific_required - - { get_param: [RoleParameters, CollectdVars]} + - {get_param: [RoleParameters, CollectdVars]} - {} outputs: @@ -400,8 +400,7 @@ outputs: volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - - - - /var/lib/kolla/config_files/collectd.json:/var/lib/kolla/config_files/config.json:ro + - - /var/lib/kolla/config_files/collectd.json:/var/lib/kolla/config_files/config.json:ro - /var/lib/containers/storage/overlay-containers:/var/lib/containers/storage/overlay-containers:ro - /var/lib/config-data/ansible-generated/collectd:/var/lib/kolla/config_files/src:ro - /var/log/containers/collectd:/var/log/collectd:rw,z diff --git a/deployment/metrics/collectd-container-puppet.yaml b/deployment/metrics/collectd-container-puppet.yaml index 4ee212e12b..9f0312fe38 100644 --- a/deployment/metrics/collectd-container-puppet.yaml +++ b/deployment/metrics/collectd-container-puppet.yaml @@ -392,12 +392,12 @@ parameters: conditions: amqp_connection: equals: [{get_param: CollectdConnectionType}, 'amqp1'] - amqp_default_connection: - equals: [{get_param: CollectdAmqpHost}, nil] - amqp_default_interval: - equals: [{get_param: CollectdAmqpInterval}, -666] - amqp_default_send_queue_limit: - equals: [{get_param: CollectdAmqpSendQueueLimit}, -1] + amqp_connection_set: + not: {equals: [{get_param: CollectdAmqpHost}, nil]} + amqp_interval_set: + not: {equals: [{get_param: CollectdAmqpInterval}, -666]} + amqp_send_queue_limit_set: + not: {equals: [{get_param: CollectdAmqpSendQueueLimit}, -1]} collectd_connection: equals: [{get_param: CollectdConnectionType}, 'network'] gnocchi_connection: @@ -408,11 +408,6 @@ conditions: and: - equals: [{get_param: CollectdGnocchiServer}, nil] - equals: [{get_param: CollectdGnocchiKeystoneEndpoint}, nil] - enable_sensubility: - equals: [{get_param: CollectdEnableSensubility}, true] - enable_stf: - equals: [{get_param: EnableSTF}, true] - enable_sqlalchemy_collectd: {equals : [{get_param: EnableSQLAlchemyCollectd}, true]} sensubility_needs_sudo: not: equals: [{get_param: CollectdSensubilityExecSudoRule}, ''] @@ -432,7 +427,7 @@ outputs: map_merge: - tripleo::profile::base::metrics::collectd::enable_file_logging: true collectd::plugin::logfile::log_file: /var/log/collectd/collectd.log - - collectd::manage_repo: false + collectd::manage_repo: false collectd::purge: true collectd::recurse: true collectd::purge_config: true @@ -468,12 +463,10 @@ outputs: - 'perf' collectd::plugin::virt::hostname_format: "hostname" tripleo.collectd.plugins.collectd: - yaql: - data: - default_plugins: {get_param: CollectdDefaultPlugins} - stf_plugins: - if: - - enable_stf + list_concat_unique: + - {get_param: CollectdDefaultPlugins} + - if: + - {get_param: EnableSTF} - - cpu - df - load @@ -481,11 +474,7 @@ outputs: - intel_rdt - ipmi - procevent - - [] - extra_plugins: {get_param: CollectdExtraPlugins} - expression: > - ($.data.default_plugins + $.data.stf_plugins + $.data.extra_plugins) - .flatten().distinct() + - {get_param: CollectdExtraPlugins} - if: # Collectd connected to QDR - amqp_connection - map_merge: @@ -497,18 +486,24 @@ outputs: get_param: CollectdAmqpInstances tripleo::profile::base::metrics::collectd::amqp_retry_delay: get_param: CollectdAmqpRetryDelay + tripleo::profile::base::metrics::collectd::amqp_interval: + if: + - amqp_interval_set + - {get_param: CollectdAmqpInterval} + tripleo::profile::base::metrics::collectd::amqp_default_send_queue_limit: + if: + - amqp_send_queue_limit_set + - {get_param: CollectdAmqpSendQueueLimit} - if: - - amqp_default_interval - - {} - - tripleo::profile::base::metrics::collectd::amqp_interval: - get_param: CollectdAmqpInterval - - if: - - amqp_default_send_queue_limit - - {} - - tripleo::profile::base::metrics::collectd::amqp_default_send_queue_limit: - get_param: CollectdAmqpSendQueueLimit - - if: - - amqp_default_connection + - amqp_connection_set + - tripleo::profile::base::metrics::collectd::amqp_host: + get_param: CollectdAmqpHost + tripleo::profile::base::metrics::collectd::amqp_port: + get_param: CollectdAmqpPort + tripleo::profile::base::metrics::collectd::amqp_user: + get_param: CollectdAmqpUser + tripleo::profile::base::metrics::collectd::amqp_password: + get_param: CollectdAmqpPassword - tripleo::profile::base::metrics::collectd::amqp_host: str_replace: template: @@ -527,15 +522,6 @@ outputs: get_param: MetricsQdrUsername tripleo::profile::base::metrics::collectd::amqp_password: get_param: MetricsQdrPassword - - tripleo::profile::base::metrics::collectd::amqp_host: - get_param: CollectdAmqpHost - tripleo::profile::base::metrics::collectd::amqp_port: - get_param: CollectdAmqpPort - tripleo::profile::base::metrics::collectd::amqp_user: - get_param: CollectdAmqpUser - tripleo::profile::base::metrics::collectd::amqp_password: - get_param: CollectdAmqpPassword - - {} - if: # Collectd connected to external collectd instance - collectd_connection - tripleo::profile::base::metrics::collectd::collectd_server: @@ -548,17 +534,14 @@ outputs: get_param: CollectdPassword tripleo::profile::base::metrics::collectd::collectd_securitylevel: get_param: CollectdSecurityLevel - - {} - if: - - enable_sqlalchemy_collectd - - map_merge: - - tripleo::profile::base::metrics::collectd::enable_sqlalchemy_collectd: + - {get_param: EnableSQLAlchemyCollectd} + - tripleo::profile::base::metrics::collectd::enable_sqlalchemy_collectd: true - - tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_bind_host: + tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_bind_host: get_param: CollectdSQLAlchemyBindHost - - tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_log_messages: + tripleo::profile::base::metrics::collectd::sqlalchemy_collectd_log_messages: get_param: CollectdSQLAlchemyLogMessages - - {} - if: # Collectd connected directly to Gnocchi DB - gnocchi_connection - map_merge: @@ -618,15 +601,13 @@ outputs: get_param: CollectdGnocchiKeystoneInterface tripleo::profile::base::metrics::collectd::gnocchi_keystone_endpoint: get_param: CollectdGnocchiKeystoneEndpoint - - {} - if: # Collectd should run collectd-sensubility via collectd-exec - - enable_sensubility + - {get_param: CollectdEnableSensubility} - map_merge: - if: - sensubility_needs_sudo - tripleo::profile::base::metrics::collectd::sensubility::exec_sudo_rule: get_param: CollectdSensubilityExecSudoRule - - {} - tripleo::profile::base::metrics::collectd::enable_sensubility: get_param: CollectdEnableSensubility tripleo::profile::base::metrics::collectd::sensubility::connection: @@ -679,12 +660,10 @@ outputs: get_param: CollectdSensubilityTransport tripleo::profile::base::metrics::collectd::sensubility::amqp_port: get_param: CollectdAmqpPort - - {} - if: - - enable_libpodstats + - {get_param: CollectdEnableLibpodstats} - tripleo::profile::base::metrics::collectd::enable_libpodstats: get_param: CollectdEnableLibpodstats - - {} service_config_settings: {} # BEGIN DOCKER SETTINGS puppet_config: diff --git a/deployment/metrics/qdr-container-ansible.yaml b/deployment/metrics/qdr-container-ansible.yaml index b4e622f683..bc712e2eda 100644 --- a/deployment/metrics/qdr-container-ansible.yaml +++ b/deployment/metrics/qdr-container-ansible.yaml @@ -155,10 +155,8 @@ parameters: conditions: - internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} role_specific_required: {not: {equals: [{ get_param: [RoleParameters, MetricsQdrVars] }, ""]}} - resources: ContainersCommon: type: ../containers-common.yaml @@ -205,10 +203,10 @@ resources: # [2] https://github.com/openstack/puppet-qdr/search?q=listener_trusted_certs tripleo_metrics_qdr_listener_trusted_certs: { get_param: MetricsQdrTrustedCerts } ###### - - { get_param: MetricsQdrVars } + - {get_param: MetricsQdrVars} - if: - role_specific_required - - { get_param: [RoleParameters, MetricsQdrVars] } + - {get_param: [RoleParameters, MetricsQdrVars]} - {} outputs: @@ -230,16 +228,13 @@ outputs: dport: - 5667 - 5668 - # For sensu monitoring_subscription: {get_param: MonitoringSubscriptionQdr} - # See https://docs.openstack.org/tripleo-docs/latest/developer/tht_walkthrough/tls_for_services.html metadata_settings: if: - - internal_tls_enabled - - - - service: metrics_qdr + - {get_param: EnableInternalTLS} + - - service: metrics_qdr network: get_param: - ServiceNetMap @@ -248,8 +243,6 @@ outputs: params: ROLENAME: {get_param: RoleName} type: node - - null - kolla_config: /var/lib/kolla/config_files/metrics_qdr.json: command: /usr/sbin/qdrouterd -c /etc/qpid-dispatch/qdrouterd.conf @@ -276,17 +269,15 @@ outputs: - optional: true owner: qdrouterd:qdrouterd path: /etc/pki/tls/private/metrics_qdr.key - service_config_settings: rsyslog: tripleo_logging_sources_metrics_qdr: - {get_param: MetricsQdrLoggingSource} - config_settings: map_merge: - tripleo::haproxy::metrics_qdr: {get_param: MetricsQdrExternalEndpoint} - if: - - internal_tls_enabled + - {get_param: EnableInternalTLS} - generate_service_certificates: true # These two are used here: https://review.opendev.org/plugins/gitiles/openstack/puppet-tripleo/+/refs/heads/master/files/certmonger-metrics-qdr-refresh.sh#8 tripleo::metrics::qdr::service_certificate: { get_param: MetricsQdrSslCertFile } @@ -318,7 +309,6 @@ outputs: template: "ROLENAMEMetricsQdrNetwork" params: ROLENAME: {get_param: RoleName} - - # Empty then docker_config: step_1: metrics_qdr: @@ -343,11 +333,9 @@ outputs: - /var/lib/metrics_qdr:/var/lib/qdrouterd:z - /var/log/containers/metrics_qdr:/var/log/qdrouterd:z - if: - - internal_tls_enabled + - {get_param: EnableInternalTLS} - - /etc/pki/tls/certs/metrics_qdr.crt:/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/metrics_qdr.crt:ro - /etc/pki/tls/private/metrics_qdr.key:/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/metrics_qdr.key:ro - - null - host_prep_tasks: - name: Create required host directories for QDR file: diff --git a/deployment/metrics/qdr-container-puppet.yaml b/deployment/metrics/qdr-container-puppet.yaml index b4f2418c8d..150bc517c9 100644 --- a/deployment/metrics/qdr-container-puppet.yaml +++ b/deployment/metrics/qdr-container-puppet.yaml @@ -153,17 +153,13 @@ parameters: certificate for this service conditions: - internal_tls_enabled: {equals: [{get_param: EnableInternalTLS}, true]} - listener_ssl_enabled: {equals: [{get_param: MetricsQdrUseSSL}, true]} - enable_stf: {equals: [{get_param: EnableSTF}, true]} - key_size_override_unset: {equals: [{get_param: QdrCertificateKeySize}, '']} - + key_size_override_set: + not: {equals: [{get_param: QdrCertificateKeySize}, '']} resources: ContainersCommon: type: ../containers-common.yaml - outputs: role_data: description: Role data for the metrics Qdr role. @@ -210,11 +206,7 @@ outputs: tripleo::profile::base::metrics::qdr::addresses: {get_param: MetricsQdrAddresses} tripleo::profile::base::metrics::qdr::autolink_addresses: {get_param: MetricsQdrAutoLinks} # ssl support - tripleo::profile::base::metrics::qdr::listener_require_ssl: - if: - - listener_ssl_enabled - - true - - false + tripleo::profile::base::metrics::qdr::listener_require_ssl: {get_param: MetricsQdrUseSSL} tripleo::profile::base::metrics::qdr::listener_require_encrypt: {get_param: MetricsQdrUseEncryption} tripleo::profile::base::metrics::qdr::listener_sasl_mech: {get_param: MetricsQdrSaslMechanisms} tripleo::profile::base::metrics::qdr::listener_ssl_cert_db: {get_param: MetricsQdrSslCertDb} @@ -226,27 +218,24 @@ outputs: qdr::log_enable: 'info+' qdr::log_output: '/var/log/qdrouterd/metrics_qdr.log' qdr::listener_auth_peer: {get_param: MetricsQdrAuthenticateClient} - - if: - - internal_tls_enabled - - - tripleo::profile::base::metrics::qdr::ssl_profiles: - list_concat: + tripleo::profile::base::metrics::qdr::ssl_profiles: + if: + - {get_param: EnableInternalTLS} + - list_concat: - get_param: MetricsQdrSSLProfiles - - name: 'tlsProfile' certFile: '/etc/pki/tls/certs/metrics_qdr.crt' keyFile: '/etc/pki/tls/private/metrics_qdr.key' caCertFile: {get_param: InternalTLSCAFile} - - tripleo::profile::base::metrics::qdr::ssl_profiles: {get_param: MetricsQdrSSLProfiles} + - {get_param: MetricsQdrSSLProfiles} - if: - - enable_stf + - {get_param: EnableSTF} - tripleo::profile::base::metrics::qdr::interior_mesh_nodes: '' tripleo::profile::base::metrics::qdr::router_mode: edge - - null metadata_settings: if: - - internal_tls_enabled - - - - service: metrics_qdr + - {get_param: EnableInternalTLS} + - - service: metrics_qdr network: get_param: - ServiceNetMap @@ -255,7 +244,6 @@ outputs: params: ROLENAME: {get_param: RoleName} type: node - - null # BEGIN DOCKER SETTINGS puppet_config: config_volume: metrics_qdr @@ -315,10 +303,9 @@ outputs: - /var/lib/metrics_qdr:/var/lib/qdrouterd:z - /var/log/containers/metrics_qdr:/var/log/qdrouterd:z - if: - - internal_tls_enabled + - {get_param: EnableInternalTLS} - - /etc/pki/tls/certs/metrics_qdr.crt:/var/lib/kolla/config_files/src-tls/etc/pki/tls/certs/metrics_qdr.crt:ro - /etc/pki/tls/private/metrics_qdr.key:/var/lib/kolla/config_files/src-tls/etc/pki/tls/private/metrics_qdr.key:ro - - null environment: KOLLA_CONFIG_STRATEGY: COPY_ALWAYS deploy_steps_tasks: @@ -369,9 +356,9 @@ outputs: {{container_cli}} restart "$container_name" key_size: if: - - key_size_override_unset - - {get_param: CertificateKeySize} + - key_size_override_set - {get_param: QdrCertificateKeySize} + - {get_param: CertificateKeySize} ca: ipa host_prep_tasks: - name: create persistent logs directory diff --git a/deployment/multipathd/multipathd-container-ansible.yaml b/deployment/multipathd/multipathd-container-ansible.yaml index 1169c52385..8febd185fc 100644 --- a/deployment/multipathd/multipathd-container-ansible.yaml +++ b/deployment/multipathd/multipathd-container-ansible.yaml @@ -94,8 +94,7 @@ outputs: volumes: list_concat: - {get_attr: [ContainersCommon, volumes]} - - - - /var/lib/kolla/config_files/multipathd.json:/var/lib/kolla/config_files/config.json:ro + - - /var/lib/kolla/config_files/multipathd.json:/var/lib/kolla/config_files/config.json:ro - /etc/iscsi:/var/lib/kolla/config_files/src-iscsid:ro - /dev/:/dev/ - /run/:/run/