Switch to Monasca API for logs

The Monasca Log API has been removed and in this change we switch
to using the unified API. If dedicated log APIs are required then
this can be supported through configuration. Out of the box the
Monasca API is used for both logs and metrics which is envisaged to
work for most use cases.

In order to use the unified API for logs, we need to disable the
legacy Kafka client. We also rename the Monasca API config file
to remove a warning about using the old style name.

Depends-On: https://review.opendev.org/#/c/728638
Change-Id: I9b6bf5b6690f4b4b3445e7d15a40e45dd42d2e84
This commit is contained in:
Doug Szumski 2020-05-14 16:21:45 +01:00
parent b480ecff0c
commit b39a0f805a
24 changed files with 42 additions and 289 deletions

View File

@ -344,7 +344,7 @@ memcached_port: "11211"
mistral_api_port: "8989"
monasca_api_port: "8070"
monasca_log_api_port: "5607"
monasca_log_api_port: "{{ monasca_api_port }}"
monasca_agent_forwarder_port: "17123"
monasca_agent_statsd_port: "8125"
monasca_grafana_server_port: "3001"

View File

@ -442,9 +442,6 @@ monasca
[monasca-grafana:children]
monasca
[monasca-log-api:children]
monasca
[monasca-log-transformer:children]
monasca

View File

@ -451,9 +451,6 @@ monasca
[monasca-grafana:children]
monasca
[monasca-log-api:children]
monasca
[monasca-log-transformer:children]
monasca

View File

@ -1,7 +1,7 @@
<match kolla.var.log.kolla.*.*.log>
@type rewrite_tag_filter
capitalize_regex_backreference yes
rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ apache_access
rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ apache_access
rewriterule2 programname ^(aodh_wsgi_access|barbican-api|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access
rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python
rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python

View File

@ -3,7 +3,7 @@
capitalize_regex_backreference yes
<rule>
key programname
pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$
pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$
tag apache_access
</rule>
<rule>

View File

@ -5,7 +5,7 @@
path {% for service in fluentd_enabled_input_openstack_services %}/var/log/kolla/{{ service }}/*.log{% if not loop.last %},{% endif %}{% endfor %}
exclude_path ["/var/log/kolla/monasca/agent*.log",
"/var/log/kolla/monasca/grafana.log",
"/var/log/kolla/monasca/monasca-log-api.log",
"/var/log/kolla/monasca/monasca-api.log",
"/var/log/kolla/neutron/dnsmasq.log",
"/var/log/kolla/*/*-access.log",
"/var/log/kolla/*/*-error.log",

View File

@ -38,8 +38,8 @@
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_log_api_version v3.0
monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
@ -95,8 +95,8 @@
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_log_api_version v3.0
monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default
@ -152,8 +152,8 @@
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_log_api_version v3.0
monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default

View File

@ -3,8 +3,8 @@
<store>
@type monasca
keystone_url {{ keystone_internal_url }}
monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_log_api_version v3.0
monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }}
monasca_api_version v2.0
username {{ monasca_agent_user }}
password {{ monasca_agent_password }}
domain_id default

View File

@ -566,6 +566,7 @@
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_log_api') == -1
- haproxy_vip_prechecks
- monasca_log_api_port != monasca_api_port
- name: Checking free port for Monasca Log API public HAProxy
wait_for:
@ -580,6 +581,7 @@
- inventory_hostname in groups['haproxy']
- haproxy_stat.find('monasca_log_api_external') == -1
- haproxy_vip_prechecks
- monasca_log_api_port != monasca_api_port
- name: Checking free port for Monasca Grafana API internal HAProxy
wait_for:

View File

@ -20,24 +20,6 @@ monasca_services:
mode: "http"
external: true
port: "{{ monasca_api_port }}"
monasca-log-api:
container_name: monasca_log_api
group: monasca-log-api
enabled: true
image: "{{ monasca_log_api_image_full }}"
volumes: "{{ monasca_log_api_default_volumes + monasca_log_api_extra_volumes }}"
dimensions: "{{ monasca_log_api_dimensions }}"
haproxy:
monasca_log_api:
enabled: "{{ enable_monasca }}"
mode: "http"
external: false
port: "{{ monasca_log_api_port }}"
monasca_log_api_external:
enabled: "{{ enable_monasca }}"
mode: "http"
external: true
port: "{{ monasca_log_api_port }}"
monasca-log-transformer:
container_name: monasca_log_transformer
group: monasca-log-transformer
@ -211,10 +193,6 @@ monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc
monasca_api_tag: "{{ monasca_tag }}"
monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}"
monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-log-api"
monasca_log_api_tag: "{{ monasca_tag }}"
monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}"
monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-logstash"
monasca_logstash_tag: "{{ monasca_tag }}"
monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}"
@ -268,11 +246,6 @@ monasca_api_default_volumes:
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
monasca_log_api_default_volumes:
- "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
- "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}"
- "kolla_logs:/var/log/kolla"
monasca_log_transformer_default_volumes:
- "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro"
- "/etc/localtime:/etc/localtime:ro"
@ -315,7 +288,6 @@ monasca_agent_collector_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_agent_statsd_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_agent_forwarder_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_api_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_log_api_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_log_transformer_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_log_persister_extra_volumes: "{{ monasca_extra_volumes }}"
monasca_log_metrics_extra_volumes: "{{ monasca_extra_volumes }}"

View File

@ -14,21 +14,6 @@
when:
- kolla_action != "config"
- name: Restart monasca-log-api container
vars:
service_name: "monasca-log-api"
service: "{{ monasca_services[service_name] }}"
become: true
kolla_docker:
action: "recreate_or_restart_container"
common_options: "{{ docker_common_options }}"
name: "{{ service.container_name }}"
image: "{{ service.image }}"
volumes: "{{ service.volumes }}"
dimensions: "{{ service.dimensions }}"
when:
- kolla_action != "config"
- name: Restart monasca-log-transformer container
vars:
service_name: "monasca-log-transformer"

View File

@ -160,40 +160,6 @@
notify:
- Restart monasca-api container
- name: Copying over monasca-log-api config
vars:
service: "{{ monasca_services['monasca-log-api'] }}"
merge_configs:
sources:
- "{{ role_path }}/templates/monasca-log-api/{{ item }}.j2"
- "{{ node_custom_config }}/monasca/{{ item }}"
- "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}"
dest: "{{ node_config_directory }}/monasca-log-api/{{ item }}"
mode: "0660"
become: true
with_items:
- log-api.conf
- log-api-paste.ini
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-api container
- name: Copying over monasca-log-api wsgi config
vars:
service: "{{ monasca_services['monasca-log-api'] }}"
template:
src: "{{ role_path }}/templates/monasca-log-api/wsgi-log-api.conf.j2"
dest: "{{ node_config_directory }}/monasca-log-api/wsgi-log-api.conf"
mode: "0660"
become: true
when:
- inventory_hostname in groups[service['group']]
- service.enabled | bool
notify:
- Restart monasca-log-api container
- name: Copying over monasca-log-transformer config
vars:
service: "{{ monasca_services['monasca-log-transformer'] }}"

View File

@ -1,14 +1,12 @@
---
- include_tasks: register.yml
when: inventory_hostname in groups['monasca-agent'] or
inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-log-api']
inventory_hostname in groups['monasca-api']
- include_tasks: config.yml
when: inventory_hostname in groups['monasca-agent'] or
inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-grafana'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics'] or
@ -27,7 +25,6 @@
when: inventory_hostname in groups['monasca-agent'] or
inventory_hostname in groups['monasca-api'] or
inventory_hostname in groups['monasca-grafana'] or
inventory_hostname in groups['monasca-log-api'] or
inventory_hostname in groups['monasca-log-transformer'] or
inventory_hostname in groups['monasca-log-persister'] or
inventory_hostname in groups['monasca-log-metrics'] or

View File

@ -22,17 +22,6 @@
- inventory_hostname in groups[monasca_services['monasca-api']['group']]
- container_facts['monasca_api'] is not defined
- name: Checking free port for monasca-log-api
wait_for:
host: "{{ api_interface_address }}"
port: "{{ monasca_log_api_port }}"
connect_timeout: 1
timeout: 1
state: stopped
when:
- inventory_hostname in groups[monasca_services['monasca-log-api']['group']]
- container_facts['monasca_log_api'] is not defined
- name: Checking free port for monasca-agent-forwarder
wait_for:
host: "{{ api_interface_address }}"

View File

@ -1,6 +1,18 @@
---
# TODO(dszumski): Remove this in the V cycle once all old containers have been
# stopped.
- name: Stopping old monasca-log-api containers
become: true
kolla_docker:
action: "stop_and_remove_container"
common_options: "{{ docker_common_options }}"
name: "monasca_log_api"
- include_tasks: config.yml
- include_tasks: register.yml
when: inventory_hostname in groups['monasca-api']
- include_tasks: bootstrap_service.yml
- name: Flush handlers

View File

@ -3,6 +3,7 @@ log_file = monasca-api.log
log_dir = /var/log/kolla/monasca
debug = {{ monasca_logging_debug }}
region = {{ openstack_region_name }}
enable_logs_api = True
[database]
database = {{ monasca_database_name }}
@ -14,7 +15,9 @@ ip_address = {{ monasca_influxdb_address }}
port = {{ monasca_influxdb_http_port }}
[kafka]
legacy_kafka_client_enabled = False
metrics_topic = {{ monasca_metrics_topic }}
logs_topics = {{ monasca_raw_logs_topic }}
uri = {{ monasca_kafka_servers }}
[messaging]
@ -58,3 +61,7 @@ dimension_values = monasca_api.v2.reference.metrics:DimensionValues
dimension_names = monasca_api.v2.reference.metrics:DimensionNames
notification_method_types = monasca_api.v2.reference.notificationstype:NotificationsType
healthchecks = monasca_api.healthchecks:HealthChecks
[log_publisher]
# Increase the maximum payload size to slightly above the default Fluentd chunk size (8MB)
max_log_size = 10485760

View File

@ -5,7 +5,7 @@
"config_files": [
{
"source": "{{ container_config_directory }}/api.conf",
"dest": "/etc/monasca/api.conf",
"dest": "/etc/monasca/monasca-api.conf",
"owner": "monasca",
"perm": "0600"
},

View File

@ -1,65 +0,0 @@
#
# Copyright 2016-2017 FUJITSU LIMITED
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
[DEFAULT]
name = main
[composite:main]
use = egg:Paste#urlmap
/: la_version
/healthcheck: la_healthcheck
/v2.0: la_api_v2
/v3.0: la_api_v3
[pipeline:la_version]
pipeline = error_trap versionapp
[pipeline:la_healthcheck]
pipeline = error_trap healthcheckapp
[pipeline:la_api_v2]
pipeline = error_trap request_id auth api_v2_app
[pipeline:la_api_v3]
pipeline = error_trap request_id auth api_v3_app
[app:versionapp]
paste.app_factory = monasca_log_api.app.api:create_version_app
[app:healthcheckapp]
paste.app_factory = monasca_log_api.app.api:create_healthcheck_app
[app:api_v2_app]
paste.app_factory = monasca_log_api.app.api:create_api_app
set api_version=v2.0
[app:api_v3_app]
paste.app_factory = monasca_log_api.app.api:create_api_app
set api_version=v3.0
[filter:auth]
paste.filter_factory = keystonemiddleware.auth_token:filter_factory
[filter:roles]
paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory
[filter:request_id]
paste.filter_factory = oslo_middleware.request_id:RequestId.factory
[filter:error_trap]
paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory

View File

@ -1,43 +0,0 @@
[DEFAULT]
log_file = monasca-log-api.log
log_dir = /var/log/kolla/monasca
debug = {{ monasca_logging_debug }}
[service]
region = {{ openstack_region_name }}
# Increase the maximum payload size to slightly above the default Fluentd chunk size (8MB)
max_log_size = 10485760
[kafka_healthcheck]
kafka_topics = {{ monasca_raw_logs_topic }}
kafka_url = {{ monasca_kafka_servers }}
[log_publisher]
topics = {{ monasca_raw_logs_topic }}
kafka_url = {{ monasca_kafka_servers }}
[monitoring]
statsd_buffer = 30
statsd_port = {{ monasca_agent_statsd_port }}
[roles_middleware]
path = /v2.0/log, /v3.0/logs
default_roles = {{ monasca_default_authorized_roles|join(', ') }}
agent_roles = {{ monasca_agent_authorized_roles|join(', ') }}
delegate_roles = {{ monasca_delegate_authorized_roles|join(', ') }}
[keystone_authtoken]
www_authenticate_uri = {{ keystone_internal_url }}
auth_url = {{ keystone_admin_url }}
auth_type = password
project_domain_id = {{ default_project_domain_id }}
user_domain_id = {{ default_user_domain_id }}
project_name = service
username = {{ monasca_keystone_user }}
password = {{ monasca_keystone_password }}
service_token_roles_required=True
cafile = {{ openstack_cacert }}
memcache_security_strategy = ENCRYPT
memcache_secret_key = {{ memcache_secret_key }}
memcached_servers = {{ monasca_memcached_servers }}

View File

@ -1,32 +0,0 @@
{% set monasca_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %}
{% set wsgi_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %}
{
"command": "/usr/sbin/{{ monasca_cmd }} -DFOREGROUND",
"config_files": [
{
"source": "{{ container_config_directory }}/log-api.conf",
"dest": "/etc/monasca/log-api.conf",
"owner": "monasca",
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/log-api-paste.ini",
"dest": "/etc/monasca/log-api-paste.ini",
"owner": "monasca",
"perm": "0600"
},
{
"source": "{{ container_config_directory }}/wsgi-log-api.conf",
"dest": "/etc/{{ wsgi_conf_dir }}/wsgi-config.conf",
"owner": "monasca",
"perm": "0600"
}
],
"permissions": [
{
"path": "/var/log/kolla/monasca",
"owner": "monasca:kolla",
"recurse": true
}
]
}

View File

@ -1,34 +0,0 @@
{% set monasca_log_dir = '/var/log/kolla/monasca' %}
{% set python_path = '/usr/lib/python' ~ distro_python_version ~ '/site-packages' if monasca_install_type == 'binary' else '/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages' %}
{% set wsgi_path = '/usr/bin' if monasca_install_type == 'binary' else '/monasca-log/monasca_log_api/app' %}
Listen {{ api_interface_address | put_address_in_context('url') }}:{{ monasca_log_api_port }}
TraceEnable off
ErrorLog "{{ monasca_log_dir }}/apache-logapi-error.log"
<IfModule log_config_module>
CustomLog "{{ monasca_log_dir }}/apache-logapi-access.log" common
</IfModule>
{% if monasca_logging_debug | bool %}
LogLevel info
{% endif %}
<VirtualHost *:{{ monasca_log_api_port }}>
ErrorLog "{{ monasca_log_dir }}/monasca-log-api-error.log"
LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat
CustomLog "{{ monasca_log_dir }}/monasca-log-api-access.log" logformat
WSGIApplicationGroup %{GLOBAL}
WSGIDaemonProcess monasca-log-api group=monasca processes={{ openstack_service_workers }} threads=1 user=monasca python-path={{ python_path }}
WSGIProcessGroup monasca-log-api
WSGIScriptAlias / {{ wsgi_path }}/wsgi.py
WSGIPassAuthorization On
SetEnv no-gzip 1
<Directory "{{ wsgi_path }}">
Require all granted
</Directory>
</VirtualHost>

View File

@ -936,7 +936,6 @@
- monasca-agent
- monasca-api
- monasca-grafana
- monasca-log-api
- monasca-log-transformer
- monasca-log-persister
- monasca-log-metrics

View File

@ -0,0 +1,7 @@
---
upgrade:
- |
The Monasca Log API has been removed. All logs now go to the unified
Monasca API when Monasca is enabled. Any custom Fluentd configuration
and inventory files will need to be updated. Any monasca_log_api
containers will be removed automatically.

View File

@ -494,9 +494,6 @@ monasca
[monasca-grafana:children]
monasca
[monasca-log-api:children]
monasca
[monasca-log-transformer:children]
monasca