From b39a0f805ae1d90b4406fb54d0672789e3c01348 Mon Sep 17 00:00:00 2001 From: Doug Szumski Date: Thu, 14 May 2020 16:21:45 +0100 Subject: [PATCH] Switch to Monasca API for logs The Monasca Log API has been removed and in this change we switch to using the unified API. If dedicated log APIs are required then this can be supported through configuration. Out of the box the Monasca API is used for both logs and metrics which is envisaged to work for most use cases. In order to use the unified API for logs, we need to disable the legacy Kafka client. We also rename the Monasca API config file to remove a warning about using the old style name. Depends-On: https://review.opendev.org/#/c/728638 Change-Id: I9b6bf5b6690f4b4b3445e7d15a40e45dd42d2e84 --- ansible/group_vars/all.yml | 2 +- ansible/inventory/all-in-one | 3 - ansible/inventory/multinode | 3 - .../conf/filter/01-rewrite-0.12.conf.j2 | 2 +- .../conf/filter/01-rewrite-0.14.conf.j2 | 2 +- .../templates/conf/input/00-global.conf.j2 | 2 +- .../templates/conf/output/00-local.conf.j2 | 12 ++-- .../templates/conf/output/02-monasca.conf.j2 | 4 +- ansible/roles/haproxy/tasks/precheck.yml | 2 + ansible/roles/monasca/defaults/main.yml | 28 -------- ansible/roles/monasca/handlers/main.yml | 15 ----- ansible/roles/monasca/tasks/config.yml | 34 ---------- ansible/roles/monasca/tasks/deploy.yml | 5 +- ansible/roles/monasca/tasks/precheck.yml | 11 ---- ansible/roles/monasca/tasks/upgrade.yml | 12 ++++ .../{api.conf.j2 => monasca-api.conf.j2} | 7 ++ .../templates/monasca-api/monasca-api.json.j2 | 2 +- .../monasca-log-api/log-api-paste.ini.j2 | 65 ------------------- .../templates/monasca-log-api/log-api.conf.j2 | 43 ------------ .../monasca-log-api/monasca-log-api.json.j2 | 32 --------- .../monasca-log-api/wsgi-log-api.conf.j2 | 34 ---------- ansible/site.yml | 1 - ...monasca-api-for-logs-8c6829e7fb1ccf8b.yaml | 7 ++ tests/templates/inventory.j2 | 3 - 24 files changed, 42 insertions(+), 289 deletions(-) rename ansible/roles/monasca/templates/monasca-api/{api.conf.j2 => monasca-api.conf.j2} (91%) delete mode 100644 ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 delete mode 100644 ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 delete mode 100644 ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 delete mode 100644 ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 create mode 100644 releasenotes/notes/use-unified-monasca-api-for-logs-8c6829e7fb1ccf8b.yaml diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index b4ef05a7a8..33abfd2460 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -344,7 +344,7 @@ memcached_port: "11211" mistral_api_port: "8989" monasca_api_port: "8070" -monasca_log_api_port: "5607" +monasca_log_api_port: "{{ monasca_api_port }}" monasca_agent_forwarder_port: "17123" monasca_agent_statsd_port: "8125" monasca_grafana_server_port: "3001" diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 58b3a41fd6..bc0022f76e 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -442,9 +442,6 @@ monasca [monasca-grafana:children] monasca -[monasca-log-api:children] -monasca - [monasca-log-transformer:children] monasca diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index dc35e07185..19b9fe2590 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -451,9 +451,6 @@ monasca [monasca-grafana:children] monasca -[monasca-log-api:children] -monasca - [monasca-log-transformer:children] monasca diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 index d6b62b8257..e36bbd183c 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 @@ -1,7 +1,7 @@ @type rewrite_tag_filter capitalize_regex_backreference yes - rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ apache_access + rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ apache_access rewriterule2 programname ^(aodh_wsgi_access|barbican-api|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 index 81a8979cdb..8252984f78 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 @@ -3,7 +3,7 @@ capitalize_regex_backreference yes key programname - pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ + pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ tag apache_access diff --git a/ansible/roles/common/templates/conf/input/00-global.conf.j2 b/ansible/roles/common/templates/conf/input/00-global.conf.j2 index 59dc6d24e1..b7fba5068b 100644 --- a/ansible/roles/common/templates/conf/input/00-global.conf.j2 +++ b/ansible/roles/common/templates/conf/input/00-global.conf.j2 @@ -5,7 +5,7 @@ path {% for service in fluentd_enabled_input_openstack_services %}/var/log/kolla/{{ service }}/*.log{% if not loop.last %},{% endif %}{% endfor %} exclude_path ["/var/log/kolla/monasca/agent*.log", "/var/log/kolla/monasca/grafana.log", - "/var/log/kolla/monasca/monasca-log-api.log", + "/var/log/kolla/monasca/monasca-api.log", "/var/log/kolla/neutron/dnsmasq.log", "/var/log/kolla/*/*-access.log", "/var/log/kolla/*/*-error.log", diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2 index 4f417108f4..2a826bc648 100644 --- a/ansible/roles/common/templates/conf/output/00-local.conf.j2 +++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2 @@ -38,8 +38,8 @@ @type monasca keystone_url {{ keystone_internal_url }} - monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} - monasca_log_api_version v3.0 + monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} + monasca_api_version v2.0 username {{ monasca_agent_user }} password {{ monasca_agent_password }} domain_id default @@ -95,8 +95,8 @@ @type monasca keystone_url {{ keystone_internal_url }} - monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} - monasca_log_api_version v3.0 + monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} + monasca_api_version v2.0 username {{ monasca_agent_user }} password {{ monasca_agent_password }} domain_id default @@ -152,8 +152,8 @@ @type monasca keystone_url {{ keystone_internal_url }} - monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} - monasca_log_api_version v3.0 + monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} + monasca_api_version v2.0 username {{ monasca_agent_user }} password {{ monasca_agent_password }} domain_id default diff --git a/ansible/roles/common/templates/conf/output/02-monasca.conf.j2 b/ansible/roles/common/templates/conf/output/02-monasca.conf.j2 index f4ed33e0c0..f0d9cc5923 100644 --- a/ansible/roles/common/templates/conf/output/02-monasca.conf.j2 +++ b/ansible/roles/common/templates/conf/output/02-monasca.conf.j2 @@ -3,8 +3,8 @@ @type monasca keystone_url {{ keystone_internal_url }} - monasca_log_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} - monasca_log_api_version v3.0 + monasca_api {{ internal_protocol }}://{{ kolla_internal_fqdn | put_address_in_context('url') }}:{{ monasca_log_api_port }} + monasca_api_version v2.0 username {{ monasca_agent_user }} password {{ monasca_agent_password }} domain_id default diff --git a/ansible/roles/haproxy/tasks/precheck.yml b/ansible/roles/haproxy/tasks/precheck.yml index 3e40a5051d..585f9972e3 100644 --- a/ansible/roles/haproxy/tasks/precheck.yml +++ b/ansible/roles/haproxy/tasks/precheck.yml @@ -566,6 +566,7 @@ - inventory_hostname in groups['haproxy'] - haproxy_stat.find('monasca_log_api') == -1 - haproxy_vip_prechecks + - monasca_log_api_port != monasca_api_port - name: Checking free port for Monasca Log API public HAProxy wait_for: @@ -580,6 +581,7 @@ - inventory_hostname in groups['haproxy'] - haproxy_stat.find('monasca_log_api_external') == -1 - haproxy_vip_prechecks + - monasca_log_api_port != monasca_api_port - name: Checking free port for Monasca Grafana API internal HAProxy wait_for: diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 2595ade979..18c5624a28 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -20,24 +20,6 @@ monasca_services: mode: "http" external: true port: "{{ monasca_api_port }}" - monasca-log-api: - container_name: monasca_log_api - group: monasca-log-api - enabled: true - image: "{{ monasca_log_api_image_full }}" - volumes: "{{ monasca_log_api_default_volumes + monasca_log_api_extra_volumes }}" - dimensions: "{{ monasca_log_api_dimensions }}" - haproxy: - monasca_log_api: - enabled: "{{ enable_monasca }}" - mode: "http" - external: false - port: "{{ monasca_log_api_port }}" - monasca_log_api_external: - enabled: "{{ enable_monasca }}" - mode: "http" - external: true - port: "{{ monasca_log_api_port }}" monasca-log-transformer: container_name: monasca_log_transformer group: monasca-log-transformer @@ -211,10 +193,6 @@ monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc monasca_api_tag: "{{ monasca_tag }}" monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}" -monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-log-api" -monasca_log_api_tag: "{{ monasca_tag }}" -monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}" - monasca_logstash_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ kolla_install_type }}-logstash" monasca_logstash_tag: "{{ monasca_tag }}" monasca_logstash_image_full: "{{ monasca_logstash_image }}:{{ monasca_logstash_tag }}" @@ -268,11 +246,6 @@ monasca_api_default_volumes: - "/etc/localtime:/etc/localtime:ro" - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - "kolla_logs:/var/log/kolla" -monasca_log_api_default_volumes: - - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro" - - "/etc/localtime:/etc/localtime:ro" - - "{{ '/etc/timezone:/etc/timezone:ro' if kolla_base_distro in ['debian', 'ubuntu'] else '' }}" - - "kolla_logs:/var/log/kolla" monasca_log_transformer_default_volumes: - "{{ node_config_directory }}/monasca-log-transformer/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" @@ -315,7 +288,6 @@ monasca_agent_collector_extra_volumes: "{{ monasca_extra_volumes }}" monasca_agent_statsd_extra_volumes: "{{ monasca_extra_volumes }}" monasca_agent_forwarder_extra_volumes: "{{ monasca_extra_volumes }}" monasca_api_extra_volumes: "{{ monasca_extra_volumes }}" -monasca_log_api_extra_volumes: "{{ monasca_extra_volumes }}" monasca_log_transformer_extra_volumes: "{{ monasca_extra_volumes }}" monasca_log_persister_extra_volumes: "{{ monasca_extra_volumes }}" monasca_log_metrics_extra_volumes: "{{ monasca_extra_volumes }}" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index fa1e811d5b..a8fe79761e 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -14,21 +14,6 @@ when: - kolla_action != "config" -- name: Restart monasca-log-api container - vars: - service_name: "monasca-log-api" - service: "{{ monasca_services[service_name] }}" - become: true - kolla_docker: - action: "recreate_or_restart_container" - common_options: "{{ docker_common_options }}" - name: "{{ service.container_name }}" - image: "{{ service.image }}" - volumes: "{{ service.volumes }}" - dimensions: "{{ service.dimensions }}" - when: - - kolla_action != "config" - - name: Restart monasca-log-transformer container vars: service_name: "monasca-log-transformer" diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index df12ddd49d..7f97189ec6 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -160,40 +160,6 @@ notify: - Restart monasca-api container -- name: Copying over monasca-log-api config - vars: - service: "{{ monasca_services['monasca-log-api'] }}" - merge_configs: - sources: - - "{{ role_path }}/templates/monasca-log-api/{{ item }}.j2" - - "{{ node_custom_config }}/monasca/{{ item }}" - - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}" - dest: "{{ node_config_directory }}/monasca-log-api/{{ item }}" - mode: "0660" - become: true - with_items: - - log-api.conf - - log-api-paste.ini - when: - - inventory_hostname in groups[service['group']] - - service.enabled | bool - notify: - - Restart monasca-log-api container - -- name: Copying over monasca-log-api wsgi config - vars: - service: "{{ monasca_services['monasca-log-api'] }}" - template: - src: "{{ role_path }}/templates/monasca-log-api/wsgi-log-api.conf.j2" - dest: "{{ node_config_directory }}/monasca-log-api/wsgi-log-api.conf" - mode: "0660" - become: true - when: - - inventory_hostname in groups[service['group']] - - service.enabled | bool - notify: - - Restart monasca-log-api container - - name: Copying over monasca-log-transformer config vars: service: "{{ monasca_services['monasca-log-transformer'] }}" diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index 88878ee2b8..705ba14d4b 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -1,14 +1,12 @@ --- - include_tasks: register.yml when: inventory_hostname in groups['monasca-agent'] or - inventory_hostname in groups['monasca-api'] or - inventory_hostname in groups['monasca-log-api'] + inventory_hostname in groups['monasca-api'] - include_tasks: config.yml when: inventory_hostname in groups['monasca-agent'] or inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-grafana'] or - inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or inventory_hostname in groups['monasca-log-metrics'] or @@ -27,7 +25,6 @@ when: inventory_hostname in groups['monasca-agent'] or inventory_hostname in groups['monasca-api'] or inventory_hostname in groups['monasca-grafana'] or - inventory_hostname in groups['monasca-log-api'] or inventory_hostname in groups['monasca-log-transformer'] or inventory_hostname in groups['monasca-log-persister'] or inventory_hostname in groups['monasca-log-metrics'] or diff --git a/ansible/roles/monasca/tasks/precheck.yml b/ansible/roles/monasca/tasks/precheck.yml index 392cb26cdc..f8ce136638 100644 --- a/ansible/roles/monasca/tasks/precheck.yml +++ b/ansible/roles/monasca/tasks/precheck.yml @@ -22,17 +22,6 @@ - inventory_hostname in groups[monasca_services['monasca-api']['group']] - container_facts['monasca_api'] is not defined -- name: Checking free port for monasca-log-api - wait_for: - host: "{{ api_interface_address }}" - port: "{{ monasca_log_api_port }}" - connect_timeout: 1 - timeout: 1 - state: stopped - when: - - inventory_hostname in groups[monasca_services['monasca-log-api']['group']] - - container_facts['monasca_log_api'] is not defined - - name: Checking free port for monasca-agent-forwarder wait_for: host: "{{ api_interface_address }}" diff --git a/ansible/roles/monasca/tasks/upgrade.yml b/ansible/roles/monasca/tasks/upgrade.yml index 20ccddc8f1..0a96ea6f21 100644 --- a/ansible/roles/monasca/tasks/upgrade.yml +++ b/ansible/roles/monasca/tasks/upgrade.yml @@ -1,6 +1,18 @@ --- +# TODO(dszumski): Remove this in the V cycle once all old containers have been +# stopped. +- name: Stopping old monasca-log-api containers + become: true + kolla_docker: + action: "stop_and_remove_container" + common_options: "{{ docker_common_options }}" + name: "monasca_log_api" + - include_tasks: config.yml +- include_tasks: register.yml + when: inventory_hostname in groups['monasca-api'] + - include_tasks: bootstrap_service.yml - name: Flush handlers diff --git a/ansible/roles/monasca/templates/monasca-api/api.conf.j2 b/ansible/roles/monasca/templates/monasca-api/monasca-api.conf.j2 similarity index 91% rename from ansible/roles/monasca/templates/monasca-api/api.conf.j2 rename to ansible/roles/monasca/templates/monasca-api/monasca-api.conf.j2 index 105ae7ac1d..a0ddb4ceef 100644 --- a/ansible/roles/monasca/templates/monasca-api/api.conf.j2 +++ b/ansible/roles/monasca/templates/monasca-api/monasca-api.conf.j2 @@ -3,6 +3,7 @@ log_file = monasca-api.log log_dir = /var/log/kolla/monasca debug = {{ monasca_logging_debug }} region = {{ openstack_region_name }} +enable_logs_api = True [database] database = {{ monasca_database_name }} @@ -14,7 +15,9 @@ ip_address = {{ monasca_influxdb_address }} port = {{ monasca_influxdb_http_port }} [kafka] +legacy_kafka_client_enabled = False metrics_topic = {{ monasca_metrics_topic }} +logs_topics = {{ monasca_raw_logs_topic }} uri = {{ monasca_kafka_servers }} [messaging] @@ -58,3 +61,7 @@ dimension_values = monasca_api.v2.reference.metrics:DimensionValues dimension_names = monasca_api.v2.reference.metrics:DimensionNames notification_method_types = monasca_api.v2.reference.notificationstype:NotificationsType healthchecks = monasca_api.healthchecks:HealthChecks + +[log_publisher] +# Increase the maximum payload size to slightly above the default Fluentd chunk size (8MB) +max_log_size = 10485760 diff --git a/ansible/roles/monasca/templates/monasca-api/monasca-api.json.j2 b/ansible/roles/monasca/templates/monasca-api/monasca-api.json.j2 index 0ccd4b3b11..657a92de95 100644 --- a/ansible/roles/monasca/templates/monasca-api/monasca-api.json.j2 +++ b/ansible/roles/monasca/templates/monasca-api/monasca-api.json.j2 @@ -5,7 +5,7 @@ "config_files": [ { "source": "{{ container_config_directory }}/api.conf", - "dest": "/etc/monasca/api.conf", + "dest": "/etc/monasca/monasca-api.conf", "owner": "monasca", "perm": "0600" }, diff --git a/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 b/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 deleted file mode 100644 index 8aa9a8f1e4..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 +++ /dev/null @@ -1,65 +0,0 @@ -# -# Copyright 2016-2017 FUJITSU LIMITED -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -[DEFAULT] -name = main - -[composite:main] -use = egg:Paste#urlmap -/: la_version -/healthcheck: la_healthcheck -/v2.0: la_api_v2 -/v3.0: la_api_v3 - -[pipeline:la_version] -pipeline = error_trap versionapp - -[pipeline:la_healthcheck] -pipeline = error_trap healthcheckapp - -[pipeline:la_api_v2] -pipeline = error_trap request_id auth api_v2_app - -[pipeline:la_api_v3] -pipeline = error_trap request_id auth api_v3_app - -[app:versionapp] -paste.app_factory = monasca_log_api.app.api:create_version_app - -[app:healthcheckapp] -paste.app_factory = monasca_log_api.app.api:create_healthcheck_app - -[app:api_v2_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v2.0 - -[app:api_v3_app] -paste.app_factory = monasca_log_api.app.api:create_api_app -set api_version=v3.0 - -[filter:auth] -paste.filter_factory = keystonemiddleware.auth_token:filter_factory - -[filter:roles] -paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory - -[filter:request_id] -paste.filter_factory = oslo_middleware.request_id:RequestId.factory - -[filter:error_trap] -paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory - diff --git a/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 b/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 deleted file mode 100644 index 56946c8bdd..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 +++ /dev/null @@ -1,43 +0,0 @@ -[DEFAULT] -log_file = monasca-log-api.log -log_dir = /var/log/kolla/monasca -debug = {{ monasca_logging_debug }} - -[service] -region = {{ openstack_region_name }} -# Increase the maximum payload size to slightly above the default Fluentd chunk size (8MB) -max_log_size = 10485760 - -[kafka_healthcheck] -kafka_topics = {{ monasca_raw_logs_topic }} -kafka_url = {{ monasca_kafka_servers }} - -[log_publisher] -topics = {{ monasca_raw_logs_topic }} -kafka_url = {{ monasca_kafka_servers }} - -[monitoring] -statsd_buffer = 30 -statsd_port = {{ monasca_agent_statsd_port }} - -[roles_middleware] -path = /v2.0/log, /v3.0/logs -default_roles = {{ monasca_default_authorized_roles|join(', ') }} -agent_roles = {{ monasca_agent_authorized_roles|join(', ') }} -delegate_roles = {{ monasca_delegate_authorized_roles|join(', ') }} - -[keystone_authtoken] -www_authenticate_uri = {{ keystone_internal_url }} -auth_url = {{ keystone_admin_url }} -auth_type = password -project_domain_id = {{ default_project_domain_id }} -user_domain_id = {{ default_user_domain_id }} -project_name = service -username = {{ monasca_keystone_user }} -password = {{ monasca_keystone_password }} -service_token_roles_required=True -cafile = {{ openstack_cacert }} - -memcache_security_strategy = ENCRYPT -memcache_secret_key = {{ memcache_secret_key }} -memcached_servers = {{ monasca_memcached_servers }} diff --git a/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 b/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 deleted file mode 100644 index c01445b1b9..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 +++ /dev/null @@ -1,32 +0,0 @@ -{% set monasca_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %} -{% set wsgi_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %} -{ - "command": "/usr/sbin/{{ monasca_cmd }} -DFOREGROUND", - "config_files": [ - { - "source": "{{ container_config_directory }}/log-api.conf", - "dest": "/etc/monasca/log-api.conf", - "owner": "monasca", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/log-api-paste.ini", - "dest": "/etc/monasca/log-api-paste.ini", - "owner": "monasca", - "perm": "0600" - }, - { - "source": "{{ container_config_directory }}/wsgi-log-api.conf", - "dest": "/etc/{{ wsgi_conf_dir }}/wsgi-config.conf", - "owner": "monasca", - "perm": "0600" - } - ], - "permissions": [ - { - "path": "/var/log/kolla/monasca", - "owner": "monasca:kolla", - "recurse": true - } - ] -} diff --git a/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 b/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 deleted file mode 100644 index 328fee0a6d..0000000000 --- a/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 +++ /dev/null @@ -1,34 +0,0 @@ -{% set monasca_log_dir = '/var/log/kolla/monasca' %} -{% set python_path = '/usr/lib/python' ~ distro_python_version ~ '/site-packages' if monasca_install_type == 'binary' else '/var/lib/kolla/venv/lib/python' ~ distro_python_version ~ '/site-packages' %} -{% set wsgi_path = '/usr/bin' if monasca_install_type == 'binary' else '/monasca-log/monasca_log_api/app' %} - -Listen {{ api_interface_address | put_address_in_context('url') }}:{{ monasca_log_api_port }} - -TraceEnable off - -ErrorLog "{{ monasca_log_dir }}/apache-logapi-error.log" - - CustomLog "{{ monasca_log_dir }}/apache-logapi-access.log" common - - -{% if monasca_logging_debug | bool %} -LogLevel info -{% endif %} - - - - ErrorLog "{{ monasca_log_dir }}/monasca-log-api-error.log" - LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat - CustomLog "{{ monasca_log_dir }}/monasca-log-api-access.log" logformat - WSGIApplicationGroup %{GLOBAL} - WSGIDaemonProcess monasca-log-api group=monasca processes={{ openstack_service_workers }} threads=1 user=monasca python-path={{ python_path }} - WSGIProcessGroup monasca-log-api - WSGIScriptAlias / {{ wsgi_path }}/wsgi.py - WSGIPassAuthorization On - SetEnv no-gzip 1 - - - Require all granted - - - diff --git a/ansible/site.yml b/ansible/site.yml index 6d3fb8584d..04bc20cf0e 100644 --- a/ansible/site.yml +++ b/ansible/site.yml @@ -936,7 +936,6 @@ - monasca-agent - monasca-api - monasca-grafana - - monasca-log-api - monasca-log-transformer - monasca-log-persister - monasca-log-metrics diff --git a/releasenotes/notes/use-unified-monasca-api-for-logs-8c6829e7fb1ccf8b.yaml b/releasenotes/notes/use-unified-monasca-api-for-logs-8c6829e7fb1ccf8b.yaml new file mode 100644 index 0000000000..52cbe1e1ee --- /dev/null +++ b/releasenotes/notes/use-unified-monasca-api-for-logs-8c6829e7fb1ccf8b.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + The Monasca Log API has been removed. All logs now go to the unified + Monasca API when Monasca is enabled. Any custom Fluentd configuration + and inventory files will need to be updated. Any monasca_log_api + containers will be removed automatically. diff --git a/tests/templates/inventory.j2 b/tests/templates/inventory.j2 index 229fc7bc35..c99c629072 100644 --- a/tests/templates/inventory.j2 +++ b/tests/templates/inventory.j2 @@ -494,9 +494,6 @@ monasca [monasca-grafana:children] monasca -[monasca-log-api:children] -monasca - [monasca-log-transformer:children] monasca