diff --git a/ansible/group_vars/all.yml b/ansible/group_vars/all.yml index 9be48d7048..67e3a4ffb1 100644 --- a/ansible/group_vars/all.yml +++ b/ansible/group_vars/all.yml @@ -236,6 +236,7 @@ memcached_port: "11211" mistral_api_port: "8989" monasca_api_port: "8070" +monasca_log_api_port: "5607" mongodb_port: "27017" mongodb_web_port: "28017" @@ -519,7 +520,7 @@ enable_destroy_images: "no" #################### elasticsearch_address: "{{ kolla_internal_fqdn }}" -enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool else 'no' }}" +enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool or enable_skydive | bool or enable_monasca | bool else 'no' }}" enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}" #################### diff --git a/ansible/inventory/all-in-one b/ansible/inventory/all-in-one index 042f0caa74..2132ea2118 100644 --- a/ansible/inventory/all-in-one +++ b/ansible/inventory/all-in-one @@ -440,6 +440,9 @@ murano [monasca-api:children] monasca +[monasca-log-api:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/inventory/multinode b/ansible/inventory/multinode index 58e02990d0..1301b66e2d 100644 --- a/ansible/inventory/multinode +++ b/ansible/inventory/multinode @@ -449,6 +449,9 @@ murano [monasca-api:children] monasca +[monasca-log-api:children] +monasca + # Ironic [ironic-api:children] ironic diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 index e5e5c76d71..5bf39693fa 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 @@ -1,7 +1,7 @@ @type rewrite_tag_filter capitalize_regex_backreference yes - rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ apache_access + rewriterule1 programname ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ apache_access rewriterule2 programname ^(aodh_wsgi_access|barbican-api|zun_api_wsgi_access|vitrage_wsgi_access)$ wsgi_access rewriterule3 programname ^(nova-api|nova-compute|nova-compute-ironic|nova-conductor|nova-consoleauth|nova-manage|nova-novncproxy|nova-scheduler|nova-placement-api|placement-api|privsep-helper)$ openstack_python rewriterule4 programname ^(sahara-api|sahara-engine)$ openstack_python diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 index e1cf581c79..a2c7578338 100644 --- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 +++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 @@ -3,7 +3,7 @@ capitalize_regex_backreference yes key programname - pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|placement-api-access|panko-api-access)$ + pattern ^(cinder-api-access|cloudkitty-api-access|gnocchi-api-access|horizon-access|keystone-apache-admin-access|keystone-apache-public-access|monasca-api-access|monasca-log-api-access|placement-api-access|panko-api-access)$ tag apache_access diff --git a/ansible/roles/haproxy/templates/haproxy.cfg.j2 b/ansible/roles/haproxy/templates/haproxy.cfg.j2 index 1da33c008f..ee87bf08ac 100644 --- a/ansible/roles/haproxy/templates/haproxy.cfg.j2 +++ b/ansible/roles/haproxy/templates/haproxy.cfg.j2 @@ -175,6 +175,13 @@ listen monasca_api_internal server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_api_port }} check inter 2000 rise 2 fall 5 {% endfor %} +listen monasca_log_api_internal + bind {{ kolla_internal_vip_address }}:{{ monasca_log_api_port }} + http-request del-header X-Forwarded-Proto if { ssl_fc } +{% for host in groups['monasca-log-api'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_log_api_port }} check inter 2000 rise 2 fall 5 +{% endfor %} + {% if haproxy_enable_external_vip | bool %} listen monasca_api_external bind {{ kolla_external_vip_address }}:{{ monasca_api_port }} {{ tls_bind_info }} @@ -183,6 +190,14 @@ listen monasca_api_external {% for host in groups['monasca-api'] %} server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_api_port }} check inter 2000 rise 2 fall 5 {% endfor %} + +listen monasca_log_api_external + bind {{ kolla_external_vip_address }}:{{ monasca_log_api_port }} {{ tls_bind_info }} + http-request del-header X-Forwarded-Proto if { ssl_fc } + http-request set-header X-Forwarded-Proto https if { ssl_fc } +{% for host in groups['monasca-log-api'] %} + server {{ hostvars[host]['ansible_hostname'] }} {{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ monasca_log_api_port }} check inter 2000 rise 2 fall 5 +{% endfor %} {% endif %} {% endif %} diff --git a/ansible/roles/monasca/defaults/main.yml b/ansible/roles/monasca/defaults/main.yml index 593aad2562..d392153814 100644 --- a/ansible/roles/monasca/defaults/main.yml +++ b/ansible/roles/monasca/defaults/main.yml @@ -9,6 +9,15 @@ monasca_services: - "{{ node_config_directory }}/monasca-api/:{{ container_config_directory }}/:ro" - "/etc/localtime:/etc/localtime:ro" - "kolla_logs:/var/log/kolla" + monasca-log-api: + container_name: monasca_log_api + group: monasca-log-api + enabled: true + image: "{{ monasca_log_api_image_full }}" + volumes: + - "{{ node_config_directory }}/monasca-log-api/:{{ container_config_directory }}/:ro" + - "/etc/localtime:/etc/localtime:ro" + - "kolla_logs:/var/log/kolla" #################### # Databases @@ -37,6 +46,10 @@ monasca_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ doc monasca_api_tag: "{{ monasca_tag }}" monasca_api_image_full: "{{ monasca_api_image }}:{{ monasca_api_tag }}" +monasca_log_api_image: "{{ docker_registry ~ '/' if docker_registry else '' }}{{ docker_namespace }}/{{ kolla_base_distro }}-{{ monasca_install_type }}-monasca-log-api" +monasca_log_api_tag: "{{ monasca_tag }}" +monasca_log_api_image_full: "{{ monasca_log_api_image }}:{{ monasca_log_api_tag }}" + #################### # OpenStack #################### @@ -56,4 +69,8 @@ monasca_api_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_api_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_api_port }}/v2.0" monasca_api_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ monasca_api_port }}/v2.0" +monasca_log_api_admin_endpoint: "{{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_log_api_port }}" +monasca_log_api_internal_endpoint: "{{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ monasca_log_api_port }}" +monasca_log_api_public_endpoint: "{{ public_protocol }}://{{ kolla_external_fqdn }}:{{ monasca_log_api_port }}" + monasca_logging_debug: "{{ openstack_logging_debug }}" diff --git a/ansible/roles/monasca/handlers/main.yml b/ansible/roles/monasca/handlers/main.yml index df23d67399..eb26f72f77 100644 --- a/ansible/roles/monasca/handlers/main.yml +++ b/ansible/roles/monasca/handlers/main.yml @@ -19,3 +19,24 @@ or monasca_api_confs.changed | bool or monasca_api_wsgi_confs.changed | bool or monasca_api_container.changed | bool + +- name: Restart monasca-log-api container + vars: + service_name: "monasca-log-api" + service: "{{ monasca_services[service_name] }}" + config_json: "{{ monasca_config_jsons.results|selectattr('item.key', 'equalto', service_name)|first }}" + monasca_log_api_container: "{{ check_monasca_containers.results|selectattr('item.key', 'equalto', service_name)|first }}" + kolla_docker: + action: "recreate_or_restart_container" + common_options: "{{ docker_common_options }}" + name: "{{ service.container_name }}" + image: "{{ service.image }}" + volumes: "{{ service.volumes }}" + when: + - kolla_action != "config" + - inventory_hostname in groups[service.group] + - service.enabled | bool + - config_json.changed | bool + or monasca_log_api_confs.changed | bool + or monasca_log_api_wsgi_confs.changed | bool + or monasca_log_api_container.changed | bool diff --git a/ansible/roles/monasca/tasks/config.yml b/ansible/roles/monasca/tasks/config.yml index 70343d98ca..391321d6b8 100644 --- a/ansible/roles/monasca/tasks/config.yml +++ b/ansible/roles/monasca/tasks/config.yml @@ -26,6 +26,7 @@ with_dict: "{{ monasca_services }}" notify: - Restart monasca-api container + - Restart monasca-log-api container - name: Copying over monasca-api config vars: @@ -63,6 +64,42 @@ notify: - Restart monasca-api container +- name: Copying over monasca-log-api config + vars: + service: "{{ monasca_services['monasca-log-api'] }}" + merge_configs: + sources: + - "{{ role_path }}/templates/monasca-log-api/{{ item }}.j2" + - "{{ node_custom_config }}/monasca/{{ item }}" + - "{{ node_custom_config }}/monasca/{{ inventory_hostname }}/{{ item }}" + dest: "{{ node_config_directory }}/monasca-log-api/{{ item }}" + mode: "0660" + become: true + register: monasca_log_api_confs + with_items: + - log-api.conf + - log-api-paste.ini + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-api container + +- name: Copying over monasca-log-api wsgi config + vars: + service: "{{ monasca_services['monasca-log-api'] }}" + template: + src: "{{ role_path }}/templates/monasca-log-api/wsgi-log-api.conf.j2" + dest: "{{ node_config_directory }}/monasca-log-api/wsgi-log-api.conf" + mode: "0660" + become: true + register: monasca_log_api_wsgi_confs + when: + - inventory_hostname in groups[service['group']] + - service.enabled | bool + notify: + - Restart monasca-log-api container + - name: Check monasca containers kolla_docker: action: "compare_container" @@ -78,3 +115,4 @@ with_dict: "{{ monasca_services }}" notify: - Restart monasca-api container + - Restart monasca-log-api container diff --git a/ansible/roles/monasca/tasks/deploy.yml b/ansible/roles/monasca/tasks/deploy.yml index 9eb3e1c6bf..decfa5ff9f 100644 --- a/ansible/roles/monasca/tasks/deploy.yml +++ b/ansible/roles/monasca/tasks/deploy.yml @@ -1,9 +1,11 @@ --- - include: register.yml - when: inventory_hostname in groups['monasca-api'] + when: inventory_hostname in groups['monasca-api'] or + inventory_hostname in groups['monasca-log-api'] - include: config.yml - when: inventory_hostname in groups['monasca-api'] + when: inventory_hostname in groups['monasca-api'] or + inventory_hostname in groups['monasca-log-api'] - include: bootstrap.yml when: inventory_hostname in groups['monasca-api'] @@ -12,4 +14,5 @@ meta: flush_handlers - include: check.yml - when: inventory_hostname in groups['monasca-api'] + when: inventory_hostname in groups['monasca-api'] or + inventory_hostname in groups['monasca-log-api'] diff --git a/ansible/roles/monasca/tasks/precheck.yml b/ansible/roles/monasca/tasks/precheck.yml index 0b18498032..1e10b4c372 100644 --- a/ansible/roles/monasca/tasks/precheck.yml +++ b/ansible/roles/monasca/tasks/precheck.yml @@ -14,3 +14,14 @@ when: - inventory_hostname in groups[monasca_services['monasca-api']['group']] - container_facts['monasca_api'] is not defined + +- name: Checking free port for monasca-log-api + wait_for: + host: "{{ api_interface_address }}" + port: "{{ monasca_log_api_port }}" + connect_timeout: 1 + timeout: 1 + state: stopped + when: + - inventory_hostname in groups[monasca_services['monasca-log-api']['group']] + - container_facts['monasca_log_api'] is not defined diff --git a/ansible/roles/monasca/tasks/register.yml b/ansible/roles/monasca/tasks/register.yml index d55b54ecd1..d91a400d0f 100644 --- a/ansible/roles/monasca/tasks/register.yml +++ b/ansible/roles/monasca/tasks/register.yml @@ -17,6 +17,24 @@ - {'interface': 'internal', 'url': '{{ monasca_api_internal_endpoint }}'} - {'interface': 'public', 'url': '{{ monasca_api_public_endpoint }}'} +- name: Creating monasca-log-api service and endpoints + kolla_toolbox: + module_name: "kolla_keystone_service" + module_args: + service_name: "monasca-log-api" + service_type: "logging" + description: "Monasca logging as a service" + endpoint_region: "{{ openstack_region_name }}" + url: "{{ item.url }}" + interface: "{{ item.interface }}" + region_name: "{{ openstack_region_name }}" + auth: "{{ monasca_openstack_auth }}" + run_once: True + with_items: + - {'interface': 'admin', 'url': '{{ monasca_log_api_admin_endpoint }}'} + - {'interface': 'internal', 'url': '{{ monasca_log_api_internal_endpoint }}'} + - {'interface': 'public', 'url': '{{ monasca_log_api_public_endpoint }}'} + - name: Creating the monasca keystone user kolla_toolbox: module_name: "kolla_keystone_user" diff --git a/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 b/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 new file mode 100644 index 0000000000..de70b31b5f --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-api/log-api-paste.ini.j2 @@ -0,0 +1,65 @@ +# +# Copyright 2016-2017 FUJITSU LIMITED +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +[DEFAULT] +name = main + +[composite:main] +use = egg:Paste#urlmap +/: la_version +/healthcheck: la_healthcheck +/v2.0: la_api_v2 +/v3.0: la_api_v3 + +[pipeline:la_version] +pipeline = error_trap versionapp + +[pipeline:la_healthcheck] +pipeline = error_trap healthcheckapp + +[pipeline:la_api_v2] +pipeline = error_trap request_id auth roles api_v2_app + +[pipeline:la_api_v3] +pipeline = error_trap request_id auth roles api_v3_app + +[app:versionapp] +paste.app_factory = monasca_log_api.app.api:create_version_app + +[app:healthcheckapp] +paste.app_factory = monasca_log_api.app.api:create_healthcheck_app + +[app:api_v2_app] +paste.app_factory = monasca_log_api.app.api:create_api_app +set api_version=v2.0 + +[app:api_v3_app] +paste.app_factory = monasca_log_api.app.api:create_api_app +set api_version=v3.0 + +[filter:auth] +paste.filter_factory = keystonemiddleware.auth_token:filter_factory + +[filter:roles] +paste.filter_factory = monasca_log_api.middleware.role_middleware:RoleMiddleware.factory + +[filter:request_id] +paste.filter_factory = oslo_middleware.request_id:RequestId.factory + +[filter:error_trap] +paste.filter_factory = oslo_middleware.catch_errors:CatchErrors.factory + diff --git a/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 b/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 new file mode 100644 index 0000000000..82fe7e68de --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-api/log-api.conf.j2 @@ -0,0 +1,35 @@ +[DEFAULT] +log_file = monasca-log-api.log +log_dir = /var/log/kolla/monasca +debug = {{ monasca_logging_debug }} + +[kafka_healthcheck] +kafka_url = {{ monasca_kafka }} + +[log_publisher] +kafka_url = {{ monasca_kafka }} + +[monitoring] +# TODO(dszumski): Deploy statsd to use this +enable = false + +[roles_middleware] +path = /v2.0/log, /v3.0/logs +default_roles = {{ monasca_default_authorized_roles|join(', ') }} +agent_roles = {{ monasca_agent_authorized_roles|join(', ') }} +delegate_roles = {{ monasca_delegate_authorized_roles|join(', ') }} + +[keystone_authtoken] +auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }} +auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }} +auth_type = password +project_domain_id = {{ default_project_domain_id }} +user_domain_id = {{ default_user_domain_id }} +project_name = service +username = {{ monasca_keystone_user }} +password = {{ monasca_keystone_password }} +service_token_roles_required=True + +memcache_security_strategy = ENCRYPT +memcache_secret_key = {{ memcache_secret_key }} +memcached_servers = {{ monasca_memcached_servers }} diff --git a/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 b/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 new file mode 100644 index 0000000000..c01445b1b9 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-api/monasca-log-api.json.j2 @@ -0,0 +1,32 @@ +{% set monasca_cmd = 'apache2' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd' %} +{% set wsgi_conf_dir = 'apache2/conf-enabled' if kolla_base_distro in ['ubuntu', 'debian'] else 'httpd/conf.d' %} +{ + "command": "/usr/sbin/{{ monasca_cmd }} -DFOREGROUND", + "config_files": [ + { + "source": "{{ container_config_directory }}/log-api.conf", + "dest": "/etc/monasca/log-api.conf", + "owner": "monasca", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/log-api-paste.ini", + "dest": "/etc/monasca/log-api-paste.ini", + "owner": "monasca", + "perm": "0600" + }, + { + "source": "{{ container_config_directory }}/wsgi-log-api.conf", + "dest": "/etc/{{ wsgi_conf_dir }}/wsgi-config.conf", + "owner": "monasca", + "perm": "0600" + } + ], + "permissions": [ + { + "path": "/var/log/kolla/monasca", + "owner": "monasca:kolla", + "recurse": true + } + ] +} diff --git a/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 b/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 new file mode 100644 index 0000000000..dbc60a5f54 --- /dev/null +++ b/ansible/roles/monasca/templates/monasca-log-api/wsgi-log-api.conf.j2 @@ -0,0 +1,24 @@ +{% set python_path = '/usr/lib/python2.7/site-packages' if kolla_install_type == 'binary' else '/var/lib/kolla/venv/lib/python2.7/site-packages' %} +{% set wsgi_path = '/usr/bin' if monasca_install_type == 'binary' else '/monasca-log/monasca_log_api/app' %} + +Listen {{ api_interface_address }}:{{ monasca_log_api_port }} + +TraceEnable off + + + + ErrorLog "/var/log/kolla/monasca/monasca-log-api-error.log" + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b %D \"%{Referer}i\" \"%{User-Agent}i\"" logformat + CustomLog "/var/log/kolla/monasca/monasca-log-api-access.log" logformat + WSGIApplicationGroup %{GLOBAL} + WSGIDaemonProcess monasca-log-api group=monasca processes={{ openstack_service_workers }} threads=1 user=monasca python-path={{ python_path }} + WSGIProcessGroup monasca-log-api + WSGIScriptAlias / {{ wsgi_path }}/wsgi.py + WSGIPassAuthorization On + SetEnv no-gzip 1 + + + Require all granted + + + diff --git a/releasenotes/notes/add-monasca-log-api-d47662a4e643cd7f.yaml b/releasenotes/notes/add-monasca-log-api-d47662a4e643cd7f.yaml new file mode 100644 index 0000000000..fbafc01b12 --- /dev/null +++ b/releasenotes/notes/add-monasca-log-api-d47662a4e643cd7f.yaml @@ -0,0 +1,5 @@ +--- +features: + - Add support for deploying the Monasca Log API which forms part of the + Monasca distributed monitoring and logging as a service platform. + See https://wiki.openstack.org/wiki/Monasca for more details.