Support OSprofile usage

OSprofile allows user/devs trace OpenStack requests.

Implements: blueprint enable-osprofiler
Co-Authored-By: Bertrand Lallau <bertrand.lallau@gmail.com>
Change-Id: I82ea85d726011ef6cbf99380f395452d6d7f8053

ansible/group_vars/all.yml

@@ -356,6 +356,7 @@ enable_neutron_bgp_dragent: "no"
 enable_nova_serialconsole_proxy: "no"
 enable_octavia: "no"
 enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' | bool }}"
+enable_osprofiler: "no"
 enable_panko: "no"
 enable_rally: "no"
 enable_sahara: "no"
@@ -393,7 +394,7 @@ enable_destroy_images: "no"
 elasticsearch_address: "{{ kolla_internal_vip_address }}"
 elasticsearch_protocol: "{{ internal_protocol }}"
 
-enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool else 'no' }}"
+enable_elasticsearch: "{{ 'yes' if enable_central_logging | bool or enable_freezer | bool or enable_osprofiler | bool else 'no' }}"
 enable_kibana: "{{ 'yes' if enable_central_logging | bool else 'no' }}"
 
 ####################

ansible/roles/cinder/templates/cinder.conf.j2

@@ -152,3 +152,13 @@ hnas_svc0_hdp = {{ hnas_nfs_svc0_hdp }}
 
 [privsep_entrypoint]
 helper_command=sudo cinder-rootwrap /etc/cinder/rootwrap.conf privsep-helper --config-file /etc/cinder/cinder.conf
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/glance/templates/glance-api.conf.j2

@@ -91,3 +91,13 @@ driver = messagingv2
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/glance/templates/glance-registry.conf.j2

@@ -50,3 +50,13 @@ driver = messagingv2
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/heat/templates/heat.conf.j2

@@ -106,3 +106,13 @@ endpoint_type = publicURL
 [oslo_middleware]
 enable_proxy_headers_parsing = True
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/keystone/templates/keystone.conf.j2

@@ -53,3 +53,13 @@ memcache_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansib
 [oslo_messaging_notifications]
 driver = messagingv2
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/magnum/templates/magnum.conf.j2

@@ -76,3 +76,13 @@ topics = 'notifications'
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/mistral/templates/mistral.conf.j2

@@ -40,3 +40,13 @@ url = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ mistral_api_port }}
 
 [oslo_messaging_notifications]
 driver = noop
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/neutron/templates/neutron.conf.j2

@@ -130,3 +130,13 @@ allow_reverse_dns_lookup = True
 ipv4_ptr_zone_prefix_size = 24
 ipv6_ptr_zone_prefix_size = 116
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/nova/templates/nova.conf.j2

@@ -247,3 +247,13 @@ os_interface = internal
 {% if enable_ceilometer | bool or enable_searchlight | bool or enable_designate | bool %}
 notify_on_state_change = vm_and_task_state
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/searchlight/templates/searchlight.conf.j2

@@ -102,3 +102,15 @@ enabled = {{ enable_neutron | bool }}
 
 [resource_plugin:os_neutron_security_group]
 enabled = {{ enable_neutron | bool }}
+
+{# TODO(blallau): enabling osprofiler when fixed in searchlight #}
+
+{# {% if enable_osprofiler | bool %} #}
+{# [profiler] #}
+{# enabled = true #}
+{# trace_sqlalchemy = true #}
+{# hmac_keys = {{ osprofiler_secret }} #}
+{# {% if enable_elasticsearch | bool %} #}
+{# connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }} #}
+{# {% endif %} #}
+{# {% endif %} #}

ansible/roles/senlin/templates/senlin.conf.j2

@@ -50,3 +50,13 @@ topics = 'notifications'
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/trove/templates/trove-conductor.conf.j2

@@ -11,3 +11,13 @@ transport_url = rabbit://{% for host in groups['rabbitmq'] %}{{ rabbitmq_user }}
 [database]
 connection = mysql+pymysql://{{ trove_database_user }}:{{ trove_database_password }}@{{ trove_database_address }}/{{ trove_database_name }}
 max_retries = -1
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/trove/templates/trove-taskmanager.conf.j2

@@ -32,3 +32,13 @@ topics = 'notifications'
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/trove/templates/trove.conf.j2

@@ -43,3 +43,13 @@ topics = 'notifications'
 {% else %}
 driver = noop
 {% endif %}
+
+{% if enable_osprofiler | bool %}
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

ansible/roles/zun/templates/zun.conf.j2

@@ -53,7 +53,6 @@ memcache_security_strategy = ENCRYPT
 memcache_secret_key = {{ memcache_secret_key }}
 memcached_servers = {% for host in groups['memcached'] %}{{ hostvars[host]['ansible_' + hostvars[host]['api_interface']]['ipv4']['address'] }}:{{ memcached_port }}{% if not loop.last %},{% endif %}{% endfor %}
 
-
 [glance_client]
 auth_uri = {{ internal_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_public_port }}
 auth_url = {{ admin_protocol }}://{{ kolla_internal_fqdn }}:{{ keystone_admin_port }}
@@ -66,3 +65,13 @@ password = {{ zun_keystone_password }}
 region_name = {{ openstack_region_name }}
 endpoint_type = internalURL
 api_version = 2
+{% if enable_osprofiler | bool %}
+
+[profiler]
+enabled = true
+trace_sqlalchemy = true
+hmac_keys = {{ osprofiler_secret }}
+{% if enable_elasticsearch | bool %}
+connection_string = elasticsearch://{{ elasticsearch_address }}:{{ elasticsearch_port }}
+{% endif %}
+{% endif %}

doc/index.rst

@@ -71,6 +71,7 @@ Services
    networking-guide
    kuryr-guide
    zun-guide
+   osprofiler-guide
 
 Developer Docs
 ==============

doc/osprofiler-guide.rst

@@ -0,0 +1,55 @@
+.. _osprofiler-guide:
+
+===================
+OSprofiler in Kolla
+===================
+
+Overview
+========
+OSProfiler provides a tiny but powerful library that is used by most
+(soon to be all) OpenStack projects and their corresponding python clients
+as well as the Openstack client.
+It provides functionality to generate 1 trace per request, that goes
+through all involved services. This trace can then be extracted and used
+to build a tree of calls which can be quite handy for a variety of reasons
+(for example in isolating cross-project performance issues).
+
+Configuration on Kolla deployment
+---------------------------------
+
+Enable OSprofiler in ``/etc/kolla/globals.yml``
+
+.. code-block:: console
+
+    enable_osprofiler: "yes"
+    enable_elasticsearch: "yes"
+
+Verify operation
+----------------
+
+Retrieve ``osprofiler_secret`` key present at ``/etc/kolla/passwords.yml``.
+
+Profiler UUIDs can be created executing OpenStack clients (Nova, Glance, Cinder, Heat, Keystone)
+with ``--profile`` option or using the official Openstack client with ``--os-profile``.
+In example to get the OSprofiler trace UUID for ``openstack server create``.
+
+.. code-block:: console
+
+    $ openstack --os-profile <OSPROFILER_SECRET> \
+      server create \
+      --image cirros \
+      --flavor m1.tiny \
+      --key-name mykey \
+      --nic net-id=${NETWORK_ID} \
+      demo
+
+
+The previous command will output the command to retrieve OSprofiler trace.
+
+.. code-block:: console
+
+    $ osprofiler trace show --html <TRACE_ID> --connection-string elasticsearch://<api_interface_address>:9200
+
+For more information about how OSprofiler works, see
+`OSProfiler – Cross-project profiling library
+<https://docs.openstack.org/developer/osprofiler/>`__.

etc/kolla/globals.yml

@@ -180,6 +180,7 @@ kolla_internal_vip_address: "10.10.10.254"
 #enable_nova_serialconsole_proxy: "no"
 #enable_octavia: "no"
 #enable_openvswitch: "{{ neutron_plugin_agent != 'linuxbridge' | bool }}"
+#enable_osprofiler: "no"
 #enable_panko: "no"
 #enable_rally: "no"
 #enable_sahara: "no"

etc/kolla/passwords.yml

@@ -149,6 +149,9 @@ zun_keystone_password:
 
 memcache_secret_key:
 
+#HMAC secret key
+osprofiler_secret:
+
 nova_ssh_key:
   private_key:
   public_key:

kolla_ansible/cmd/genpwd.py

@@ -69,7 +69,8 @@ def main():
     blank_keys = ['docker_registry_password']
 
     # HMAC-MD5 keys
-    hmac_md5_keys = ['designate_rndc_key']
+    hmac_md5_keys = ['designate_rndc_key',
+                     'osprofiler_secret']
 
     # HMAC-SHA256 keys
     hmac_sha256_keys = ['barbican_crypto_key']

releasenotes/notes/osprofiler-support-ac1df8db57bf7789.yaml

@@ -0,0 +1,4 @@
+---
+features:
+  - |
+    Osprofiler support has been implemented in OpenStack services