From 2bdad11ee27ab7d04a49f3bf9346f84f2992b10f Mon Sep 17 00:00:00 2001 From: Alexei Kornienko Date: Wed, 11 Sep 2013 12:33:49 +0300 Subject: [PATCH] Added openstack patches for nova This patches will be applied to nove source code before deployment Change-Id: Ia0c6beb44440c8572c1bd7a06646eb6525e73037 --- openstack-patches/README | 8 ++ .../0001-Adding-tomograph-middleware.patch | 59 ++++++++ .../nova/0002-Added-SQL-queries-trace.patch | 88 ++++++++++++ .../nova/0003-Added-RPC-trace.patch | 132 ++++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 openstack-patches/README create mode 100644 openstack-patches/nova/0001-Adding-tomograph-middleware.patch create mode 100644 openstack-patches/nova/0002-Added-SQL-queries-trace.patch create mode 100644 openstack-patches/nova/0003-Added-RPC-trace.patch diff --git a/openstack-patches/README b/openstack-patches/README new file mode 100644 index 0000000000..e205d20ef6 --- /dev/null +++ b/openstack-patches/README @@ -0,0 +1,8 @@ +This directory is used to patch existing openstack code + +All patches are applied to appropriate projects to add needed functionality (e.g. profiling code) + +git format-patch is used to create new patches + +Since this patches will become outdated we have a test that makes sure that they can be applied to latest openstack repositories. +In case this test fails patches must be updated diff --git a/openstack-patches/nova/0001-Adding-tomograph-middleware.patch b/openstack-patches/nova/0001-Adding-tomograph-middleware.patch new file mode 100644 index 0000000000..69be4dc1c1 --- /dev/null +++ b/openstack-patches/nova/0001-Adding-tomograph-middleware.patch @@ -0,0 +1,59 @@ +From d7c903d7e83aaef400242ef1a64fecd92e617dc8 Mon Sep 17 00:00:00 2001 +From: Alexei Kornienko +Date: Thu, 22 Aug 2013 15:28:07 +0300 +Subject: [PATCH 1/3] Adding tomograph middleware + +Change-Id: I33e4e87162851a748d49af6c1b15fabe4139bd7d +--- + etc/nova/api-paste.ini | 16 ++++++++++------ + requirements.txt | 3 +++ + 2 files changed, 13 insertions(+), 6 deletions(-) + +diff --git a/etc/nova/api-paste.ini b/etc/nova/api-paste.ini +index 0f8d2e1..583f2f7 100644 +--- a/etc/nova/api-paste.ini ++++ b/etc/nova/api-paste.ini +@@ -65,15 +65,15 @@ use = call:nova.api.openstack.urlmap:urlmap_factory + + [composite:openstack_compute_api_v2] + use = call:nova.api.auth:pipeline_factory +-noauth = faultwrap sizelimit noauth ratelimit osapi_compute_app_v2 +-keystone = faultwrap sizelimit authtoken keystonecontext ratelimit osapi_compute_app_v2 +-keystone_nolimit = faultwrap sizelimit authtoken keystonecontext osapi_compute_app_v2 ++noauth = faultwrap sizelimit tomograph noauth ratelimit osapi_compute_app_v2 ++keystone = faultwrap sizelimit tomograph authtoken keystonecontext ratelimit osapi_compute_app_v2 ++keystone_nolimit = faultwrap sizelimit tomograph authtoken keystonecontext osapi_compute_app_v2 + + [composite:openstack_compute_api_v3] + use = call:nova.api.auth:pipeline_factory +-noauth = faultwrap sizelimit noauth_v3 ratelimit_v3 osapi_compute_app_v3 +-keystone = faultwrap sizelimit authtoken keystonecontext ratelimit_v3 osapi_compute_app_v3 +-keystone_nolimit = faultwrap sizelimit authtoken keystonecontext osapi_compute_app_v3 ++noauth = faultwrap sizelimit tomograph noauth_v3 ratelimit_v3 osapi_compute_app_v3 ++keystone = faultwrap sizelimit tomograph authtoken keystonecontext ratelimit_v3 osapi_compute_app_v3 ++keystone_nolimit = faultwrap sizelimit tomograph authtoken keystonecontext osapi_compute_app_v3 + + [filter:faultwrap] + paste.filter_factory = nova.api.openstack:FaultWrapper.factory +@@ -126,3 +126,7 @@ admin_password = %SERVICE_PASSWORD% + #signing_dir = /var/lib/nova/keystone-signing + # Workaround for https://bugs.launchpad.net/nova/+bug/1154809 + auth_version = v2.0 ++ ++[filter:tomograph] ++paste.filter_factory = tomograph:Middleware.factory ++service_name = nova +diff --git a/requirements.txt b/requirements.txt +index 341a41e..3e9ae71 100644 +--- a/requirements.txt ++++ b/requirements.txt +@@ -31,3 +31,6 @@ websockify>=0.5.1,<0.6 + + -f http://tarballs.openstack.org/oslo.config/oslo.config-1.2.0a3.tar.gz#egg=oslo.config-1.2.0a3 + oslo.config>=1.2.0a3 ++ ++-f https://github.com/Alexei-Kornienko/tomograph/archive/master.tar.gz#egg=tomograph-0.0.1 ++tomograph>=0.0.1 +-- +1.8.1.2 + diff --git a/openstack-patches/nova/0002-Added-SQL-queries-trace.patch b/openstack-patches/nova/0002-Added-SQL-queries-trace.patch new file mode 100644 index 0000000000..846bfd966b --- /dev/null +++ b/openstack-patches/nova/0002-Added-SQL-queries-trace.patch @@ -0,0 +1,88 @@ +From 8a4337a1647aa0b668ff3be52fbdeebc30bda502 Mon Sep 17 00:00:00 2001 +From: Alexei Kornienko +Date: Fri, 23 Aug 2013 15:14:23 +0300 +Subject: [PATCH 2/3] Added SQL queries trace + +Change-Id: I3b64de08d5760f0a69b4a3c5abc020f4420755b0 +--- + nova/db/sqlalchemy/migration.py | 33 ++++++++++++++++---------- + nova/openstack/common/db/sqlalchemy/session.py | 5 ++++ + 2 files changed, 25 insertions(+), 13 deletions(-) + +diff --git a/nova/db/sqlalchemy/migration.py b/nova/db/sqlalchemy/migration.py +index 4170c93..a452b2f 100644 +--- a/nova/db/sqlalchemy/migration.py ++++ b/nova/db/sqlalchemy/migration.py +@@ -18,6 +18,7 @@ + + import distutils.version as dist_version + import os ++import tomograph.config + + from nova.db import migration + from nova import exception +@@ -63,19 +64,25 @@ get_engine = db_session.get_engine + + + def db_sync(version=None): +- if version is not None: +- try: +- version = int(version) +- except ValueError: +- raise exception.NovaException(_("version should be an integer")) +- +- current_version = db_version() +- repository = _find_migrate_repo() +- if version is None or version > current_version: +- return versioning_api.upgrade(get_engine(), repository, version) +- else: +- return versioning_api.downgrade(get_engine(), repository, +- version) ++ try: ++ tomograph.config.db_tracing_enabled = False ++ ++ if version is not None: ++ try: ++ version = int(version) ++ except ValueError: ++ msg = _("version should be an integer") ++ raise exception.NovaException(msg) ++ ++ current_version = db_version() ++ repository = _find_migrate_repo() ++ if version is None or version > current_version: ++ return versioning_api.upgrade(get_engine(), repository, version) ++ else: ++ return versioning_api.downgrade(get_engine(), repository, ++ version) ++ finally: ++ tomograph.config.db_tracing_enabled = True + + + def db_version(): +diff --git a/nova/openstack/common/db/sqlalchemy/session.py b/nova/openstack/common/db/sqlalchemy/session.py +index be1a78a..7dd9758 100644 +--- a/nova/openstack/common/db/sqlalchemy/session.py ++++ b/nova/openstack/common/db/sqlalchemy/session.py +@@ -244,6 +244,8 @@ Efficient use of soft deletes: + import os.path + import re + import time ++import tomograph ++import tomograph.config + + from eventlet import greenthread + from oslo.config import cfg +@@ -559,6 +561,9 @@ def get_engine(sqlite_fk=False, slave_engine=False): + if engine is None: + engine = create_engine(db_uri, + sqlite_fk=sqlite_fk) ++ if engine.name != 'sqlite': ++ sqlalchemy.event.listen(engine, 'before_execute', tomograph.before_execute('nova')) ++ sqlalchemy.event.listen(engine, 'after_execute', tomograph.after_execute('nova')) + if slave_engine: + _SLAVE_ENGINE = engine + else: +-- +1.8.1.2 + diff --git a/openstack-patches/nova/0003-Added-RPC-trace.patch b/openstack-patches/nova/0003-Added-RPC-trace.patch new file mode 100644 index 0000000000..56ac4c06ab --- /dev/null +++ b/openstack-patches/nova/0003-Added-RPC-trace.patch @@ -0,0 +1,132 @@ +From a89cec4349d2cfcd6dd37a89499a4ff09db86c58 Mon Sep 17 00:00:00 2001 +From: Alexei Kornienko +Date: Fri, 23 Aug 2013 12:39:59 +0300 +Subject: [PATCH 3/3] Added RPC trace + +Change-Id: I4b602abdf766df57217da2c705f05ff0cb6afe55 +--- + nova/openstack/common/rpc/__init__.py | 9 +++++++++ + nova/openstack/common/rpc/amqp.py | 10 ++++++++-- + 2 files changed, 17 insertions(+), 2 deletions(-) + +diff --git a/nova/openstack/common/rpc/__init__.py b/nova/openstack/common/rpc/__init__.py +index fc2a111..0d28214 100644 +--- a/nova/openstack/common/rpc/__init__.py ++++ b/nova/openstack/common/rpc/__init__.py +@@ -27,6 +27,8 @@ For some wrappers that add message versioning to rpc, see: + + import inspect + ++import tomograph ++ + from oslo.config import cfg + + from nova.openstack.common.gettextutils import _ # noqa +@@ -136,6 +138,7 @@ def call(context, topic, msg, timeout=None, check_for_lock=False): + """ + if check_for_lock: + _check_for_lock() ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().call(CONF, context, topic, msg, timeout) + + +@@ -154,6 +157,7 @@ def cast(context, topic, msg): + + :returns: None + """ ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().cast(CONF, context, topic, msg) + + +@@ -175,6 +179,7 @@ def fanout_cast(context, topic, msg): + + :returns: None + """ ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().fanout_cast(CONF, context, topic, msg) + + +@@ -209,6 +214,7 @@ def multicall(context, topic, msg, timeout=None, check_for_lock=False): + """ + if check_for_lock: + _check_for_lock() ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().multicall(CONF, context, topic, msg, timeout) + + +@@ -223,6 +229,7 @@ def notify(context, topic, msg, envelope=False): + + :returns: None + """ ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().notify(cfg.CONF, context, topic, msg, envelope) + + +@@ -251,6 +258,7 @@ def cast_to_server(context, server_params, topic, msg): + + :returns: None + """ ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().cast_to_server(CONF, context, server_params, topic, + msg) + +@@ -267,6 +275,7 @@ def fanout_cast_to_server(context, server_params, topic, msg): + + :returns: None + """ ++ msg['trace_info'] = tomograph.get_trace_info() + return _get_impl().fanout_cast_to_server(CONF, context, server_params, + topic, msg) + +diff --git a/nova/openstack/common/rpc/amqp.py b/nova/openstack/common/rpc/amqp.py +index bde9e43..54d3acc 100644 +--- a/nova/openstack/common/rpc/amqp.py ++++ b/nova/openstack/common/rpc/amqp.py +@@ -28,6 +28,8 @@ AMQP, but is deprecated and predates this code. + import collections + import inspect + import sys ++import socket ++import tomograph + import uuid + + from eventlet import greenpool +@@ -438,15 +440,16 @@ class ProxyCallback(_ThreadPoolWithWait): + args = message_data.get('args', {}) + version = message_data.get('version') + namespace = message_data.get('namespace') ++ trace_info = message_data.get('trace_info', None) + if not method: + LOG.warn(_('no method for message: %s') % message_data) + ctxt.reply(_('No method for message: %s') % message_data, + connection_pool=self.connection_pool) + return + self.pool.spawn_n(self._process_data, ctxt, version, method, +- namespace, args) ++ namespace, args, trace_info) + +- def _process_data(self, ctxt, version, method, namespace, args): ++ def _process_data(self, ctxt, version, method, namespace, args, trace_info): + """Process a message in a new thread. + + If the proxy object we have has a dispatch method +@@ -455,6 +458,7 @@ class ProxyCallback(_ThreadPoolWithWait): + the old behavior of magically calling the specified method on the + proxy we have here. + """ ++ tomograph.start("rpc", str(method), socket.gethostname(), 0, trace_info) + ctxt.update_store() + try: + rval = self.proxy.dispatch(ctxt, version, method, namespace, +@@ -479,6 +483,8 @@ class ProxyCallback(_ThreadPoolWithWait): + LOG.error(_('Exception during message handling'), + exc_info=exc_info) + ctxt.reply(None, exc_info, connection_pool=self.connection_pool) ++ finally: ++ tomograph.stop(str(method)) + + + class MulticallProxyWaiter(object): +-- +1.8.1.2 +