openstack/osprofiler
Code Issues Proposed changes

Merge "Allow N-keys (one should apply)"

Browse Source
This commit is contained in:
Jenkins 2014-08-05 23:34:37 +00:00 committed by Gerrit Code Review
commit 737f351c38
5 changed files with 166 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

33
README.rst
View File

@ -269,21 +269,32 @@ There are 4 topics related to integration OSprofiler & `OpenStack`_:
* Put in python clients headers with trace info (if profiler is inited)
* Add `OSprofiler WSGI middleware`_ to service, that initializes
profiler, if there are special trace headers, that are signed by HMAC
from api-paste.ini
* Add `OSprofiler WSGI middleware`_ to your service, this initializes
the profiler, if and only if there are special trace headers, that
are signed by one of the HMAC keys from api-paste.ini (if multiple
keys exist the signing process will continue to use the key that was
accepted during validation).
* The common items that are used to configure the middleware are the
following (these can be provided when initializing the middleware
object or when setting up the api-paste.ini file)::
hmac_keys = KEY1, KEY2 (can be a single key as well)
Actually the algorithm is a bit more complex. The Python client will
also sign the trace info with a `HMAC`_ key passed to profiler.init,
and on reception the WSGI middleware will check that it's signed with
the **same** HMAC key that is specified in api-paste.ini. This ensures
that only the user that knows the HMAC key in api-paste.ini can init
a profiler properly and send trace info that will be actually
processed. This ensures that trace info that is sent in that does
**not** pass the HMAC validation will be discarded.
also sign the trace info with a `HMAC`_ key (lets call that key ``A``)
passed to profiler.init, and on reception the WSGI middleware will
check that it's signed with *one of* the HMAC keys (the wsgi
server should have key ``A`` as well, but may also have keys ``B``
and ``C``) that are specified in api-paste.ini. This ensures that only
the user that knows the HMAC key ``A`` in api-paste.ini can init a
profiler properly and send trace info that will be actually
processed. This ensures that trace info that is sent in that
does **not** pass the HMAC validation will be discarded. **NOTE:** The
application of many possible *validation* keys makes it possible to
roll out a key upgrade in a non-impactful manner (by adding a key into
the list and rolling out that change and then removing the older key at
some time in the future).
* RPC API

52
osprofiler/_utils.py
View File

@ -46,6 +46,21 @@ except (AttributeError, ImportError):
return result == 0
def split(text, strip=True):
"""Splits a comma separated text blob into its components.
Does nothing if already a list or tuple.
"""
if isinstance(text, (tuple, list)):
return text
if not isinstance(text, six.string_types):
raise TypeError("Unknown how to split '%s': %s" % (text, type(text)))
if strip:
return [t.strip() for t in text.split(",") if t.strip()]
else:
return text.split(",")
def binary_encode(text, encoding='utf-8'):
"""Converts a string of into a binary type using given encoding.
@ -74,7 +89,6 @@ def binary_decode(data, encoding='utf-8'):
def generate_hmac(data, hmac_key):
"""Generate a hmac using a known key given the provided content."""
h = hmac.new(binary_encode(hmac_key), digestmod=hashlib.sha1)
h.update(binary_encode(data))
return h.hexdigest()
@ -82,7 +96,6 @@ def generate_hmac(data, hmac_key):
def signed_pack(data, hmac_key):
"""Pack and sign data with hmac_key."""
raw_data = base64.urlsafe_b64encode(binary_encode(json.dumps(data)))
# NOTE(boris-42): Don't generate_hmac if there is no hmac_key, mostly
@ -92,30 +105,39 @@ def signed_pack(data, hmac_key):
return raw_data, generate_hmac(raw_data, hmac_key) if hmac_key else None
def signed_unpack(data, hmac_data, hmac_key):
def signed_unpack(data, hmac_data, hmac_keys):
"""Unpack data and check that it was signed with hmac_key.
:param data: json string that was singed_packed.
:param hmac_data: hmac data that was generated from json by hmac_key on
user side
:param hmac_key: server side hmac_key, that should be the same as user
:param hmac_keys: server side hmac_keys, one of these should be the same
as user used to sign with
:returns: None in case of something wrong, Object in case of everything OK.
"""
# NOTE(boris-42): For security reason, if there is no hmac_data or hmac_key
# we don't trust data => return None.
if not (hmac_key and hmac_data):
# NOTE(boris-42): For security reason, if there is no hmac_data or
# hmac_keys we don't trust data => return None.
if not (hmac_keys and hmac_data):
return None
hmac_data = hmac_data.strip()
try:
user_hmac_data = generate_hmac(data, hmac_key)
if not compare_digest(hmac_data, user_hmac_data):
return None
return json.loads(binary_decode(base64.urlsafe_b64decode(data)))
except Exception:
if not hmac_data:
return None
for hmac_key in hmac_keys:
try:
user_hmac_data = generate_hmac(data, hmac_key)
except Exception:
pass
else:
if compare_digest(hmac_data, user_hmac_data):
try:
contents = json.loads(
binary_decode(base64.urlsafe_b64decode(data)))
contents['hmac_key'] = hmac_key
return contents
except Exception:
return None
return None
def itersubclasses(cls, _seen=None):

34
osprofiler/web.py
View File

@ -13,12 +13,19 @@
# License for the specific language governing permissions and limitations
# under the License.
import six
import webob.dec
from osprofiler import _utils as utils
from osprofiler import profiler
# Trace keys that are required or optional, any other
# keys that are present will cause the trace to be rejected...
_REQUIRED_KEYS = ('base_id', 'hmac_key')
_OPTIONAL_KEYS = ('parent_id',)
def get_trace_id_headers():
"""Adds the trace id headers (and any hmac) into provided dictionary."""
p = profiler.get()
@ -54,19 +61,21 @@ def enable():
class WsgiMiddleware(object):
"""WSGI Middleware that enables tracing for an application."""
def __init__(self, application, hmac_key, enabled=False):
def __init__(self, application, hmac_keys, enabled=False):
"""Initialize middleware with api-paste.ini arguments.
:application: wsgi app
:hmac_key: Only trace header that was signed with this hmac key will be
processed. This limitation is essential, cause it allows
to profile OpenStack who knows this key => avoid DDOS.
:hmac_keys: Only trace header that was signed with one of these
hmac keys will be processed. This limitation is
essential, because it allows to profile OpenStack
by only those who knows this key which helps
avoid DDOS.
:enabled: This middleware can be turned off fully if enabled is False.
"""
self.application = application
self.name = "wsgi"
self.enabled = enabled
self.hmac_key = hmac_key
self.hmac_keys = utils.split(hmac_keys or "")
@classmethod
def factory(cls, global_conf, **local_conf):
@ -75,7 +84,14 @@ class WsgiMiddleware(object):
return filter_
def _trace_is_valid(self, trace_info):
return (isinstance(trace_info, dict) and "base_id" in trace_info)
if not isinstance(trace_info, dict):
return False
trace_keys = set(six.iterkeys(trace_info))
if not all(k in trace_keys for k in _REQUIRED_KEYS):
return False
if trace_keys.difference(_REQUIRED_KEYS + _OPTIONAL_KEYS):
return False
return True
@webob.dec.wsgify
def __call__(self, request):
@ -84,14 +100,12 @@ class WsgiMiddleware(object):
trace_info = utils.signed_unpack(request.headers.get("X-Trace-Info"),
request.headers.get("X-Trace-HMAC"),
self.hmac_key)
self.hmac_keys)
if not self._trace_is_valid(trace_info):
return request.get_response(self.application)
profiler.init(self.hmac_key,
base_id=trace_info.get("base_id"),
parent_id=trace_info.get("parent_id"))
profiler.init(**trace_info)
info = {
"request": {
"host_url": request.host_url,

22
tests/test_utils.py
View File

@ -55,8 +55,26 @@ class UtilsTestCase(test.TestCase):
packed_data, hmac_data = utils.signed_pack(data, hmac)
self.assertEqual(utils.signed_unpack(packed_data, hmac_data, hmac),
data)
process_data = utils.signed_unpack(packed_data, hmac_data, [hmac])
self.assertIn("hmac_key", process_data)
process_data.pop('hmac_key')
self.assertEqual(data, process_data)
def test_signed_pack_unpack_many_keys(self):
keys = ['secret', 'secret2', 'secret3']
data = {"some": "data"}
packed_data, hmac_data = utils.signed_pack(data, keys[-1])
process_data = utils.signed_unpack(packed_data, hmac_data, keys)
self.assertEqual(keys[-1], process_data['hmac_key'])
def test_signed_pack_unpack_many_wrong_keys(self):
keys = ['secret', 'secret2', 'secret3']
data = {"some": "data"}
packed_data, hmac_data = utils.signed_pack(data, 'password')
process_data = utils.signed_unpack(packed_data, hmac_data, keys)
self.assertIsNone(process_data)
def test_signed_unpack_wrong_key(self):
data = {"some": "data"}

67
tests/test_web.py
View File

@ -47,7 +47,9 @@ class WebTestCase(test.TestCase):
sorted(["X-Trace-Info", "X-Trace-HMAC"]))
trace_info = utils.signed_unpack(headers["X-Trace-Info"],
headers["X-Trace-HMAC"], "key")
headers["X-Trace-HMAC"], ["key"])
self.assertIn('hmac_key', trace_info)
self.assertEqual('key', trace_info.pop('hmac_key'))
self.assertEqual({"parent_id": 'z', 'base_id': 'y'}, trace_info)
@mock.patch("osprofiler.profiler.get")
@ -69,7 +71,7 @@ class WebMiddlewareTestCase(test.TestCase):
def test_factory(self):
mock_app = mock.MagicMock()
local_conf = {"enabled": True, "hmac_key": "123"}
local_conf = {"enabled": True, "hmac_keys": "123"}
factory = web.WsgiMiddleware.factory(None, **local_conf)
wsgi = factory(mock_app)
@ -77,7 +79,7 @@ class WebMiddlewareTestCase(test.TestCase):
self.assertEqual(wsgi.application, mock_app)
self.assertEqual(wsgi.name, "wsgi")
self.assertTrue(wsgi.enabled)
self.assertEqual(wsgi.hmac_key, local_conf["hmac_key"])
self.assertEqual(wsgi.hmac_keys, [local_conf["hmac_keys"]])
def _test_wsgi_middleware_with_invalid_trace(self, headers, hmac_key,
mock_profiler_init,
@ -165,6 +167,62 @@ class WebMiddlewareTestCase(test.TestCase):
self._test_wsgi_middleware_with_invalid_trace(headers, hmac_key,
mock_profiler_init)
@mock.patch("osprofiler.web.profiler.init")
def test_wsgi_middleware_key_passthrough(self, mock_profiler_init):
hmac_key = "secret2"
request = mock.MagicMock()
request.get_response.return_value = "yeah!"
request.url = "someurl"
request.host_url = "someurl"
request.path = "path"
request.query_string = "query"
request.method = "method"
request.scheme = "scheme"
pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
request.headers = {
"a": "1",
"b": "2",
"X-Trace-Info": pack[0],
"X-Trace-HMAC": pack[1]
}
middleware = web.WsgiMiddleware("app", "secret1,%s" % hmac_key,
enabled=True)
self.assertEqual("yeah!", middleware(request))
mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
base_id="1",
parent_id="2")
@mock.patch("osprofiler.web.profiler.init")
def test_wsgi_middleware_key_passthrough2(self, mock_profiler_init):
hmac_key = "secret1"
request = mock.MagicMock()
request.get_response.return_value = "yeah!"
request.url = "someurl"
request.host_url = "someurl"
request.path = "path"
request.query_string = "query"
request.method = "method"
request.scheme = "scheme"
pack = utils.signed_pack({"base_id": "1", "parent_id": "2"}, hmac_key)
request.headers = {
"a": "1",
"b": "2",
"X-Trace-Info": pack[0],
"X-Trace-HMAC": pack[1]
}
middleware = web.WsgiMiddleware("app", "%s,secret2" % hmac_key,
enabled=True)
self.assertEqual("yeah!", middleware(request))
mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
base_id="1",
parent_id="2")
@mock.patch("osprofiler.web.profiler.Trace")
@mock.patch("osprofiler.web.profiler.init")
def test_wsgi_middleware(self, mock_profiler_init, mock_profiler_trace):
@ -189,7 +247,8 @@ class WebMiddlewareTestCase(test.TestCase):
middleware = web.WsgiMiddleware("app", hmac_key, enabled=True)
self.assertEqual("yeah!", middleware(request))
mock_profiler_init.assert_called_once_with(hmac_key, base_id="1",
mock_profiler_init.assert_called_once_with(hmac_key=hmac_key,
base_id="1",
parent_id="2")
expected_info = {
"request": {