From 30aca66ff889684bb8ed6e61def1a4650a384123 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:05:49 -0800 Subject: [PATCH 1/4] Created mini XPath implementation, to simplify mapping logic --- nova/tests/test_minixpath.py | 141 +++++++++++++++++++++++++++++++++++ nova/utils.py | 45 +++++++++++ 2 files changed, 186 insertions(+) create mode 100644 nova/tests/test_minixpath.py diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py new file mode 100644 index 000000000..7fddcf9e9 --- /dev/null +++ b/nova/tests/test_minixpath.py @@ -0,0 +1,141 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2011 Justin Santa Barbara +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from nova import test +from nova import utils +from nova import exception + + +class MiniXPathTestCase(test.TestCase): + def test_tolerates_nones(self): + xp = utils.minixpath_select + + input = [] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [None] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': None}] + self.assertEquals([], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': None}}] + self.assertEquals([{'b': None}], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': None}] + self.assertEquals([{'b': {'c': None}}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] + self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) + self.assertEquals([{'c': None}], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + def test_does_select(self): + xp = utils.minixpath_select + + input = [{'a': 'a_1'}] + self.assertEquals(['a_1'], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': 'b_1'}}] + self.assertEquals([{'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] + self.assertEquals([{'b': {'c': 'c_1'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': None}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) + self.assertEquals(['c_1'], xp(input, "a/b/c")) + + input = [{'a': {'b': {'c': 'c_1'}}}, + {'a': {'b': {'c': 'c_2'}}}] + self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], + xp(input, "a")) + self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], + xp(input, "a/b")) + self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) + + self.assertEquals([], xp(input, "a/b/c/d")) + self.assertEquals([], xp(input, "c/a/b/d")) + self.assertEquals([], xp(input, "i/r/t")) + + def test_flattens_lists(self): + xp = utils.minixpath_select + + input = [{'a': [1, 2, 3]}] + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}] + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = [{'a': [1, 2, {'b': 'b_1'}]}] + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + def test_bad_xpath(self): + xp = utils.minixpath_select + + self.assertRaises(exception.Error, xp, [], None) + self.assertRaises(exception.Error, xp, [], "") + self.assertRaises(exception.Error, xp, [], "/") + self.assertRaises(exception.Error, xp, [], "/a") + self.assertRaises(exception.Error, xp, [], "/a/") + self.assertRaises(exception.Error, xp, [], "//") + self.assertRaises(exception.Error, xp, [], "//a") + self.assertRaises(exception.Error, xp, [], "a//a") + self.assertRaises(exception.Error, xp, [], "a//a/") + self.assertRaises(exception.Error, xp, [], "a/a/") diff --git a/nova/utils.py b/nova/utils.py index 42efa0008..2f926bd82 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -2,6 +2,7 @@ # Copyright 2010 United States Government as represented by the # Administrator of the National Aeronautics and Space Administration. +# Copyright 2011 Justin Santa Barbara # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -34,6 +35,7 @@ import time from xml.sax import saxutils import re import netaddr +import types from eventlet import event from eventlet import greenthread @@ -499,3 +501,46 @@ def ensure_b64_encoding(val): return val except TypeError: return base64.b64encode(val) + + +def minixpath_select(items, minixpath): + """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each + item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of + the intermediate results are lists it will treat each list item + individually. A 'None' in items or any child expressions will be ignored, + this function will not throw because of None (anywhere) in items""" + + if minixpath is None: + raise exception.Error("Invalid mini_xpath") + + (first_token, sep, remainder) = minixpath.partition("/") + + if first_token == "": + raise exception.Error("Invalid mini_xpath") + + results = [] + + if items is None: + return results + + for item in items: + if item is None: + continue + get_method = getattr(item, "get", None) + if get_method is None: + continue + child = get_method(first_token) + if child is None: + continue + if isinstance(child, types.ListType): + # Flatten intermediate lists + for x in child: + results.append(x) + else: + results.append(child) + + if not sep: + # No more tokens + return results + else: + return minixpath_select(results, remainder) From 7baacbe15998aa475203b0052f4b564d07ba214a Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:36:09 -0800 Subject: [PATCH 2/4] Cope when we pass a non-list to xpath_select - wrap it in a list --- nova/tests/test_minixpath.py | 38 ++++++++++++++++++++++++++++++++++++ nova/utils.py | 8 +++++++- 2 files changed, 45 insertions(+), 1 deletion(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 7fddcf9e9..3b1bdf40b 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -139,3 +139,41 @@ class MiniXPathTestCase(test.TestCase): self.assertRaises(exception.Error, xp, [], "a//a") self.assertRaises(exception.Error, xp, [], "a//a/") self.assertRaises(exception.Error, xp, [], "a/a/") + + def test_real_failure1(self): + # Real world failure case... + # We weren't coping when the input was a Dictionary instead of a List + # This led to test_accepts_dictionaries + xp = utils.minixpath_select + + inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], + 'address': '192.168.0.3'}, + 'hostname': ''} + + private_ips = xp(inst, 'fixed_ip/address') + public_ips = xp(inst, 'fixed_ip/floating_ips/address') + self.assertEquals(['192.168.0.3'], private_ips) + self.assertEquals(['1.2.3.4'], public_ips) + + def test_accepts_dictionaries(self): + xp = utils.minixpath_select + + input = {'a': [1, 2, 3]} + self.assertEquals([1, 2, 3], xp(input, "a")) + self.assertEquals([], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': {'b': [1, 2, 3]}} + self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) + self.assertEquals([1, 2, 3], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} + self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) + self.assertEquals([], xp(input, "a/b/c")) + + input = {'a': [1, 2, {'b': 'b_1'}]} + self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) + self.assertEquals(['b_1'], xp(input, "a/b")) + + diff --git a/nova/utils.py b/nova/utils.py index 2f926bd82..c2cbeb2a7 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -508,7 +508,8 @@ def minixpath_select(items, minixpath): item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of the intermediate results are lists it will treat each list item individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items""" + this function will not throw because of None (anywhere) in items. The + returned list will contain no None values.""" if minixpath is None: raise exception.Error("Invalid mini_xpath") @@ -523,6 +524,10 @@ def minixpath_select(items, minixpath): if items is None: return results + if not isinstance(items, types.ListType): + # Wrap single objects in a list + items = [items] + for item in items: if item is None: continue @@ -532,6 +537,7 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue + #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: From 890acb4c177cc3bdbdbad671b325990843f87640 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 12:55:07 -0800 Subject: [PATCH 3/4] Fix pep8 violation (trailing whitespace) --- nova/tests/test_minixpath.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py index 3b1bdf40b..cc4a35ef3 100644 --- a/nova/tests/test_minixpath.py +++ b/nova/tests/test_minixpath.py @@ -175,5 +175,3 @@ class MiniXPathTestCase(test.TestCase): input = {'a': [1, 2, {'b': 'b_1'}]} self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) self.assertEquals(['b_1'], xp(input, "a/b")) - - From 2d51a6048beb6d7add9451169492d7c2bbdbfca8 Mon Sep 17 00:00:00 2001 From: Justin Santa Barbara Date: Wed, 23 Feb 2011 14:07:08 -0800 Subject: [PATCH 4/4] Rename minixpath_select to get_from_path --- nova/tests/test_minixpath.py | 177 ----------------------------------- nova/utils.py | 24 ++--- 2 files changed, 12 insertions(+), 189 deletions(-) delete mode 100644 nova/tests/test_minixpath.py diff --git a/nova/tests/test_minixpath.py b/nova/tests/test_minixpath.py deleted file mode 100644 index cc4a35ef3..000000000 --- a/nova/tests/test_minixpath.py +++ /dev/null @@ -1,177 +0,0 @@ -# vim: tabstop=4 shiftwidth=4 softtabstop=4 - -# Copyright 2011 Justin Santa Barbara -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from nova import test -from nova import utils -from nova import exception - - -class MiniXPathTestCase(test.TestCase): - def test_tolerates_nones(self): - xp = utils.minixpath_select - - input = [] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [None] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': None}] - self.assertEquals([], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': None}}] - self.assertEquals([{'b': None}], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': None}] - self.assertEquals([{'b': {'c': None}}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': None}}}, {'a': {'b': None}}] - self.assertEquals([{'b': {'c': None}}, {'b': None}], xp(input, "a")) - self.assertEquals([{'c': None}], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - def test_does_select(self): - xp = utils.minixpath_select - - input = [{'a': 'a_1'}] - self.assertEquals(['a_1'], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': 'b_1'}}] - self.assertEquals([{'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}], xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, {'a': None}] - self.assertEquals([{'b': {'c': 'c_1'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': None}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': None}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}], xp(input, "a/b")) - self.assertEquals(['c_1'], xp(input, "a/b/c")) - - input = [{'a': {'b': {'c': 'c_1'}}}, - {'a': {'b': {'c': 'c_2'}}}] - self.assertEquals([{'b': {'c': 'c_1'}}, {'b': {'c': 'c_2'}}], - xp(input, "a")) - self.assertEquals([{'c': 'c_1'}, {'c': 'c_2'}], - xp(input, "a/b")) - self.assertEquals(['c_1', 'c_2'], xp(input, "a/b/c")) - - self.assertEquals([], xp(input, "a/b/c/d")) - self.assertEquals([], xp(input, "c/a/b/d")) - self.assertEquals([], xp(input, "i/r/t")) - - def test_flattens_lists(self): - xp = utils.minixpath_select - - input = [{'a': [1, 2, 3]}] - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}] - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': {'b': [1, 2, 3]}}, {'a': {'b': [4, 5, 6]}}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]}] - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = [{'a': [1, 2, {'b': 'b_1'}]}] - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) - - def test_bad_xpath(self): - xp = utils.minixpath_select - - self.assertRaises(exception.Error, xp, [], None) - self.assertRaises(exception.Error, xp, [], "") - self.assertRaises(exception.Error, xp, [], "/") - self.assertRaises(exception.Error, xp, [], "/a") - self.assertRaises(exception.Error, xp, [], "/a/") - self.assertRaises(exception.Error, xp, [], "//") - self.assertRaises(exception.Error, xp, [], "//a") - self.assertRaises(exception.Error, xp, [], "a//a") - self.assertRaises(exception.Error, xp, [], "a//a/") - self.assertRaises(exception.Error, xp, [], "a/a/") - - def test_real_failure1(self): - # Real world failure case... - # We weren't coping when the input was a Dictionary instead of a List - # This led to test_accepts_dictionaries - xp = utils.minixpath_select - - inst = {'fixed_ip': {'floating_ips': [{'address': '1.2.3.4'}], - 'address': '192.168.0.3'}, - 'hostname': ''} - - private_ips = xp(inst, 'fixed_ip/address') - public_ips = xp(inst, 'fixed_ip/floating_ips/address') - self.assertEquals(['192.168.0.3'], private_ips) - self.assertEquals(['1.2.3.4'], public_ips) - - def test_accepts_dictionaries(self): - xp = utils.minixpath_select - - input = {'a': [1, 2, 3]} - self.assertEquals([1, 2, 3], xp(input, "a")) - self.assertEquals([], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': {'b': [1, 2, 3]}} - self.assertEquals([{'b': [1, 2, 3]}], xp(input, "a")) - self.assertEquals([1, 2, 3], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [{'b': [1, 2, 3]}, {'b': [4, 5, 6]}]} - self.assertEquals([1, 2, 3, 4, 5, 6], xp(input, "a/b")) - self.assertEquals([], xp(input, "a/b/c")) - - input = {'a': [1, 2, {'b': 'b_1'}]} - self.assertEquals([1, 2, {'b': 'b_1'}], xp(input, "a")) - self.assertEquals(['b_1'], xp(input, "a/b")) diff --git a/nova/utils.py b/nova/utils.py index c2cbeb2a7..65e28c648 100644 --- a/nova/utils.py +++ b/nova/utils.py @@ -32,10 +32,10 @@ import string import struct import sys import time +import types from xml.sax import saxutils import re import netaddr -import types from eventlet import event from eventlet import greenthread @@ -503,18 +503,19 @@ def ensure_b64_encoding(val): return base64.b64encode(val) -def minixpath_select(items, minixpath): - """ Takes an xpath-like expression e.g. prop1/prop2/prop3, and for each - item in items, looks up items[prop1][prop2][prop3]. Like XPath, if any of - the intermediate results are lists it will treat each list item - individually. A 'None' in items or any child expressions will be ignored, - this function will not throw because of None (anywhere) in items. The - returned list will contain no None values.""" +def get_from_path(items, path): + """ Returns a list of items matching the specified path. Takes an + XPath-like expression e.g. prop1/prop2/prop3, and for each item in items, + looks up items[prop1][prop2][prop3]. Like XPath, if any of the + intermediate results are lists it will treat each list item individually. + A 'None' in items or any child expressions will be ignored, this function + will not throw because of None (anywhere) in items. The returned list + will contain no None values.""" - if minixpath is None: + if path is None: raise exception.Error("Invalid mini_xpath") - (first_token, sep, remainder) = minixpath.partition("/") + (first_token, sep, remainder) = path.partition("/") if first_token == "": raise exception.Error("Invalid mini_xpath") @@ -537,7 +538,6 @@ def minixpath_select(items, minixpath): child = get_method(first_token) if child is None: continue - #print "%s => %s" % (first_token, child) if isinstance(child, types.ListType): # Flatten intermediate lists for x in child: @@ -549,4 +549,4 @@ def minixpath_select(items, minixpath): # No more tokens return results else: - return minixpath_select(results, remainder) + return get_from_path(results, remainder)