Fix dict and set order related issues in tests

There are several tests where the expected output
relies on order of strings generated from the arbitrary
ordering of dictionaries or sets. The new version of
tox causes that ordering to be different between tests
runs so tests must be resilient and test either the
content of the dicts or sets or their sorted output.

Note: These fixes are only of those cases where a failure
was generated. If there are issues in skipped tests, this
patch has not found nor fixed them. The vast majority of
skipped tests are related to availability of functionality
so there's little that can be done about that with the
current testing setup. The best we can do is be on the
lookout for failures that could be related PYTHONHASHSEED
down the road and fix them as they happen.

Closes-Bug: 1348818
Change-Id: I6b27ca2597c51b0656f441f325f9ffd0e31a606d

ceilometer/api/controllers/v2.py

@@ -411,7 +411,7 @@ def _validate_query(query, db_func, internal_keys=None,
                                                 '%s' % i.field)
             else:
                 msg = ("unrecognized field in query: %s, "
-                       "valid keys: %s") % (query, valid_keys)
+                       "valid keys: %s") % (query, sorted(valid_keys))
                 raise wsme.exc.UnknownArgument(key, msg)
 
 

ceilometer/tests/alarm/test_notifier.py

@@ -15,6 +15,7 @@
 # License for the specific language governing permissions and limitations
 # under the License.
 
+import anyjson
 import mock
 import requests
 import six.moves.urllib.parse as urlparse
@@ -26,9 +27,11 @@ from ceilometer.openstack.common.fixture import mockpatch
 from ceilometer.tests import base as tests_base
 
 
-DATA_JSON = ('{"current": "ALARM", "alarm_id": "foobar",'
+DATA_JSON = anyjson.loads(
-             ' "reason": "what ?", "reason_data": {"test": "test"},'
+    '{"current": "ALARM", "alarm_id": "foobar",'
-             ' "previous": "OK"}')
+    ' "reason": "what ?", "reason_data": {"test": "test"},'
+    ' "previous": "OK"}'
+)
 NOTIFICATION = dict(alarm_id='foobar',
                     condition=dict(threshold=42),
                     reason='what ?',
@@ -110,8 +113,11 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS)
+                                          headers=mock.ANY)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     def test_notify_alarm_rest_action_with_ssl_client_cert(self):
         action = 'https://host/action'
@@ -124,9 +130,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS,
+                                          headers=mock.ANY,
                                           cert=certificate, verify=True)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     def test_notify_alarm_rest_action_with_ssl_client_cert_and_key(self):
         action = 'https://host/action'
@@ -142,9 +151,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS,
+                                          headers=mock.ANY,
                                           cert=(certificate, key), verify=True)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     def test_notify_alarm_rest_action_with_ssl_verify_disable_by_cfg(self):
         action = 'https://host/action'
@@ -156,9 +168,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS,
+                                          headers=mock.ANY,
                                           verify=False)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     def test_notify_alarm_rest_action_with_ssl_verify_disable(self):
         action = 'https://host/action?ceilometer-alarm-ssl-verify=0'
@@ -167,9 +182,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS,
+                                          headers=mock.ANY,
                                           verify=False)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     def test_notify_alarm_rest_action_with_ssl_verify_enable_by_user(self):
         action = 'https://host/action?ceilometer-alarm-ssl-verify=1'
@@ -181,9 +199,12 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
             with mock.patch.object(requests.Session, 'post') as poster:
                 self.service.notify_alarm(context.get_admin_context(),
                                           self._notification(action))
-                poster.assert_called_with(action, data=DATA_JSON,
+                poster.assert_called_with(action, data=mock.ANY,
-                                          headers=self.HTTP_HEADERS,
+                                          headers=mock.ANY,
                                           verify=True)
+                args, kwargs = poster.call_args
+                self.assertEqual(self.HTTP_HEADERS, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))
 
     @staticmethod
     def _fake_urlsplit(*args, **kwargs):
@@ -234,4 +255,7 @@ class TestAlarmNotifier(tests_base.BaseTestCase):
                 headers = {'X-Auth-Token': 'token_1234'}
                 headers.update(self.HTTP_HEADERS)
                 poster.assert_called_with(
-                    url, data=DATA_JSON, headers=headers)
+                    url, data=mock.ANY, headers=mock.ANY)
+                args, kwargs = poster.call_args
+                self.assertEqual(headers, kwargs['headers'])
+                self.assertEqual(DATA_JSON, anyjson.loads(kwargs['data']))

ceilometer/tests/api/v2/test_alarm_scenarios.py

@@ -1909,11 +1909,11 @@ class TestAlarms(v2.FunctionalTest,
         resp = self._get_alarm_history(alarm, query=query,
                                        expect_errors=True, status=400)
         self.assertEqual('Unknown argument: "alarm_id": unrecognized'
-                         ' field in query: [<Query u\'alarm_id\' eq'
+                         " field in query: [<Query u'alarm_id' eq"
-                         ' u\'b\' Unset>], valid keys: set('
+                         " u'b' Unset>], valid keys: ['end_timestamp',"
-                         '[\'start_timestamp\', \'end_timestamp_op\','
+                         " 'end_timestamp_op', 'project',"
-                         ' \'project\', \'user\', \'start_timestamp_op\''
+                         " 'start_timestamp', 'start_timestamp_op',"
-                         ', \'type\', \'end_timestamp\'])',
+                         " 'type', 'user']",
                          resp.json['error_message']['faultstring'])
 
     def test_get_alarm_history_constrained_by_not_supported_rule(self):
@@ -1922,11 +1922,11 @@ class TestAlarms(v2.FunctionalTest,
         resp = self._get_alarm_history(alarm, query=query,
                                        expect_errors=True, status=400)
         self.assertEqual('Unknown argument: "abcd": unrecognized'
-                         ' field in query: [<Query u\'abcd\' eq'
+                         " field in query: [<Query u'abcd' eq"
-                         ' u\'abcd\' Unset>], valid keys: set('
+                         " u'abcd' Unset>], valid keys: ['end_timestamp',"
-                         '[\'start_timestamp\', \'end_timestamp_op\','
+                         " 'end_timestamp_op', 'project',"
-                         ' \'project\', \'user\', \'start_timestamp_op\''
+                         " 'start_timestamp', 'start_timestamp_op',"
-                         ', \'type\', \'end_timestamp\'])',
+                         " 'type', 'user']",
                          resp.json['error_message']['faultstring'])
 
     def test_get_nonexistent_alarm_history(self):

ceilometer/tests/test_utils.py

@@ -139,9 +139,10 @@ class TestUtils(test.BaseTestCase):
                 'nested2': [{'c': 'A'}, {'c': 'B'}]
                 }
         pairs = list(utils.dict_to_keyval(data))
-        self.assertEqual(set([('a', 'A'), ('b', 'B'),
+        self.assertEqual([('a', 'A'),
-                              ('nested2[0].c', 'A'),
+                          ('b', 'B'),
-                              ('nested2[1].c', 'B'),
+                         ('nested.a', 'A'),
-                              ('nested.a', 'A'),
+                         ('nested.b', 'B'),
-                              ('nested.b', 'B')]),
+                         ('nested2[0].c', 'A'),
-                         set(pairs))
+                         ('nested2[1].c', 'B')],
+                         sorted(pairs, key=lambda x: x[0]))

tox.ini

@@ -8,11 +8,8 @@ deps = -r{toxinidir}/requirements.txt
        -r{toxinidir}/test-requirements.txt
 install_command = pip install -U --allow-external pytidylib --allow-insecure pytidylib --allow-external netifaces --allow-insecure netifaces {opts} {packages}
 usedevelop = True
-# Note the hash seed is set to 0 until ceilometer can be tested with a
-# random hash seed successfully.
 setenv = VIRTUAL_ENV={envdir}
          EVENTLET_NO_GREENDNS=yes
-         PYTHONHASHSEED=0
 commands =
    bash -x {toxinidir}/setup-test-env.sh python setup.py testr --slowest --testr-args="{posargs}"
 downloadcache = {toxworkdir}/_download