openstack
/
trove
Code Issues Proposed changes

Preserve data type when parsing MySQL configs 

The data type information was lost in conversions.

IniCodec should deserialize Python objects (like other codecs).
guestagent_utils.to_bytes should return byte values as ints.

* The IniCodec is also used by Cassandra.
  Tested with both MySQL and Cassandra scenario tests.

Change-Id: Ibb703b3db6814fc0c9ea4c6d96399f6c881cea03
Closes-Bug: 1599656
This commit is contained in:
Petr Malik 2016-07-06 20:40:43 -04:00
parent 6e2922bc49
commit df509b7252
9 changed files with 94 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apidocs/src/samples/db-get-default-instance-configuration-response-json.txt
View File

@ -1,5 +1,5 @@
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 1124
Content-Length: 1094
Date: Mon, 18 Mar 2013 19:09:17 GMT

30
apidocs/src/samples/db-get-default-instance-configuration-response.json
View File

@ -2,42 +2,42 @@
"instance": {
"configuration": {
"basedir": "/usr",
"connect_timeout": "15",
"connect_timeout": 15,
"datadir": "/var/lib/mysql/data",
"default_storage_engine": "innodb",
"innodb_buffer_pool_size": "150M",
"innodb_data_file_path": "ibdata1:10M:autoextend",
"innodb_file_per_table": "1",
"innodb_file_per_table": 1,
"innodb_log_buffer_size": "25M",
"innodb_log_file_size": "50M",
"innodb_log_files_in_group": "2",
"innodb_log_files_in_group": 2,
"join_buffer_size": "1M",
"key_buffer_size": "50M",
"local-infile": "0",
"local-infile": 0,
"max_allowed_packet": "1024K",
"max_connections": "100",
"max_connections": 100,
"max_heap_table_size": "16M",
"max_user_connections": "100",
"max_user_connections": 100,
"myisam-recover-options": "BACKUP,FORCE",
"open_files_limit": "512",
"open_files_limit": 512,
"pid_file": "/var/run/mysqld/mysqld.pid",
"port": "3306",
"port": 3306,
"query_cache_limit": "1M",
"query_cache_size": "8M",
"query_cache_type": "1",
"query_cache_type": 1,
"read_buffer_size": "512K",
"read_rnd_buffer_size": "512K",
"server_id": "271898715",
"skip-external-locking": "1",
"server_id": 271898715,
"skip-external-locking": 1,
"sort_buffer_size": "1M",
"table_definition_cache": "256",
"table_open_cache": "256",
"thread_cache_size": "4",
"table_definition_cache": 256,
"table_open_cache": 256,
"thread_cache_size": 4,
"thread_stack": "192K",
"tmp_table_size": "16M",
"tmpdir": "/var/tmp",
"user": "mysql",
"wait_timeout": "120",
"wait_timeout": 120,
"performance_schema": "ON"
}
}

7
releasenotes/notes/mysql-config-preserve-types-77b970162bf6df08.yaml Normal file
View File

@ -0,0 +1,7 @@
---
fixes:
- Fix IniCodec to deserialize Python objects.
This also brings it in line with other codecs.
guestagent_utils.to_bytes return the byte values
as ints.
See bug 1599656

16
trove/common/stream_codecs.py
View File

@ -179,8 +179,8 @@ class IniCodec(StreamCodec):
...
The above file content would be represented as:
{'section_1': {'key': 'value', 'key': 'value', ...},
'section_2': {'key': 'value', 'key': 'value', ...}
{'section_1': {'key': value, 'key': value, ...},
'section_2': {'key': value, 'key': value, ...}
...
}
"""
@ -190,9 +190,8 @@ class IniCodec(StreamCodec):
:param default_value: Default value for keys with no value.
If set, all keys are written as 'key = value'.
The key is written without trailing '=' if None.
:type default_value: string
:type default_value: object
"""
self._value_converter = StringConverter({default_value: None})
self._default_value = default_value
self._comment_markers = comment_markers
@ -207,7 +206,8 @@ class IniCodec(StreamCodec):
parser = self._init_config_parser()
parser.readfp(self._pre_parse(stream))
return {s: {k: self._value_converter.to_strings(v)
return {s: {k:
StringConverter({None: self._default_value}).to_objects(v)
for k, v in parser.items(s, raw=True)}
for s in parser.sections()}
@ -231,8 +231,11 @@ class IniCodec(StreamCodec):
for section in sections:
parser.add_section(section)
for key, value in sections[section].items():
str_val = StringConverter(
{self._default_value: None}).to_strings(value)
parser.set(section, key,
self._value_converter.to_strings(value))
str(str_val) if str_val is not None
else str_val)
return parser
@ -383,6 +386,7 @@ class KeyValueCodec(PropertiesCodec):
...
}
"""
def __init__(self, delimiter='=', comment_markers=('#'),
unpack_singletons=True, string_mappings=None):
super(KeyValueCodec, self).__init__(

2
trove/guestagent/common/guestagent_utils.py
View File

@ -117,6 +117,6 @@ def to_bytes(value):
'G': 1024 ** 3,
}[suffix]
return str(int(round(factor * float(value))))
return int(round(factor * float(value)))
return value

5
trove/tests/unittests/configuration/test_configuration_controller.py
View File

@ -24,6 +24,7 @@ from trove.tests.unittests import trove_testtools
class TestConfigurationParser(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationParser, self).setUp()
@ -42,11 +43,12 @@ class TestConfigurationParser(trove_testtools.TestCase):
d_parsed = dict(parsed)
self.assertIsNotNone(d_parsed)
self.assertEqual("/var/run/mysqld/mysqld.pid", d_parsed["pid-file"])
self.assertEqual('15', d_parsed["connect_timeout"])
self.assertEqual(15, d_parsed["connect_timeout"])
self.assertEqual('1', d_parsed["skip-external-locking"])
class TestConfigurationController(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationController, self).setUp()
self.controller = ConfigurationsController()
@ -179,6 +181,7 @@ class TestConfigurationController(trove_testtools.TestCase):
class TestConfigurationsParameterController(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationsParameterController, self).setUp()
self.controller = service.ConfigurationsParameterController()

72
trove/tests/unittests/guestagent/test_configuration.py
View File

@ -122,27 +122,27 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
'value': '1.4142'}},
1,
{'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}}
'value': 1.4142}}
)
user_overrides_v2 = ('id2',
{'Section_1': {'is_number': 'False'}},
{'Section_1': {'is_number': False}},
2,
{'Section_1': {'is_number': 'False'}}
{'Section_1': {'is_number': False}}
)
system_overrides_v1 = ('id1',
{'Section_1': {'name': 'e',
'value': '2.7183'}},
'value': 2.7183}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'}}
'value': 2.7183}}
)
system_overrides_v2 = ('id2',
{'Section_2': {'is_number': 'True'}},
{'Section_2': {'is_number': True}},
2,
{'Section_2': {'is_number': 'True'}}
{'Section_2': {'is_number': True}}
)
self._test_import_override_strategy(
@ -153,36 +153,36 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# single file.
user_overrides_v1 = ('id1',
{'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}},
'value': 1.4142}},
1,
{'Section_1': {'name': 'sqrt(2)',
'is_number': 'False',
'value': '1.4142'}}
'is_number': False,
'value': 1.4142}}
)
user_overrides_v2 = ('id1',
{'Section_1': {'is_number': 'False'}},
{'Section_1': {'is_number': False}},
1,
{'Section_1': {'name': 'sqrt(2)',
'is_number': 'False',
'value': '1.4142'}}
'is_number': False,
'value': 1.4142}}
)
system_overrides_v1 = ('id1',
{'Section_1': {'name': 'e',
'value': '2.7183'}},
'value': 2.7183}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'},
'Section_2': {'is_number': 'True'}}
'value': 2.7183},
'Section_2': {'is_number': True}}
)
system_overrides_v2 = ('id1',
{'Section_2': {'is_number': 'True'}},
{'Section_2': {'is_number': True}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'},
'Section_2': {'is_number': 'True'}}
'value': 2.7183},
'Section_2': {'is_number': True}}
)
self._test_import_override_strategy(
@ -193,8 +193,8 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
def _test_import_override_strategy(
self, system_overrides, user_overrides, test_multi_rev):
base_config_contents = {'Section_1': {'name': 'pi',
'is_number': 'True',
'value': '3.1415'}
'is_number': True,
'value': 3.1415}
}
codec = IniCodec()
@ -361,21 +361,21 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
@patch.multiple(operating_system, chmod=Mock(), chown=Mock())
def _assert_get_value(self, override_strategy):
base_config_contents = {'Section_1': {'name': 'pi',
'is_number': 'True',
'value': '3.1415'}
'is_number': True,
'value': 3.1415}
}
config_overrides_v1a = {'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}
'value': 1.4142}
}
config_overrides_v2 = {'Section_1': {'name': 'e',
'value': '2.7183'},
'value': 2.7183},
'Section_2': {'foo': 'bar'}
}
config_overrides_v1b = {'Section_1': {'name': 'sqrt(4)',
'value': '2.0'}
'value': 2.0}
}
codec = IniCodec()
@ -397,22 +397,22 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# Test value before applying overrides.
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
# Test value after applying overrides.
manager.apply_user_override(config_overrides_v1a, change_id='id1')
self.assertEqual('sqrt(2)', manager.get_value('Section_1')['name'])
self.assertEqual('1.4142', manager.get_value('Section_1')['value'])
self.assertEqual(1.4142, manager.get_value('Section_1')['value'])
manager.apply_user_override(config_overrides_v2, change_id='id2')
self.assertEqual('e', manager.get_value('Section_1')['name'])
self.assertEqual('2.7183', manager.get_value('Section_1')['value'])
self.assertEqual(2.7183, manager.get_value('Section_1')['value'])
self.assertEqual('bar', manager.get_value('Section_2')['foo'])
# Editing change 'id1' become visible only after removing
# change 'id2', which overrides 'id1'.
manager.apply_user_override(config_overrides_v1b, change_id='id1')
self.assertEqual('e', manager.get_value('Section_1')['name'])
self.assertEqual('2.7183', manager.get_value('Section_1')['value'])
self.assertEqual(2.7183, manager.get_value('Section_1')['value'])
# Test value after removing overrides.
@ -420,35 +420,35 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# removing 'id2'.
manager.remove_user_override(change_id='id2')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# Back to the base.
manager.remove_user_override(change_id='id1')
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
self.assertIsNone(manager.get_value('Section_2'))
# Test system overrides.
manager.apply_system_override(
config_overrides_v1b, change_id='id1')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# The system values should take precedence over the user
# override.
manager.apply_user_override(
config_overrides_v1a, change_id='id1')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# The user values should become visible only after removing the
# system change.
manager.remove_system_override(change_id='id1')
self.assertEqual('sqrt(2)', manager.get_value('Section_1')['name'])
self.assertEqual('1.4142', manager.get_value('Section_1')['value'])
self.assertEqual(1.4142, manager.get_value('Section_1')['value'])
# Back to the base.
manager.remove_user_override(change_id='id1')
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
self.assertIsNone(manager.get_value('Section_2'))

6
trove/tests/unittests/guestagent/test_guestagent_utils.py
View File

@ -137,9 +137,9 @@ class TestGuestagentUtils(trove_testtools.TestCase):
def test_to_bytes(self):
self.assertEqual('1024', guestagent_utils.to_bytes('1024'))
self.assertEqual('1048576', guestagent_utils.to_bytes('1024K'))
self.assertEqual('1073741824', guestagent_utils.to_bytes('1024M'))
self.assertEqual('1099511627776', guestagent_utils.to_bytes('1024G'))
self.assertEqual(1048576, guestagent_utils.to_bytes('1024K'))
self.assertEqual(1073741824, guestagent_utils.to_bytes('1024M'))
self.assertEqual(1099511627776, guestagent_utils.to_bytes('1024G'))
self.assertEqual('1024T', guestagent_utils.to_bytes('1024T'))
self.assertEqual(1024, guestagent_utils.to_bytes(1024))
self.assertEqual('Hello!', guestagent_utils.to_bytes('Hello!'))

35
trove/tests/unittests/guestagent/test_operating_system.py
View File

@ -55,32 +55,23 @@ class TestOperatingSystem(trove_testtools.TestCase):
def test_ini_file_codec(self):
data_no_none = {"Section1": {"s1k1": 's1v1',
"s1k2": '3.1415926535'},
"Section2": {"s2k1": '1',
"s2k2": 'True'}}
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True}}
self._test_file_codec(data_no_none, IniCodec())
data_with_none = {"Section1": {"s1k1": 's1v1',
"s1k2": '3.1415926535'},
"Section2": {"s2k1": '1',
"s2k2": 'True',
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True,
"s2k3": None}}
# Keys with None values will be written without value.
self._test_file_codec(data_with_none, IniCodec())
# Non-string values will be converted to strings.
data_with_none_as_objects = {"Section1": {"s1k1": 's1v1',
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True,
"s2k3": None}}
self._test_file_codec(data_with_none_as_objects, IniCodec(),
expected_data=data_with_none)
# None will be replaced with 'default_value'.
default_value = '1'
default_value = 1
expected_data = guestagent_utils.update_dict(
{"Section2": {"s2k3": default_value}}, dict(data_with_none))
self._test_file_codec(data_with_none,
@ -92,7 +83,11 @@ class TestOperatingSystem(trove_testtools.TestCase):
"Section2": {"s2k1": '1',
"s2k2": 'True'},
"Section3": {"Section4": {"s4k1": '3.1415926535',
"s4k2": None}}
"s4k2": None}},
"Section5": {"s5k1": 1,
"s5k2": True},
"Section6": {"Section7": {"s7k1": 3.1415926535,
"s7k2": None}}
}
self._test_file_codec(data, YamlCodec())
@ -128,7 +123,11 @@ class TestOperatingSystem(trove_testtools.TestCase):
"Section2": {"s2k1": '1',
"s2k2": 'True'},
"Section3": {"Section4": {"s4k1": '3.1415926535',
"s4k2": None}}
"s4k2": None}},
"Section5": {"s5k1": 1,
"s5k2": True},
"Section6": {"Section7": {"s7k1": 3.1415926535,
"s7k2": None}}
}
self._test_file_codec(data, JsonCodec())