openstack/trove
Code Issues Proposed changes

Preserve data type when parsing MySQL configs

Browse Source 
The data type information was lost in conversions.

IniCodec should deserialize Python objects (like other codecs).
guestagent_utils.to_bytes should return byte values as ints.

* The IniCodec is also used by Cassandra.
  Tested with both MySQL and Cassandra scenario tests.

Change-Id: Ibb703b3db6814fc0c9ea4c6d96399f6c881cea03
Closes-Bug: 1599656
This commit is contained in:
Petr Malik 2016-07-06 20:40:43 -04:00
parent 6e2922bc49
commit df509b7252
9 changed files with 94 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apidocs/src/samples/db-get-default-instance-configuration-response-json.txt
View File

@ -1,5 +1,5 @@
HTTP/1.1 200 OK
Content-Type: application/json
Content-Length: 1124
Content-Length: 1094
Date: Mon, 18 Mar 2013 19:09:17 GMT

30
apidocs/src/samples/db-get-default-instance-configuration-response.json
View File

@ -2,42 +2,42 @@
"instance": {
"configuration": {
"basedir": "/usr",
"connect_timeout": "15",
"connect_timeout": 15,
"datadir": "/var/lib/mysql/data",
"default_storage_engine": "innodb",
"innodb_buffer_pool_size": "150M",
"innodb_data_file_path": "ibdata1:10M:autoextend",
"innodb_file_per_table": "1",
"innodb_file_per_table": 1,
"innodb_log_buffer_size": "25M",
"innodb_log_file_size": "50M",
"innodb_log_files_in_group": "2",
"innodb_log_files_in_group": 2,
"join_buffer_size": "1M",
"key_buffer_size": "50M",
"local-infile": "0",
"local-infile": 0,
"max_allowed_packet": "1024K",
"max_connections": "100",
"max_connections": 100,
"max_heap_table_size": "16M",
"max_user_connections": "100",
"max_user_connections": 100,
"myisam-recover-options": "BACKUP,FORCE",
"open_files_limit": "512",
"open_files_limit": 512,
"pid_file": "/var/run/mysqld/mysqld.pid",
"port": "3306",
"port": 3306,
"query_cache_limit": "1M",
"query_cache_size": "8M",
"query_cache_type": "1",
"query_cache_type": 1,
"read_buffer_size": "512K",
"read_rnd_buffer_size": "512K",
"server_id": "271898715",
"skip-external-locking": "1",
"server_id": 271898715,
"skip-external-locking": 1,
"sort_buffer_size": "1M",
"table_definition_cache": "256",
"table_open_cache": "256",
"thread_cache_size": "4",
"table_definition_cache": 256,
"table_open_cache": 256,
"thread_cache_size": 4,
"thread_stack": "192K",
"tmp_table_size": "16M",
"tmpdir": "/var/tmp",
"user": "mysql",
"wait_timeout": "120",
"wait_timeout": 120,
"performance_schema": "ON"
}
}

7
releasenotes/notes/mysql-config-preserve-types-77b970162bf6df08.yaml Normal file
View File

@ -0,0 +1,7 @@
---
fixes:
- Fix IniCodec to deserialize Python objects.
This also brings it in line with other codecs.
guestagent_utils.to_bytes return the byte values
as ints.
See bug 1599656

16
trove/common/stream_codecs.py
View File

@ -179,8 +179,8 @@ class IniCodec(StreamCodec):
...
The above file content would be represented as:
{'section_1': {'key': 'value', 'key': 'value', ...},
'section_2': {'key': 'value', 'key': 'value', ...}
{'section_1': {'key': value, 'key': value, ...},
'section_2': {'key': value, 'key': value, ...}
...
}
"""
@ -190,9 +190,8 @@ class IniCodec(StreamCodec):
:param default_value: Default value for keys with no value.
If set, all keys are written as 'key = value'.
The key is written without trailing '=' if None.
:type default_value: string
:type default_value: object
"""
self._value_converter = StringConverter({default_value: None})
self._default_value = default_value
self._comment_markers = comment_markers
@ -207,7 +206,8 @@ class IniCodec(StreamCodec):
parser = self._init_config_parser()
parser.readfp(self._pre_parse(stream))
return {s: {k: self._value_converter.to_strings(v)
return {s: {k:
StringConverter({None: self._default_value}).to_objects(v)
for k, v in parser.items(s, raw=True)}
for s in parser.sections()}
@ -231,8 +231,11 @@ class IniCodec(StreamCodec):
for section in sections:
parser.add_section(section)
for key, value in sections[section].items():
str_val = StringConverter(
{self._default_value: None}).to_strings(value)
parser.set(section, key,
self._value_converter.to_strings(value))
str(str_val) if str_val is not None
else str_val)
return parser
@ -383,6 +386,7 @@ class KeyValueCodec(PropertiesCodec):
...
}
"""
def __init__(self, delimiter='=', comment_markers=('#'),
unpack_singletons=True, string_mappings=None):
super(KeyValueCodec, self).__init__(

2
trove/guestagent/common/guestagent_utils.py
View File

@ -117,6 +117,6 @@ def to_bytes(value):
'G': 1024 ** 3,
}[suffix]
return str(int(round(factor * float(value))))
return int(round(factor * float(value)))
return value

5
trove/tests/unittests/configuration/test_configuration_controller.py
View File

@ -24,6 +24,7 @@ from trove.tests.unittests import trove_testtools
class TestConfigurationParser(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationParser, self).setUp()
@ -42,11 +43,12 @@ class TestConfigurationParser(trove_testtools.TestCase):
d_parsed = dict(parsed)
self.assertIsNotNone(d_parsed)
self.assertEqual("/var/run/mysqld/mysqld.pid", d_parsed["pid-file"])
self.assertEqual('15', d_parsed["connect_timeout"])
self.assertEqual(15, d_parsed["connect_timeout"])
self.assertEqual('1', d_parsed["skip-external-locking"])
class TestConfigurationController(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationController, self).setUp()
self.controller = ConfigurationsController()
@ -179,6 +181,7 @@ class TestConfigurationController(trove_testtools.TestCase):
class TestConfigurationsParameterController(trove_testtools.TestCase):
def setUp(self):
super(TestConfigurationsParameterController, self).setUp()
self.controller = service.ConfigurationsParameterController()

72
trove/tests/unittests/guestagent/test_configuration.py
View File

@ -122,27 +122,27 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
'value': '1.4142'}},
1,
{'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}}
'value': 1.4142}}
)
user_overrides_v2 = ('id2',
{'Section_1': {'is_number': 'False'}},
{'Section_1': {'is_number': False}},
2,
{'Section_1': {'is_number': 'False'}}
{'Section_1': {'is_number': False}}
)
system_overrides_v1 = ('id1',
{'Section_1': {'name': 'e',
'value': '2.7183'}},
'value': 2.7183}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'}}
'value': 2.7183}}
)
system_overrides_v2 = ('id2',
{'Section_2': {'is_number': 'True'}},
{'Section_2': {'is_number': True}},
2,
{'Section_2': {'is_number': 'True'}}
{'Section_2': {'is_number': True}}
)
self._test_import_override_strategy(
@ -153,36 +153,36 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# single file.
user_overrides_v1 = ('id1',
{'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}},
'value': 1.4142}},
1,
{'Section_1': {'name': 'sqrt(2)',
'is_number': 'False',
'value': '1.4142'}}
'is_number': False,
'value': 1.4142}}
)
user_overrides_v2 = ('id1',
{'Section_1': {'is_number': 'False'}},
{'Section_1': {'is_number': False}},
1,
{'Section_1': {'name': 'sqrt(2)',
'is_number': 'False',
'value': '1.4142'}}
'is_number': False,
'value': 1.4142}}
)
system_overrides_v1 = ('id1',
{'Section_1': {'name': 'e',
'value': '2.7183'}},
'value': 2.7183}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'},
'Section_2': {'is_number': 'True'}}
'value': 2.7183},
'Section_2': {'is_number': True}}
)
system_overrides_v2 = ('id1',
{'Section_2': {'is_number': 'True'}},
{'Section_2': {'is_number': True}},
1,
{'Section_1': {'name': 'e',
'value': '2.7183'},
'Section_2': {'is_number': 'True'}}
'value': 2.7183},
'Section_2': {'is_number': True}}
)
self._test_import_override_strategy(
@ -193,8 +193,8 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
def _test_import_override_strategy(
self, system_overrides, user_overrides, test_multi_rev):
base_config_contents = {'Section_1': {'name': 'pi',
'is_number': 'True',
'value': '3.1415'}
'is_number': True,
'value': 3.1415}
}
codec = IniCodec()
@ -361,21 +361,21 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
@patch.multiple(operating_system, chmod=Mock(), chown=Mock())
def _assert_get_value(self, override_strategy):
base_config_contents = {'Section_1': {'name': 'pi',
'is_number': 'True',
'value': '3.1415'}
'is_number': True,
'value': 3.1415}
}
config_overrides_v1a = {'Section_1': {'name': 'sqrt(2)',
'value': '1.4142'}
'value': 1.4142}
}
config_overrides_v2 = {'Section_1': {'name': 'e',
'value': '2.7183'},
'value': 2.7183},
'Section_2': {'foo': 'bar'}
}
config_overrides_v1b = {'Section_1': {'name': 'sqrt(4)',
'value': '2.0'}
'value': 2.0}
}
codec = IniCodec()
@ -397,22 +397,22 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# Test value before applying overrides.
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
# Test value after applying overrides.
manager.apply_user_override(config_overrides_v1a, change_id='id1')
self.assertEqual('sqrt(2)', manager.get_value('Section_1')['name'])
self.assertEqual('1.4142', manager.get_value('Section_1')['value'])
self.assertEqual(1.4142, manager.get_value('Section_1')['value'])
manager.apply_user_override(config_overrides_v2, change_id='id2')
self.assertEqual('e', manager.get_value('Section_1')['name'])
self.assertEqual('2.7183', manager.get_value('Section_1')['value'])
self.assertEqual(2.7183, manager.get_value('Section_1')['value'])
self.assertEqual('bar', manager.get_value('Section_2')['foo'])
# Editing change 'id1' become visible only after removing
# change 'id2', which overrides 'id1'.
manager.apply_user_override(config_overrides_v1b, change_id='id1')
self.assertEqual('e', manager.get_value('Section_1')['name'])
self.assertEqual('2.7183', manager.get_value('Section_1')['value'])
self.assertEqual(2.7183, manager.get_value('Section_1')['value'])
# Test value after removing overrides.
@ -420,35 +420,35 @@ class TestConfigurationOverrideStrategy(trove_testtools.TestCase):
# removing 'id2'.
manager.remove_user_override(change_id='id2')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# Back to the base.
manager.remove_user_override(change_id='id1')
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
self.assertIsNone(manager.get_value('Section_2'))
# Test system overrides.
manager.apply_system_override(
config_overrides_v1b, change_id='id1')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# The system values should take precedence over the user
# override.
manager.apply_user_override(
config_overrides_v1a, change_id='id1')
self.assertEqual('sqrt(4)', manager.get_value('Section_1')['name'])
self.assertEqual('2.0', manager.get_value('Section_1')['value'])
self.assertEqual(2.0, manager.get_value('Section_1')['value'])
# The user values should become visible only after removing the
# system change.
manager.remove_system_override(change_id='id1')
self.assertEqual('sqrt(2)', manager.get_value('Section_1')['name'])
self.assertEqual('1.4142', manager.get_value('Section_1')['value'])
self.assertEqual(1.4142, manager.get_value('Section_1')['value'])
# Back to the base.
manager.remove_user_override(change_id='id1')
self.assertEqual('pi', manager.get_value('Section_1')['name'])
self.assertEqual('3.1415', manager.get_value('Section_1')['value'])
self.assertEqual(3.1415, manager.get_value('Section_1')['value'])
self.assertIsNone(manager.get_value('Section_2'))

6
trove/tests/unittests/guestagent/test_guestagent_utils.py
View File

@ -137,9 +137,9 @@ class TestGuestagentUtils(trove_testtools.TestCase):
def test_to_bytes(self):
self.assertEqual('1024', guestagent_utils.to_bytes('1024'))
self.assertEqual('1048576', guestagent_utils.to_bytes('1024K'))
self.assertEqual('1073741824', guestagent_utils.to_bytes('1024M'))
self.assertEqual('1099511627776', guestagent_utils.to_bytes('1024G'))
self.assertEqual(1048576, guestagent_utils.to_bytes('1024K'))
self.assertEqual(1073741824, guestagent_utils.to_bytes('1024M'))
self.assertEqual(1099511627776, guestagent_utils.to_bytes('1024G'))
self.assertEqual('1024T', guestagent_utils.to_bytes('1024T'))
self.assertEqual(1024, guestagent_utils.to_bytes(1024))
self.assertEqual('Hello!', guestagent_utils.to_bytes('Hello!'))

35
trove/tests/unittests/guestagent/test_operating_system.py
View File

@ -55,32 +55,23 @@ class TestOperatingSystem(trove_testtools.TestCase):
def test_ini_file_codec(self):
data_no_none = {"Section1": {"s1k1": 's1v1',
"s1k2": '3.1415926535'},
"Section2": {"s2k1": '1',
"s2k2": 'True'}}
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True}}
self._test_file_codec(data_no_none, IniCodec())
data_with_none = {"Section1": {"s1k1": 's1v1',
"s1k2": '3.1415926535'},
"Section2": {"s2k1": '1',
"s2k2": 'True',
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True,
"s2k3": None}}
# Keys with None values will be written without value.
self._test_file_codec(data_with_none, IniCodec())
# Non-string values will be converted to strings.
data_with_none_as_objects = {"Section1": {"s1k1": 's1v1',
"s1k2": 3.1415926535},
"Section2": {"s2k1": 1,
"s2k2": True,
"s2k3": None}}
self._test_file_codec(data_with_none_as_objects, IniCodec(),
expected_data=data_with_none)
# None will be replaced with 'default_value'.
default_value = '1'
default_value = 1
expected_data = guestagent_utils.update_dict(
{"Section2": {"s2k3": default_value}}, dict(data_with_none))
self._test_file_codec(data_with_none,
@ -92,7 +83,11 @@ class TestOperatingSystem(trove_testtools.TestCase):
"Section2": {"s2k1": '1',
"s2k2": 'True'},
"Section3": {"Section4": {"s4k1": '3.1415926535',
"s4k2": None}}
"s4k2": None}},
"Section5": {"s5k1": 1,
"s5k2": True},
"Section6": {"Section7": {"s7k1": 3.1415926535,
"s7k2": None}}
}
self._test_file_codec(data, YamlCodec())
@ -128,7 +123,11 @@ class TestOperatingSystem(trove_testtools.TestCase):
"Section2": {"s2k1": '1',
"s2k2": 'True'},
"Section3": {"Section4": {"s4k1": '3.1415926535',
"s4k2": None}}
"s4k2": None}},
"Section5": {"s5k1": 1,
"s5k2": True},
"Section6": {"Section7": {"s7k1": 3.1415926535,
"s7k2": None}}
}
self._test_file_codec(data, JsonCodec())