diff --git a/tests/data/merge_sources/expected1.yaml b/tests/data/merge_sources/expected1.yaml new file mode 100644 index 00000000..640d282b --- /dev/null +++ b/tests/data/merge_sources/expected1.yaml @@ -0,0 +1 @@ +Blah: ['blah2', 'b'] diff --git a/tests/data/merge_sources/expected2.yaml b/tests/data/merge_sources/expected2.yaml new file mode 100644 index 00000000..6eccc2cf --- /dev/null +++ b/tests/data/merge_sources/expected2.yaml @@ -0,0 +1,3 @@ +Blah: 3 +Blah2: 2 +Blah3: [1] diff --git a/tests/data/merge_sources/expected3.yaml b/tests/data/merge_sources/expected3.yaml new file mode 100644 index 00000000..32d9ad48 --- /dev/null +++ b/tests/data/merge_sources/expected3.yaml @@ -0,0 +1 @@ +Blah: [blah2, 'blah1'] diff --git a/tests/data/merge_sources/expected4.yaml b/tests/data/merge_sources/expected4.yaml new file mode 100644 index 00000000..d88d8f73 --- /dev/null +++ b/tests/data/merge_sources/expected4.yaml @@ -0,0 +1,2 @@ +#cloud-config +Blah: {} diff --git a/tests/data/merge_sources/source1-1.yaml b/tests/data/merge_sources/source1-1.yaml new file mode 100644 index 00000000..38e4e5e0 --- /dev/null +++ b/tests/data/merge_sources/source1-1.yaml @@ -0,0 +1,3 @@ +#cloud-config +Blah: ['blah2'] + diff --git a/tests/data/merge_sources/source1-2.yaml b/tests/data/merge_sources/source1-2.yaml new file mode 100644 index 00000000..2cd0e0e5 --- /dev/null +++ b/tests/data/merge_sources/source1-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +Blah: ['b'] + +merge_how: 'dict(recurse_array,no_replace)+list(append)' diff --git a/tests/data/merge_sources/source2-1.yaml b/tests/data/merge_sources/source2-1.yaml new file mode 100644 index 00000000..c7a33aaa --- /dev/null +++ b/tests/data/merge_sources/source2-1.yaml @@ -0,0 +1,6 @@ +#cloud-config + + +Blah: 1 +Blah2: 2 +Blah3: 3 diff --git a/tests/data/merge_sources/source2-2.yaml b/tests/data/merge_sources/source2-2.yaml new file mode 100644 index 00000000..8f2fdc1a --- /dev/null +++ b/tests/data/merge_sources/source2-2.yaml @@ -0,0 +1,5 @@ +#cloud-config + +Blah: 3 +Blah2: 2 +Blah3: [1] diff --git a/tests/data/merge_sources/source3-1.yaml b/tests/data/merge_sources/source3-1.yaml new file mode 100644 index 00000000..2303e906 --- /dev/null +++ b/tests/data/merge_sources/source3-1.yaml @@ -0,0 +1,4 @@ +#cloud-config +Blah: ['blah1'] + + diff --git a/tests/data/merge_sources/source3-2.yaml b/tests/data/merge_sources/source3-2.yaml new file mode 100644 index 00000000..dca2ad10 --- /dev/null +++ b/tests/data/merge_sources/source3-2.yaml @@ -0,0 +1,4 @@ +#cloud-config +Blah: ['blah2'] + +merge_how: 'dict(recurse_array,no_replace)+list(prepend)' diff --git a/tests/data/merge_sources/source4-1.yaml b/tests/data/merge_sources/source4-1.yaml new file mode 100644 index 00000000..e5b16872 --- /dev/null +++ b/tests/data/merge_sources/source4-1.yaml @@ -0,0 +1,3 @@ +#cloud-config +Blah: + b: 1 diff --git a/tests/data/merge_sources/source4-2.yaml b/tests/data/merge_sources/source4-2.yaml new file mode 100644 index 00000000..1844e0f8 --- /dev/null +++ b/tests/data/merge_sources/source4-2.yaml @@ -0,0 +1,6 @@ +#cloud-config +Blah: + b: null + + +merge_how: 'dict(allow_delete,no_replace)+list()' diff --git a/tests/unittests/test_merging.py b/tests/unittests/test_merging.py index ad137e85..470b18c7 100644 --- a/tests/unittests/test_merging.py +++ b/tests/unittests/test_merging.py @@ -1,142 +1,56 @@ from tests.unittests import helpers -from cloudinit import mergers +from cloudinit.handlers import cloud_config +from cloudinit.handlers import (CONTENT_START, CONTENT_END) + +from cloudinit import helpers as c_helpers +from cloudinit import util + +import collections +import glob +import os +import re -class TestSimpleRun(helpers.MockerTestCase): - def test_basic_merge(self): - source = { - 'Blah': ['blah2'], - 'Blah3': 'c', - } - merge_with = { - 'Blah2': ['blah3'], - 'Blah3': 'b', - 'Blah': ['123'], - } - # Basic merge should not do thing special - merge_how = "list()+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], ['blah2']) - self.assertEquals(merged['Blah2'], ['blah3']) - self.assertEquals(merged['Blah3'], 'c') +class TestSimpleRun(helpers.ResourceUsingTestCase): + def _load_merge_files(self, data_dir): + merge_root = self.resourceLocation(data_dir) + tests = [] + source_ids = collections.defaultdict(list) + expected_files = {} + for fn in glob.glob(os.path.join(merge_root, "source*.*yaml")): + base_fn = os.path.basename(fn) + file_id = re.match(r"source(\d+)\-(\d+)[.]yaml", base_fn) + if not file_id: + raise IOError("File %s does not have a numeric identifier" + % (fn)) + file_id = int(file_id.group(1)) + source_ids[file_id].append(fn) + expected_fn = os.path.join(merge_root, + "expected%s.yaml" % (file_id)) + if not os.path.isfile(expected_fn): + raise IOError("No expected file found at %s" % (expected_fn)) + expected_files[file_id] = expected_fn + for id in sorted(source_ids.keys()): + source_file_contents = [] + for fn in sorted(source_ids[id]): + source_file_contents.append(util.load_file(fn)) + expected = util.load_yaml(util.load_file(expected_files[id])) + tests.append((source_file_contents, expected)) + return tests - def test_dict_overwrite(self): - source = { - 'Blah': ['blah2'], - } - merge_with = { - 'Blah': ['123'], - } - # Now lets try a dict overwrite - merge_how = "list()+dict(overwrite)+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], ['123']) - - def test_string_append(self): - source = { - 'Blah': 'blah2', - } - merge_with = { - 'Blah': '345', - } - merge_how = "list()+dict()+str(append)" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah'], 'blah2345') - - def test_list_extend(self): - source = ['abc'] - merge_with = ['123'] - merge_how = "list(extend)+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged, ['abc', '123']) - - def test_deep_merge(self): - source = { - 'a': [1, 'b', 2], - 'b': 'blahblah', - 'c': { - 'e': [1, 2, 3], - 'f': 'bigblobof', - 'iamadict': { - 'ok': 'ok', - } - }, - 'run': [ - 'runme', - 'runme2', - ], - 'runmereally': [ - 'e', ['a'], 'd', - ], - } - merge_with = { - 'a': ['e', 'f', 'g'], - 'b': 'more', - 'c': { - 'a': 'b', - 'f': 'stuff', - }, - 'run': [ - 'morecmd', - 'moremoremore', - ], - 'runmereally': [ - 'blah', ['b'], 'e', - ], - } - merge_how = "list(extend)+dict()+str(append)" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['a'], [1, 'b', 2, 'e', 'f', 'g']) - self.assertEquals(merged['b'], 'blahblahmore') - self.assertEquals(merged['c']['f'], 'bigblobofstuff') - self.assertEquals(merged['run'], ['runme', 'runme2', 'morecmd', - 'moremoremore']) - self.assertEquals(merged['runmereally'], ['e', ['a'], 'd', 'blah', - ['b'], 'e']) - - def test_dict_overwrite_layered(self): - source = { - 'Blah3': { - 'f': '3', - 'g': { - 'a': 'b', - } - } - } - merge_with = { - 'Blah3': { - 'e': '2', - 'g': { - 'e': 'f', - } - } - } - merge_how = "list()+dict()+str()" - merger_set = mergers.string_extract_mergers(merge_how) - self.assertEquals(3, len(merger_set)) - merger = mergers.construct(merger_set) - merged = merger.merge(source, merge_with) - self.assertEquals(merged['Blah3'], { - 'e': '2', - 'f': '3', - 'g': { - 'a': 'b', - 'e': 'f', - } - }) + def test_merge_samples(self): + tests = self._load_merge_files('merge_sources') + paths = c_helpers.Paths({}) + cc_handler = cloud_config.CloudConfigPartHandler(paths) + cc_handler.cloud_fn = None + for (payloads, expected_merge) in tests: + cc_handler.handle_part(None, CONTENT_START, None, + None, None, None) + for (i, p) in enumerate(payloads): + cc_handler.handle_part(None, None, "t-%s.yaml" % (i + 1), + p, None, {}) + merged_buf = cc_handler.cloud_buf + cc_handler.handle_part(None, CONTENT_END, None, + None, None, None) + self.assertEquals(expected_merge, merged_buf)