improvements for launch index, one fix for cloud-archive
1. Docs for launch-index + examples 2. Tests for launch-index + data files 3. Fixing a bug with cloud-archive yaml types allowed (likes a tuple not a list for some reason) (LP: #1044594) 4. Setting the 'part' content-type if what we actually use is different.
This commit is contained in:
commit
5c12dc3f13
|
@ -44,7 +44,7 @@ class Filter(object):
|
||||||
return True
|
return True
|
||||||
|
|
||||||
def _do_filter(self, message):
|
def _do_filter(self, message):
|
||||||
# Don't use walk() here since we want to do the reforming of the
|
# Don't use walk() here since we want to do the reforming of the
|
||||||
# messages ourselves and not flatten the message listings...
|
# messages ourselves and not flatten the message listings...
|
||||||
if not self._select(message):
|
if not self._select(message):
|
||||||
return None
|
return None
|
||||||
|
|
|
@ -54,7 +54,7 @@ ATTACHMENT_FIELD = 'Number-Attachments'
|
||||||
|
|
||||||
# Only the following content types can have there launch index examined
|
# Only the following content types can have there launch index examined
|
||||||
# in there payload, evey other content type can still provide a header
|
# in there payload, evey other content type can still provide a header
|
||||||
EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config", "text/cloud-config-archive"]
|
EXAMINE_FOR_LAUNCH_INDEX = ["text/cloud-config"]
|
||||||
|
|
||||||
|
|
||||||
class UserDataProcessor(object):
|
class UserDataProcessor(object):
|
||||||
|
@ -84,6 +84,12 @@ class UserDataProcessor(object):
|
||||||
if ctype is None:
|
if ctype is None:
|
||||||
ctype = ctype_orig
|
ctype = ctype_orig
|
||||||
|
|
||||||
|
if ctype != ctype_orig:
|
||||||
|
if CONTENT_TYPE in part:
|
||||||
|
part.replace_header(CONTENT_TYPE, ctype)
|
||||||
|
else:
|
||||||
|
part[CONTENT_TYPE] = ctype
|
||||||
|
|
||||||
if ctype in INCLUDE_TYPES:
|
if ctype in INCLUDE_TYPES:
|
||||||
self._do_include(payload, append_msg)
|
self._do_include(payload, append_msg)
|
||||||
continue
|
continue
|
||||||
|
@ -92,6 +98,8 @@ class UserDataProcessor(object):
|
||||||
self._explode_archive(payload, append_msg)
|
self._explode_archive(payload, append_msg)
|
||||||
continue
|
continue
|
||||||
|
|
||||||
|
# Should this be happening, shouldn't
|
||||||
|
# the part header be modified and not the base?
|
||||||
if CONTENT_TYPE in base_msg:
|
if CONTENT_TYPE in base_msg:
|
||||||
base_msg.replace_header(CONTENT_TYPE, ctype)
|
base_msg.replace_header(CONTENT_TYPE, ctype)
|
||||||
else:
|
else:
|
||||||
|
@ -180,7 +188,7 @@ class UserDataProcessor(object):
|
||||||
self._process_msg(new_msg, append_msg)
|
self._process_msg(new_msg, append_msg)
|
||||||
|
|
||||||
def _explode_archive(self, archive, append_msg):
|
def _explode_archive(self, archive, append_msg):
|
||||||
entries = util.load_yaml(archive, default=[], allowed=[list, set])
|
entries = util.load_yaml(archive, default=[], allowed=(list, set))
|
||||||
for ent in entries:
|
for ent in entries:
|
||||||
# ent can be one of:
|
# ent can be one of:
|
||||||
# dict { 'filename' : 'value', 'content' :
|
# dict { 'filename' : 'value', 'content' :
|
||||||
|
|
|
@ -0,0 +1,30 @@
|
||||||
|
#cloud-config-archive
|
||||||
|
|
||||||
|
# This is an example of a cloud archive
|
||||||
|
# format which includes a set of launch indexes
|
||||||
|
# that will be filtered on (thus only showing
|
||||||
|
# up in instances with that launch index), this
|
||||||
|
# is done by adding the 'launch-index' key which
|
||||||
|
# maps to the integer 'launch-index' that the
|
||||||
|
# corresponding content should be used with.
|
||||||
|
#
|
||||||
|
# It is possible to leave this value out which
|
||||||
|
# will mean that the content will be applicable
|
||||||
|
# for all instances
|
||||||
|
|
||||||
|
- type: foo/wark
|
||||||
|
filename: bar
|
||||||
|
content: |
|
||||||
|
This is my payload
|
||||||
|
hello
|
||||||
|
launch-index: 1 # I will only be used on launch-index 1
|
||||||
|
- this is also payload
|
||||||
|
- |
|
||||||
|
multi line payload
|
||||||
|
here
|
||||||
|
-
|
||||||
|
type: text/upstart-job
|
||||||
|
filename: my-upstart.conf
|
||||||
|
content: |
|
||||||
|
whats this, yo?
|
||||||
|
launch-index: 0 # I will only be used on launch-index 0
|
|
@ -0,0 +1,23 @@
|
||||||
|
#cloud-config
|
||||||
|
# vim: syntax=yaml
|
||||||
|
|
||||||
|
#
|
||||||
|
# This is the configuration syntax that can be provided to have
|
||||||
|
# a given set of cloud config data show up on a certain launch
|
||||||
|
# index (and not other launches) by provided a key here which
|
||||||
|
# will act as a filter on the instances userdata. When
|
||||||
|
# this key is left out (or non-integer) then the content
|
||||||
|
# of this file will always be used for all launch-indexes
|
||||||
|
# (ie the previous behavior).
|
||||||
|
launch-index: 5
|
||||||
|
|
||||||
|
# Upgrade the instance on first boot
|
||||||
|
# (ie run apt-get upgrade)
|
||||||
|
#
|
||||||
|
# Default: false
|
||||||
|
#
|
||||||
|
apt_upgrade: true
|
||||||
|
|
||||||
|
# Other yaml keys below...
|
||||||
|
# .......
|
||||||
|
# .......
|
|
@ -0,0 +1,30 @@
|
||||||
|
#cloud-config-archive
|
||||||
|
---
|
||||||
|
- content: "\n blah: true\n launch-index: 3\n"
|
||||||
|
type: text/cloud-config
|
||||||
|
- content: "\n blah: true\n launch-index: 4\n"
|
||||||
|
type: text/cloud-config
|
||||||
|
- content: The quick brown fox jumps over the lazy dog
|
||||||
|
filename: b0.txt
|
||||||
|
launch-index: 0
|
||||||
|
type: plain/text
|
||||||
|
- content: The quick brown fox jumps over the lazy dog
|
||||||
|
filename: b3.txt
|
||||||
|
launch-index: 3
|
||||||
|
type: plain/text
|
||||||
|
- content: The quick brown fox jumps over the lazy dog
|
||||||
|
filename: b2.txt
|
||||||
|
launch-index: 2
|
||||||
|
type: plain/text
|
||||||
|
- content: '#!/bin/bash \n echo "stuff"'
|
||||||
|
filename: b2.txt
|
||||||
|
launch-index: 2
|
||||||
|
- content: '#!/bin/bash \n echo "stuff"'
|
||||||
|
filename: b2.txt
|
||||||
|
launch-index: 1
|
||||||
|
- content: '#!/bin/bash \n echo "stuff"'
|
||||||
|
filename: b2.txt
|
||||||
|
# Use a string to see if conversion works
|
||||||
|
launch-index: "1"
|
||||||
|
...
|
||||||
|
|
|
@ -0,0 +1,11 @@
|
||||||
|
From nobody Fri Aug 31 17:17:00 2012
|
||||||
|
Content-Type: text/plain; charset="us-ascii"
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
|
||||||
|
|
||||||
|
#cloud-config
|
||||||
|
b: c
|
||||||
|
launch-index: 2
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,39 @@
|
||||||
|
From nobody Fri Aug 31 17:43:04 2012
|
||||||
|
Content-Type: multipart/mixed; boundary="===============1668325974=="
|
||||||
|
MIME-Version: 1.0
|
||||||
|
|
||||||
|
--===============1668325974==
|
||||||
|
Content-Type: text/cloud-config; charset="us-ascii"
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
|
||||||
|
|
||||||
|
#cloud-config
|
||||||
|
b: c
|
||||||
|
launch-index: 2
|
||||||
|
|
||||||
|
|
||||||
|
--===============1668325974==
|
||||||
|
Content-Type: text/plain; charset="us-ascii"
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
|
||||||
|
|
||||||
|
#cloud-config-archive
|
||||||
|
- content: The quick brown fox jumps over the lazy dog
|
||||||
|
filename: b3.txt
|
||||||
|
launch-index: 3
|
||||||
|
type: plain/text
|
||||||
|
|
||||||
|
--===============1668325974==
|
||||||
|
Content-Type: text/plain; charset="us-ascii"
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
|
||||||
|
|
||||||
|
#cloud-config
|
||||||
|
b: c
|
||||||
|
launch-index: 2
|
||||||
|
|
||||||
|
|
||||||
|
--===============1668325974==--
|
|
@ -0,0 +1,11 @@
|
||||||
|
From nobody Fri Aug 31 17:17:00 2012
|
||||||
|
Content-Type: text/plain; charset="us-ascii"
|
||||||
|
MIME-Version: 1.0
|
||||||
|
Launch-Index: 5
|
||||||
|
Content-Transfer-Encoding: 7bit
|
||||||
|
|
||||||
|
|
||||||
|
#cloud-config
|
||||||
|
b: c
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,42 @@
|
||||||
|
import os
|
||||||
|
|
||||||
|
from mocker import MockerTestCase
|
||||||
|
|
||||||
|
from cloudinit import helpers as ch
|
||||||
|
|
||||||
|
|
||||||
|
class ResourceUsingTestCase(MockerTestCase):
|
||||||
|
def __init__(self, methodName="runTest"):
|
||||||
|
MockerTestCase.__init__(self, methodName)
|
||||||
|
self.resource_path = None
|
||||||
|
|
||||||
|
def resourceLocation(self, subname=None):
|
||||||
|
if self.resource_path is None:
|
||||||
|
paths = [
|
||||||
|
os.path.join('tests', 'data'),
|
||||||
|
os.path.join('data'),
|
||||||
|
os.path.join(os.pardir, 'tests', 'data'),
|
||||||
|
os.path.join(os.pardir, 'data'),
|
||||||
|
]
|
||||||
|
for p in paths:
|
||||||
|
if os.path.isdir(p):
|
||||||
|
self.resource_path = p
|
||||||
|
break
|
||||||
|
self.assertTrue((self.resource_path and
|
||||||
|
os.path.isdir(self.resource_path)),
|
||||||
|
msg="Unable to locate test resource data path!")
|
||||||
|
if not subname:
|
||||||
|
return self.resource_path
|
||||||
|
return os.path.join(self.resource_path, subname)
|
||||||
|
|
||||||
|
def readResource(self, name):
|
||||||
|
where = self.resourceLocation(name)
|
||||||
|
with open(where, 'r') as fh:
|
||||||
|
return fh.read()
|
||||||
|
|
||||||
|
def getCloudPaths(self):
|
||||||
|
cp = ch.Paths({
|
||||||
|
'cloud_dir': self.makeDir(),
|
||||||
|
'templates_dir': self.resourceLocation(),
|
||||||
|
})
|
||||||
|
return cp
|
|
@ -0,0 +1,134 @@
|
||||||
|
import copy
|
||||||
|
|
||||||
|
import helpers as th
|
||||||
|
|
||||||
|
import itertools
|
||||||
|
|
||||||
|
from cloudinit.filters import launch_index
|
||||||
|
from cloudinit import user_data as ud
|
||||||
|
from cloudinit import util
|
||||||
|
|
||||||
|
|
||||||
|
def count_messages(root):
|
||||||
|
am = 0
|
||||||
|
for m in root.walk():
|
||||||
|
if ud.is_skippable(m):
|
||||||
|
continue
|
||||||
|
am += 1
|
||||||
|
return am
|
||||||
|
|
||||||
|
|
||||||
|
class TestLaunchFilter(th.ResourceUsingTestCase):
|
||||||
|
|
||||||
|
def assertCounts(self, message, expected_counts):
|
||||||
|
orig_message = copy.deepcopy(message)
|
||||||
|
for (index, count) in expected_counts.items():
|
||||||
|
index = util.safe_int(index)
|
||||||
|
filtered_message = launch_index.Filter(index).apply(message)
|
||||||
|
self.assertEquals(count_messages(filtered_message), count)
|
||||||
|
# Ensure original message still ok/not modified
|
||||||
|
self.assertTrue(self.equivalentMessage(message, orig_message))
|
||||||
|
|
||||||
|
def equivalentMessage(self, msg1, msg2):
|
||||||
|
msg1_count = count_messages(msg1)
|
||||||
|
msg2_count = count_messages(msg2)
|
||||||
|
if msg1_count != msg2_count:
|
||||||
|
return False
|
||||||
|
# Do some basic payload checking
|
||||||
|
msg1_msgs = [m for m in msg1.walk()]
|
||||||
|
msg1_msgs = [m for m in
|
||||||
|
itertools.ifilterfalse(ud.is_skippable, msg1_msgs)]
|
||||||
|
msg2_msgs = [m for m in msg2.walk()]
|
||||||
|
msg2_msgs = [m for m in
|
||||||
|
itertools.ifilterfalse(ud.is_skippable, msg2_msgs)]
|
||||||
|
for i in range(0, len(msg2_msgs)):
|
||||||
|
m1_msg = msg1_msgs[i]
|
||||||
|
m2_msg = msg2_msgs[i]
|
||||||
|
if m1_msg.get_charset() != m2_msg.get_charset():
|
||||||
|
return False
|
||||||
|
if m1_msg.is_multipart() != m2_msg.is_multipart():
|
||||||
|
return False
|
||||||
|
m1_py = m1_msg.get_payload(decode=True)
|
||||||
|
m2_py = m2_msg.get_payload(decode=True)
|
||||||
|
if m1_py != m2_py:
|
||||||
|
return False
|
||||||
|
return True
|
||||||
|
|
||||||
|
def testMultiEmailIndex(self):
|
||||||
|
test_data = self.readResource('filter_cloud_multipart_2.email')
|
||||||
|
ud_proc = ud.UserDataProcessor(self.getCloudPaths())
|
||||||
|
message = ud_proc.process(test_data)
|
||||||
|
self.assertTrue(count_messages(message) > 0)
|
||||||
|
# This file should have the following
|
||||||
|
# indexes -> amount mapping in it
|
||||||
|
expected_counts = {
|
||||||
|
3: 1,
|
||||||
|
2: 2,
|
||||||
|
None: 3,
|
||||||
|
-1: 0,
|
||||||
|
}
|
||||||
|
self.assertCounts(message, expected_counts)
|
||||||
|
|
||||||
|
def testHeaderEmailIndex(self):
|
||||||
|
test_data = self.readResource('filter_cloud_multipart_header.email')
|
||||||
|
ud_proc = ud.UserDataProcessor(self.getCloudPaths())
|
||||||
|
message = ud_proc.process(test_data)
|
||||||
|
self.assertTrue(count_messages(message) > 0)
|
||||||
|
# This file should have the following
|
||||||
|
# indexes -> amount mapping in it
|
||||||
|
expected_counts = {
|
||||||
|
5: 1,
|
||||||
|
-1: 0,
|
||||||
|
'c': 1,
|
||||||
|
None: 1,
|
||||||
|
}
|
||||||
|
self.assertCounts(message, expected_counts)
|
||||||
|
|
||||||
|
def testConfigEmailIndex(self):
|
||||||
|
test_data = self.readResource('filter_cloud_multipart_1.email')
|
||||||
|
ud_proc = ud.UserDataProcessor(self.getCloudPaths())
|
||||||
|
message = ud_proc.process(test_data)
|
||||||
|
self.assertTrue(count_messages(message) > 0)
|
||||||
|
# This file should have the following
|
||||||
|
# indexes -> amount mapping in it
|
||||||
|
expected_counts = {
|
||||||
|
2: 1,
|
||||||
|
-1: 0,
|
||||||
|
None: 1,
|
||||||
|
}
|
||||||
|
self.assertCounts(message, expected_counts)
|
||||||
|
|
||||||
|
def testNoneIndex(self):
|
||||||
|
test_data = self.readResource('filter_cloud_multipart.yaml')
|
||||||
|
ud_proc = ud.UserDataProcessor(self.getCloudPaths())
|
||||||
|
message = ud_proc.process(test_data)
|
||||||
|
start_count = count_messages(message)
|
||||||
|
self.assertTrue(start_count > 0)
|
||||||
|
filtered_message = launch_index.Filter(None).apply(message)
|
||||||
|
self.assertTrue(self.equivalentMessage(message, filtered_message))
|
||||||
|
|
||||||
|
def testIndexes(self):
|
||||||
|
test_data = self.readResource('filter_cloud_multipart.yaml')
|
||||||
|
ud_proc = ud.UserDataProcessor(self.getCloudPaths())
|
||||||
|
message = ud_proc.process(test_data)
|
||||||
|
start_count = count_messages(message)
|
||||||
|
self.assertTrue(start_count > 0)
|
||||||
|
# This file should have the following
|
||||||
|
# indexes -> amount mapping in it
|
||||||
|
expected_counts = {
|
||||||
|
2: 2,
|
||||||
|
3: 2,
|
||||||
|
1: 2,
|
||||||
|
0: 1,
|
||||||
|
4: 1,
|
||||||
|
7: 0,
|
||||||
|
-1: 0,
|
||||||
|
100: 0,
|
||||||
|
# None should just give all back
|
||||||
|
None: start_count,
|
||||||
|
# Non ints should be ignored
|
||||||
|
'c': start_count,
|
||||||
|
# Strings should be converted
|
||||||
|
'1': 2,
|
||||||
|
}
|
||||||
|
self.assertCounts(message, expected_counts)
|
Loading…
Reference in New Issue