openstack/deb-python-kafka
Code Issues Proposed changes

Convert serveral tests to struct.pack

Browse Source
This commit is contained in:
Mark Roberts
2014-04-08 22:10:06 -07:00
parent d7c5bbf7d4
commit ac9cf9e944
1 changed files with 93 additions and 81 deletions
Download Patch File Download Diff File Expand all files Collapse all files

174
test/test_protocol.py
View File

@@ -37,24 +37,24 @@ class TestProtocol(unittest.TestCase):
self.assertEqual(msg.key, None) self.assertEqual(msg.key, None)
# Need to decode to check since gzipped payload is non-deterministic # Need to decode to check since gzipped payload is non-deterministic
decoded = gzip_decode(msg.value) decoded = gzip_decode(msg.value)
expect = ( expect = "".join([
"\x00\x00\x00\x00\x00\x00\x00\x00" # MsgSet1 Offset struct.pack(">q", 0), # MsgSet offset
"\x00\x00\x00\x10" # MsgSet1 Size struct.pack(">i", 16), # MsgSet size
"\x4c\x9f\x5b\xc2" # Msg1 CRC struct.pack(">i", 1285512130), # CRC
"\x00" # Msg1 Magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg1 Flags struct.pack(">i", -1), # -1 indicates a null key
"\xff\xff\xff\xff" # Msg1, null key struct.pack(">i", 2), # Msg length (bytes)
"\x00\x00\x00\x02" # Msg1, msg Size "v1", # Message contents
"v1" # Msg1, contents
"\x00\x00\x00\x00\x00\x00\x00\x00" # MsgSet2 Offset struct.pack(">q", 0), # MsgSet offset
"\x00\x00\x00\x10" # MsgSet2 Size struct.pack(">i", 16), # MsgSet size
"\xd5\x96\x0a\x78" # Msg2, CRC struct.pack(">i", -711587208), # CRC
"\x00" # Msg2, magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg2, flags struct.pack(">i", -1), # -1 indicates a null key
"\xff\xff\xff\xff" # Msg2, null key struct.pack(">i", 2), # Msg length (bytes)
"\x00\x00\x00\x02" # Msg2, msg size "v2", # Message contents
"v2" # Msg2, contents ])
)
self.assertEqual(decoded, expect) self.assertEqual(decoded, expect)
@unittest.skipUnless(has_snappy(), "Snappy not available") @unittest.skipUnless(has_snappy(), "Snappy not available")
@@ -65,89 +65,101 @@ class TestProtocol(unittest.TestCase):
self.assertEqual(msg.attributes, KafkaProtocol.ATTRIBUTE_CODEC_MASK & self.assertEqual(msg.attributes, KafkaProtocol.ATTRIBUTE_CODEC_MASK &
KafkaProtocol.CODEC_SNAPPY) KafkaProtocol.CODEC_SNAPPY)
self.assertEqual(msg.key, None) self.assertEqual(msg.key, None)
expect = ( expect = "".join([
"\x00\x00\x00\x00\x00\x00\x00\x00" # MsgSet1 Offset struct.pack(">q", 0), # MsgSet offset
"\x00\x00\x00\x10" # MsgSet1 Size struct.pack(">i", 16), # MsgSet size
"\x4c\x9f\x5b\xc2" # Msg1 CRC struct.pack(">i", 1285512130), # CRC
"\x00" # Msg1 Magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg1 Flags struct.pack(">i", -1), # -1 indicates a null key
"\xff\xff\xff\xff" # Msg1, null key struct.pack(">i", 2), # Msg length (bytes)
"\x00\x00\x00\x02" # Msg1, msg Size "v1", # Message contents
"v1" # Msg1, contents
"\x00\x00\x00\x00\x00\x00\x00\x00" # MsgSet2 Offset struct.pack(">q", 0), # MsgSet offset
"\x00\x00\x00\x10" # MsgSet2 Size struct.pack(">i", 16), # MsgSet size
"\xd5\x96\x0a\x78" # Msg2, CRC struct.pack(">i", -711587208), # CRC
"\x00" # Msg2, magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg2, flags struct.pack(">i", -1), # -1 indicates a null key
"\xff\xff\xff\xff" # Msg2, null key struct.pack(">i", 2), # Msg length (bytes)
"\x00\x00\x00\x02" # Msg2, msg size "v2", # Message contents
"v2" # Msg2, contents ])
)
self.assertEqual(msg.value, expect) self.assertEqual(msg.value, expect)
def test_encode_message_header(self): def test_encode_message_header(self):
expect = ( expect = "".join([
"\x00\n" # API Key struct.pack(">h", 10), # API Key
"\x00\x00" # API Version struct.pack(">h", 0), # API Version
"\x00\x00\x00\x04" # CorrelationId struct.pack(">i", 4), # Correlation Id
"\x00\x07" # Client length struct.pack(">h", len("client3")), # Length of clientId
"client3" # Client Id "client3", # ClientId
) ])
encoded = KafkaProtocol._encode_message_header("client3", 4, 10) encoded = KafkaProtocol._encode_message_header("client3", 4, 10)
self.assertEqual(encoded, expect) self.assertEqual(encoded, expect)
def test_encode_message(self): def test_encode_message(self):
message = create_message("test", "key") message = create_message("test", "key")
encoded = KafkaProtocol._encode_message(message) encoded = KafkaProtocol._encode_message(message)
expect = ( expect = "".join([
"\xaa\xf1\x8f\x5b" # CRC struct.pack(">i", -1427009701), # CRC
"\x00" # Magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Flags struct.pack(">i", 3), # Length of key
"\x00\x00\x00\x03" # Key Length "key", # key
"key" # Key contents struct.pack(">i", 4), # Length of value
"\x00\x00\x00\x04" # Msg Length "test", # value
"test" # Msg contents ])
)
self.assertEqual(encoded, expect) self.assertEqual(encoded, expect)
def test_decode_message(self):
encoded = "".join([
struct.pack(">i", -1427009701), # CRC
struct.pack(">bb", 0, 0), # Magic, flags
struct.pack(">i", 3), # Length of key
"key", # key
struct.pack(">i", 4), # Length of value
"test", # value
])
offset = 10
(returned_offset, decoded_message) = list(KafkaProtocol._decode_message(encoded, offset))[0]
self.assertEqual(returned_offset, offset)
self.assertEqual(decoded_message, create_message("test", "key"))
def test_encode_message_failure(self): def test_encode_message_failure(self):
with self.assertRaises(ProtocolError): with self.assertRaises(ProtocolError):
KafkaProtocol._encode_message(Message(1, 0, "key", "test")) KafkaProtocol._encode_message(Message(1, 0, "key", "test"))
def test_encode_message_set(self): def test_encode_message_set(self):
message_set = [create_message("v1", "k1"), create_message("v2", "k2")] message_set = [
create_message("v1", "k1"),
create_message("v2", "k2")
]
encoded = KafkaProtocol._encode_message_set(message_set) encoded = KafkaProtocol._encode_message_set(message_set)
expect = ( expect = "".join([
"\x00\x00\x00\x00\x00\x00\x00\x00" # Msgset1, Offset (Meaningless) struct.pack(">q", 0), # MsgSet Offset
"\x00\x00\x00\x12" # Msgset1, Msg Size struct.pack(">i", 18), # Msg Size
"\x57\xe7\x49\x6e" # Msg1, CRC struct.pack(">i", 1474775406), # CRC
"\x00" # Msg1, Magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg1, Flags struct.pack(">i", 2), # Length of key
"\x00\x00\x00\x02" # Msg1, key size "k1", # Key
"k1" # Msg1, key struct.pack(">i", 2), # Length of value
"\x00\x00\x00\x02" # Msg1, value size "v1", # Value
"v1" # Msg1, value
"\x00\x00\x00\x00\x00\x00\x00\x00" # Msgset2, Offset (Meaningless) struct.pack(">q", 0), # MsgSet Offset
"\x00\x00\x00\x12" # Msgset2, Msg Size struct.pack(">i", 18), # Msg Size
"\xff\x06\x02\x49" # Msg2, CRC struct.pack(">i", -16383415), # CRC
"\x00" # Msg2, Magic struct.pack(">bb", 0, 0), # Magic, flags
"\x00" # Msg2, flags struct.pack(">i", 2), # Length of key
"\x00\x00\x00\x02" # Msg2, key size "k2", # Key
"k2" # Msg2, key struct.pack(">i", 2), # Length of value
"\x00\x00\x00\x02" # Msg2, value size "v2", # Value
"v2" # MSg2, value ])
)
self.assertEqual(encoded, expect) self.assertEqual(encoded, expect)
def test_decode_message(self):
encoded = "\xaa\xf1\x8f[\x00\x00\x00\x00\x00\x03key\x00\x00\x00\x04test"
offset = 10
(returned_offset, decoded_message) = \
list(KafkaProtocol._decode_message(encoded, offset))[0]
self.assertEqual(returned_offset, offset)
self.assertEqual(decoded_message, create_message("test", "key"))
def test_decode_message_set(self): def test_decode_message_set(self):
encoded = ('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10L\x9f[\xc2' encoded = ('\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x10L\x9f[\xc2'
'\x00\x00\xff\xff\xff\xff\x00\x00\x00\x02v1\x00\x00\x00\x00' '\x00\x00\xff\xff\xff\xff\x00\x00\x00\x02v1\x00\x00\x00\x00'