openstack/swift
Code Issues Proposed changes

Fix a few edges where we lost python 2.6 support

Browse Source 
a.k.a. put my head in the sand about the reality of not supporting python 2.6
a little while longer.  We need to get something in the next release notes
about deprecating support for python 2.6 ASAP.  I don't really care enough
about it to keep cleaning up the junk we're going to let slip through not
testing python 2.6 in the gate.

Change-Id: Ib36cd66bda29d75d3b5f4ef0a0ef7b824923df28
This commit is contained in:
Clay Gerrard 2015-01-08 18:22:40 -08:00
parent fee3dcf1f1
commit 92fd28aa6a
3 changed files with 83 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
swift/common/splice.py
View File

@ -28,6 +28,9 @@ __all__ = ['tee', 'splice']
c_loff_t = ctypes.c_long
# python 2.6 doesn't have c_ssize_t
c_ssize_t = getattr(ctypes, 'c_ssize_t', ctypes.c_long)
class Tee(object):
'''Binding to `tee`'''
@ -50,7 +53,7 @@ class Tee(object):
ctypes.c_uint
]
c_tee.restype = ctypes.c_ssize_t
c_tee.restype = c_ssize_t
def errcheck(result, func, arguments):
if result == -1:
@ -131,7 +134,7 @@ class Splice(object):
ctypes.c_uint
]
c_splice.restype = ctypes.c_ssize_t
c_splice.restype = c_ssize_t
def errcheck(result, func, arguments):
if result == -1:

101
test/unit/common/test_splice.py
View File

@ -22,8 +22,10 @@ import logging
import tempfile
import unittest
import contextlib
import re
import mock
import nose
from swift.common.splice import splice, tee
@ -52,10 +54,13 @@ def pipe():
safe_close(fds[1])
@unittest.skipUnless(splice.available, 'splice not available')
class TestSplice(unittest.TestCase):
'''Tests for `splice`'''
def setUp(self):
if not splice.available:
raise nose.SkipTest('splice not available')
def test_flags(self):
'''Test flag attribute availability'''
@ -73,49 +78,52 @@ class TestSplice(unittest.TestCase):
def test_splice_pipe_to_pipe(self):
'''Test `splice` from a pipe to a pipe'''
with pipe() as (p1a, p1b), pipe() as (p2a, p2b):
os.write(p1b, 'abcdef')
res = splice(p1a, None, p2b, None, 3, 0)
self.assertEqual(res, (3, None, None))
self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 3), 'def')
with pipe() as (p1a, p1b):
with pipe() as (p2a, p2b):
os.write(p1b, 'abcdef')
res = splice(p1a, None, p2b, None, 3, 0)
self.assertEqual(res, (3, None, None))
self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 3), 'def')
def test_splice_file_to_pipe(self):
'''Test `splice` from a file to a pipe'''
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb):
fd.write('abcdef')
fd.seek(0, os.SEEK_SET)
with tempfile.NamedTemporaryFile(bufsize=0) as fd:
with pipe() as (pa, pb):
fd.write('abcdef')
fd.seek(0, os.SEEK_SET)
res = splice(fd, None, pb, None, 3, 0)
self.assertEqual(res, (3, None, None))
# `fd.tell()` isn't updated...
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 3)
res = splice(fd, None, pb, None, 3, 0)
self.assertEqual(res, (3, None, None))
# `fd.tell()` isn't updated...
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 3)
fd.seek(0, os.SEEK_SET)
res = splice(fd, 3, pb, None, 3, 0)
self.assertEqual(res, (3, 6, None))
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 0)
fd.seek(0, os.SEEK_SET)
res = splice(fd, 3, pb, None, 3, 0)
self.assertEqual(res, (3, 6, None))
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 0)
self.assertEquals(os.read(pa, 6), 'abcdef')
self.assertEquals(os.read(pa, 6), 'abcdef')
def test_splice_pipe_to_file(self):
'''Test `splice` from a pipe to a file'''
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb):
os.write(pb, 'abcdef')
with tempfile.NamedTemporaryFile(bufsize=0) as fd:
with pipe() as (pa, pb):
os.write(pb, 'abcdef')
res = splice(pa, None, fd, None, 3, 0)
self.assertEqual(res, (3, None, None))
self.assertEqual(fd.tell(), 3)
res = splice(pa, None, fd, None, 3, 0)
self.assertEqual(res, (3, None, None))
self.assertEqual(fd.tell(), 3)
fd.seek(0, os.SEEK_SET)
fd.seek(0, os.SEEK_SET)
res = splice(pa, None, fd, 3, 3, 0)
self.assertEqual(res, (3, None, 6))
self.assertEqual(fd.tell(), 0)
res = splice(pa, None, fd, 3, 3, 0)
self.assertEqual(res, (3, None, 6))
self.assertEqual(fd.tell(), 0)
self.assertEqual(fd.read(6), 'abcdef')
self.assertEqual(fd.read(6), 'abcdef')
@mock.patch.object(splice, '_c_splice')
def test_fileno(self, mock_splice):
@ -157,9 +165,12 @@ class TestSplice(unittest.TestCase):
with open('/dev/null', 'r') as fd:
err = errno.EBADF
msg = r'\[Errno %d\] splice: %s' % (err, os.strerror(err))
self.assertRaisesRegexp(IOError, msg, splice, fd, None, fd, None,
3, 0)
try:
splice(fd, None, fd, None, 3, 0)
except IOError as e:
self.assertTrue(re.match(msg, str(e)))
else:
self.fail('Expected IOError was not raised')
self.assertEqual(ctypes.get_errno(), 0)
@ -198,10 +209,13 @@ class TestSplice(unittest.TestCase):
self.assertTrue(libc.splice_retrieved)
@unittest.skipUnless(tee.available, 'tee not available')
class TestTee(unittest.TestCase):
'''Tests for `tee`'''
def setUp(self):
if not tee.available:
raise nose.SkipTest('tee not available')
@mock.patch('swift.common.splice.tee._c_tee', None)
def test_available(self):
'''Test `available` attribute correctness'''
@ -211,12 +225,13 @@ class TestTee(unittest.TestCase):
def test_tee_pipe_to_pipe(self):
'''Test `tee` from a pipe to a pipe'''
with pipe() as (p1a, p1b), pipe() as (p2a, p2b):
os.write(p1b, 'abcdef')
res = tee(p1a, p2b, 3, 0)
self.assertEqual(res, 3)
self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 6), 'abcdef')
with pipe() as (p1a, p1b):
with pipe() as (p2a, p2b):
os.write(p1b, 'abcdef')
res = tee(p1a, p2b, 3, 0)
self.assertEqual(res, 3)
self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 6), 'abcdef')
@mock.patch.object(tee, '_c_tee')
def test_fileno(self, mock_tee):
@ -251,8 +266,12 @@ class TestTee(unittest.TestCase):
with open('/dev/null', 'r') as fd:
err = errno.EBADF
msg = r'\[Errno %d\] tee: %s' % (err, os.strerror(err))
self.assertRaisesRegexp(IOError, msg, tee, fd, fd, 3, 0)
try:
tee(fd, fd, 3, 0)
except IOError as e:
self.assertTrue(re.match(msg, str(e)))
else:
self.fail('Expected IOError was not raised')
self.assertEqual(ctypes.get_errno(), 0)

29
test/unit/obj/test_diskfile.py
View File

@ -1,4 +1,4 @@
#-*- coding:utf-8 -*-
# -*- coding:utf-8 -*-
# Copyright (c) 2010-2012 OpenStack Foundation
#
# Licensed under the Apache License, Version 2.0 (the "License");
@ -25,6 +25,7 @@ import email
import tempfile
import uuid
import xattr
import re
from shutil import rmtree
from time import time
from tempfile import mkdtemp
@ -767,7 +768,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase):
]
self.assertEqual(locations, expected)
#now without a logger
# now without a logger
locations = [(loc.path, loc.device, loc.partition)
for loc in diskfile.object_audit_location_generator(
devices=tmpdir, mount_check=False)]
@ -2301,9 +2302,12 @@ class TestDiskFile(unittest.TestCase):
with open('/dev/null', 'w') as devnull:
exc_re = (r'tee\(\) failed: tried to move \d+ bytes, but only '
'moved -?\d+')
self.assertRaisesRegexp(Exception, exc_re,
reader.zero_copy_send,
devnull.fileno())
try:
reader.zero_copy_send(devnull.fileno())
except Exception as e:
self.assertTrue(re.match(exc_re, str(e)))
else:
self.fail('Expected Exception was not raised')
def test_splice_to_wsockfd_blocks(self):
if not self._system_can_zero_copy():
@ -2315,12 +2319,7 @@ class TestDiskFile(unittest.TestCase):
reader = df.reader()
self.assertTrue(reader.can_zero_copy_send())
with mock.patch('swift.obj.diskfile.splice') as mock_splice, \
mock.patch.object(reader, 'close', side_effect=reader.close) \
as mock_close, \
open('/dev/null', 'w') as devnull, \
mock.patch('swift.obj.diskfile.trampoline') as mock_trampoline:
def _run_test():
# Set up mock of `splice`
splice_called = [False] # State hack
@ -2370,6 +2369,14 @@ class TestDiskFile(unittest.TestCase):
else:
self.fail('`splice` not called with expected arguments')
with mock.patch('swift.obj.diskfile.splice') as mock_splice:
with mock.patch.object(
reader, 'close', side_effect=reader.close) as mock_close:
with open('/dev/null', 'w') as devnull:
with mock.patch('swift.obj.diskfile.trampoline') as \
mock_trampoline:
_run_test()
def test_create_unlink_cleanup_DiskFileNoSpace(self):
# Test cleanup when DiskFileNoSpace() is raised.
df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123',