openstack/swift
Code Issues Proposed changes

Fix a few edges where we lost python 2.6 support

Browse Source 
a.k.a. put my head in the sand about the reality of not supporting python 2.6
a little while longer.  We need to get something in the next release notes
about deprecating support for python 2.6 ASAP.  I don't really care enough
about it to keep cleaning up the junk we're going to let slip through not
testing python 2.6 in the gate.

Change-Id: Ib36cd66bda29d75d3b5f4ef0a0ef7b824923df28
This commit is contained in:
Clay Gerrard 2015-01-08 18:22:40 -08:00
parent fee3dcf1f1
commit 92fd28aa6a
3 changed files with 83 additions and 54 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
swift/common/splice.py
View File

@ -28,6 +28,9 @@ __all__ = ['tee', 'splice']
c_loff_t = ctypes.c_long c_loff_t = ctypes.c_long
# python 2.6 doesn't have c_ssize_t
c_ssize_t = getattr(ctypes, 'c_ssize_t', ctypes.c_long)
class Tee(object): class Tee(object):
'''Binding to `tee`''' '''Binding to `tee`'''
@ -50,7 +53,7 @@ class Tee(object):
ctypes.c_uint ctypes.c_uint
] ]
c_tee.restype = ctypes.c_ssize_t c_tee.restype = c_ssize_t
def errcheck(result, func, arguments): def errcheck(result, func, arguments):
if result == -1: if result == -1:
@ -131,7 +134,7 @@ class Splice(object):
ctypes.c_uint ctypes.c_uint
] ]
c_splice.restype = ctypes.c_ssize_t c_splice.restype = c_ssize_t
def errcheck(result, func, arguments): def errcheck(result, func, arguments):
if result == -1: if result == -1:

101
test/unit/common/test_splice.py
View File

@ -22,8 +22,10 @@ import logging
import tempfile import tempfile
import unittest import unittest
import contextlib import contextlib
import re
import mock import mock
import nose
from swift.common.splice import splice, tee from swift.common.splice import splice, tee
@ -52,10 +54,13 @@ def pipe():
safe_close(fds[1]) safe_close(fds[1])
@unittest.skipUnless(splice.available, 'splice not available')
class TestSplice(unittest.TestCase): class TestSplice(unittest.TestCase):
'''Tests for `splice`''' '''Tests for `splice`'''
def setUp(self):
if not splice.available:
raise nose.SkipTest('splice not available')
def test_flags(self): def test_flags(self):
'''Test flag attribute availability''' '''Test flag attribute availability'''
@ -73,49 +78,52 @@ class TestSplice(unittest.TestCase):
def test_splice_pipe_to_pipe(self): def test_splice_pipe_to_pipe(self):
'''Test `splice` from a pipe to a pipe''' '''Test `splice` from a pipe to a pipe'''
with pipe() as (p1a, p1b), pipe() as (p2a, p2b): with pipe() as (p1a, p1b):
os.write(p1b, 'abcdef') with pipe() as (p2a, p2b):
res = splice(p1a, None, p2b, None, 3, 0) os.write(p1b, 'abcdef')
self.assertEqual(res, (3, None, None)) res = splice(p1a, None, p2b, None, 3, 0)
self.assertEqual(os.read(p2a, 3), 'abc') self.assertEqual(res, (3, None, None))
self.assertEqual(os.read(p1a, 3), 'def') self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 3), 'def')
def test_splice_file_to_pipe(self): def test_splice_file_to_pipe(self):
'''Test `splice` from a file to a pipe''' '''Test `splice` from a file to a pipe'''
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb): with tempfile.NamedTemporaryFile(bufsize=0) as fd:
fd.write('abcdef') with pipe() as (pa, pb):
fd.seek(0, os.SEEK_SET) fd.write('abcdef')
fd.seek(0, os.SEEK_SET)
res = splice(fd, None, pb, None, 3, 0) res = splice(fd, None, pb, None, 3, 0)
self.assertEqual(res, (3, None, None)) self.assertEqual(res, (3, None, None))
# `fd.tell()` isn't updated... # `fd.tell()` isn't updated...
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 3) self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 3)
fd.seek(0, os.SEEK_SET) fd.seek(0, os.SEEK_SET)
res = splice(fd, 3, pb, None, 3, 0) res = splice(fd, 3, pb, None, 3, 0)
self.assertEqual(res, (3, 6, None)) self.assertEqual(res, (3, 6, None))
self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 0) self.assertEqual(os.lseek(fd.fileno(), 0, os.SEEK_CUR), 0)
self.assertEquals(os.read(pa, 6), 'abcdef') self.assertEquals(os.read(pa, 6), 'abcdef')
def test_splice_pipe_to_file(self): def test_splice_pipe_to_file(self):
'''Test `splice` from a pipe to a file''' '''Test `splice` from a pipe to a file'''
with tempfile.NamedTemporaryFile(bufsize=0) as fd, pipe() as (pa, pb): with tempfile.NamedTemporaryFile(bufsize=0) as fd:
os.write(pb, 'abcdef') with pipe() as (pa, pb):
os.write(pb, 'abcdef')
res = splice(pa, None, fd, None, 3, 0) res = splice(pa, None, fd, None, 3, 0)
self.assertEqual(res, (3, None, None)) self.assertEqual(res, (3, None, None))
self.assertEqual(fd.tell(), 3) self.assertEqual(fd.tell(), 3)
fd.seek(0, os.SEEK_SET) fd.seek(0, os.SEEK_SET)
res = splice(pa, None, fd, 3, 3, 0) res = splice(pa, None, fd, 3, 3, 0)
self.assertEqual(res, (3, None, 6)) self.assertEqual(res, (3, None, 6))
self.assertEqual(fd.tell(), 0) self.assertEqual(fd.tell(), 0)
self.assertEqual(fd.read(6), 'abcdef') self.assertEqual(fd.read(6), 'abcdef')
@mock.patch.object(splice, '_c_splice') @mock.patch.object(splice, '_c_splice')
def test_fileno(self, mock_splice): def test_fileno(self, mock_splice):
@ -157,9 +165,12 @@ class TestSplice(unittest.TestCase):
with open('/dev/null', 'r') as fd: with open('/dev/null', 'r') as fd:
err = errno.EBADF err = errno.EBADF
msg = r'\[Errno %d\] splice: %s' % (err, os.strerror(err)) msg = r'\[Errno %d\] splice: %s' % (err, os.strerror(err))
try:
self.assertRaisesRegexp(IOError, msg, splice, fd, None, fd, None, splice(fd, None, fd, None, 3, 0)
3, 0) except IOError as e:
self.assertTrue(re.match(msg, str(e)))
else:
self.fail('Expected IOError was not raised')
self.assertEqual(ctypes.get_errno(), 0) self.assertEqual(ctypes.get_errno(), 0)
@ -198,10 +209,13 @@ class TestSplice(unittest.TestCase):
self.assertTrue(libc.splice_retrieved) self.assertTrue(libc.splice_retrieved)
@unittest.skipUnless(tee.available, 'tee not available')
class TestTee(unittest.TestCase): class TestTee(unittest.TestCase):
'''Tests for `tee`''' '''Tests for `tee`'''
def setUp(self):
if not tee.available:
raise nose.SkipTest('tee not available')
@mock.patch('swift.common.splice.tee._c_tee', None) @mock.patch('swift.common.splice.tee._c_tee', None)
def test_available(self): def test_available(self):
'''Test `available` attribute correctness''' '''Test `available` attribute correctness'''
@ -211,12 +225,13 @@ class TestTee(unittest.TestCase):
def test_tee_pipe_to_pipe(self): def test_tee_pipe_to_pipe(self):
'''Test `tee` from a pipe to a pipe''' '''Test `tee` from a pipe to a pipe'''
with pipe() as (p1a, p1b), pipe() as (p2a, p2b): with pipe() as (p1a, p1b):
os.write(p1b, 'abcdef') with pipe() as (p2a, p2b):
res = tee(p1a, p2b, 3, 0) os.write(p1b, 'abcdef')
self.assertEqual(res, 3) res = tee(p1a, p2b, 3, 0)
self.assertEqual(os.read(p2a, 3), 'abc') self.assertEqual(res, 3)
self.assertEqual(os.read(p1a, 6), 'abcdef') self.assertEqual(os.read(p2a, 3), 'abc')
self.assertEqual(os.read(p1a, 6), 'abcdef')
@mock.patch.object(tee, '_c_tee') @mock.patch.object(tee, '_c_tee')
def test_fileno(self, mock_tee): def test_fileno(self, mock_tee):
@ -251,8 +266,12 @@ class TestTee(unittest.TestCase):
with open('/dev/null', 'r') as fd: with open('/dev/null', 'r') as fd:
err = errno.EBADF err = errno.EBADF
msg = r'\[Errno %d\] tee: %s' % (err, os.strerror(err)) msg = r'\[Errno %d\] tee: %s' % (err, os.strerror(err))
try:
self.assertRaisesRegexp(IOError, msg, tee, fd, fd, 3, 0) tee(fd, fd, 3, 0)
except IOError as e:
self.assertTrue(re.match(msg, str(e)))
else:
self.fail('Expected IOError was not raised')
self.assertEqual(ctypes.get_errno(), 0) self.assertEqual(ctypes.get_errno(), 0)

29
test/unit/obj/test_diskfile.py
View File

@ -1,4 +1,4 @@
#-*- coding:utf-8 -*- # -*- coding:utf-8 -*-
# Copyright (c) 2010-2012 OpenStack Foundation # Copyright (c) 2010-2012 OpenStack Foundation
# #
# Licensed under the Apache License, Version 2.0 (the "License"); # Licensed under the Apache License, Version 2.0 (the "License");
@ -25,6 +25,7 @@ import email
import tempfile import tempfile
import uuid import uuid
import xattr import xattr
import re
from shutil import rmtree from shutil import rmtree
from time import time from time import time
from tempfile import mkdtemp from tempfile import mkdtemp
@ -767,7 +768,7 @@ class TestObjectAuditLocationGenerator(unittest.TestCase):
] ]
self.assertEqual(locations, expected) self.assertEqual(locations, expected)
#now without a logger # now without a logger
locations = [(loc.path, loc.device, loc.partition) locations = [(loc.path, loc.device, loc.partition)
for loc in diskfile.object_audit_location_generator( for loc in diskfile.object_audit_location_generator(
devices=tmpdir, mount_check=False)] devices=tmpdir, mount_check=False)]
@ -2301,9 +2302,12 @@ class TestDiskFile(unittest.TestCase):
with open('/dev/null', 'w') as devnull: with open('/dev/null', 'w') as devnull:
exc_re = (r'tee\(\) failed: tried to move \d+ bytes, but only ' exc_re = (r'tee\(\) failed: tried to move \d+ bytes, but only '
'moved -?\d+') 'moved -?\d+')
self.assertRaisesRegexp(Exception, exc_re, try:
reader.zero_copy_send, reader.zero_copy_send(devnull.fileno())
devnull.fileno()) except Exception as e:
self.assertTrue(re.match(exc_re, str(e)))
else:
self.fail('Expected Exception was not raised')
def test_splice_to_wsockfd_blocks(self): def test_splice_to_wsockfd_blocks(self):
if not self._system_can_zero_copy(): if not self._system_can_zero_copy():
@ -2315,12 +2319,7 @@ class TestDiskFile(unittest.TestCase):
reader = df.reader() reader = df.reader()
self.assertTrue(reader.can_zero_copy_send()) self.assertTrue(reader.can_zero_copy_send())
with mock.patch('swift.obj.diskfile.splice') as mock_splice, \ def _run_test():
mock.patch.object(reader, 'close', side_effect=reader.close) \
as mock_close, \
open('/dev/null', 'w') as devnull, \
mock.patch('swift.obj.diskfile.trampoline') as mock_trampoline:
# Set up mock of `splice` # Set up mock of `splice`
splice_called = [False] # State hack splice_called = [False] # State hack
@ -2370,6 +2369,14 @@ class TestDiskFile(unittest.TestCase):
else: else:
self.fail('`splice` not called with expected arguments') self.fail('`splice` not called with expected arguments')
with mock.patch('swift.obj.diskfile.splice') as mock_splice:
with mock.patch.object(
reader, 'close', side_effect=reader.close) as mock_close:
with open('/dev/null', 'w') as devnull:
with mock.patch('swift.obj.diskfile.trampoline') as \
mock_trampoline:
_run_test()
def test_create_unlink_cleanup_DiskFileNoSpace(self): def test_create_unlink_cleanup_DiskFileNoSpace(self):
# Test cleanup when DiskFileNoSpace() is raised. # Test cleanup when DiskFileNoSpace() is raised.
df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123', df = self.df_mgr.get_diskfile(self.existing_device, '0', 'abc', '123',