diff --git a/oauth2client/contrib/_fcntl_opener.py b/oauth2client/contrib/_fcntl_opener.py new file mode 100644 index 0000000..4e758b9 --- /dev/null +++ b/oauth2client/contrib/_fcntl_opener.py @@ -0,0 +1,85 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import time + +import fcntl + +from oauth2client.contrib.locked_file import _Opener +from oauth2client.contrib.locked_file import AlreadyLockedException +from oauth2client.contrib.locked_file import logger +from oauth2client.contrib.locked_file import validate_file + + +class _FcntlOpener(_Opener): + """Open, lock, and unlock a file using fcntl.lockf.""" + + def open_and_lock(self, timeout, delay): + """Open the file and lock it. + + Args: + timeout: float, How long to try to lock for. + delay: float, How long to wait between retries + + Raises: + AlreadyLockedException: if the lock is already acquired. + IOError: if the open fails. + CredentialsFileSymbolicLinkError: if the file is a symbolic + link. + """ + if self._locked: + raise AlreadyLockedException('File %s is already locked' % + self._filename) + start_time = time.time() + + validate_file(self._filename) + try: + self._fh = open(self._filename, self._mode) + except IOError as e: + # If we can't access with _mode, try _fallback_mode and + # don't lock. + if e.errno in (errno.EPERM, errno.EACCES): + self._fh = open(self._filename, self._fallback_mode) + return + + # We opened in _mode, try to lock the file. + while True: + try: + fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX) + self._locked = True + return + except IOError as e: + # If not retrying, then just pass on the error. + if timeout == 0: + raise + if e.errno != errno.EACCES: + raise + # We could not acquire the lock. Try again. + if (time.time() - start_time) >= timeout: + logger.warn('Could not lock %s in %s seconds', + self._filename, timeout) + if self._fh: + self._fh.close() + self._fh = open(self._filename, self._fallback_mode) + return + time.sleep(delay) + + def unlock_and_close(self): + """Close and unlock the file using the fcntl.lockf primitive.""" + if self._locked: + fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN) + self._locked = False + if self._fh: + self._fh.close() diff --git a/oauth2client/contrib/_win32_opener.py b/oauth2client/contrib/_win32_opener.py new file mode 100644 index 0000000..4a0580e --- /dev/null +++ b/oauth2client/contrib/_win32_opener.py @@ -0,0 +1,109 @@ +# Copyright 2016 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import errno +import time + +import pywintypes +import win32con +import win32file + +from oauth2client.contrib.locked_file import _Opener +from oauth2client.contrib.locked_file import AlreadyLockedException +from oauth2client.contrib.locked_file import logger +from oauth2client.contrib.locked_file import validate_file + + +class _Win32Opener(_Opener): + """Open, lock, and unlock a file using windows primitives.""" + + # Error #33: + # 'The process cannot access the file because another process' + FILE_IN_USE_ERROR = 33 + + # Error #158: + # 'The segment is already unlocked.' + FILE_ALREADY_UNLOCKED_ERROR = 158 + + def open_and_lock(self, timeout, delay): + """Open the file and lock it. + + Args: + timeout: float, How long to try to lock for. + delay: float, How long to wait between retries + + Raises: + AlreadyLockedException: if the lock is already acquired. + IOError: if the open fails. + CredentialsFileSymbolicLinkError: if the file is a symbolic + link. + """ + if self._locked: + raise AlreadyLockedException('File %s is already locked' % + self._filename) + start_time = time.time() + + validate_file(self._filename) + try: + self._fh = open(self._filename, self._mode) + except IOError as e: + # If we can't access with _mode, try _fallback_mode + # and don't lock. + if e.errno == errno.EACCES: + self._fh = open(self._filename, self._fallback_mode) + return + + # We opened in _mode, try to lock the file. + while True: + try: + hfile = win32file._get_osfhandle(self._fh.fileno()) + win32file.LockFileEx( + hfile, + (win32con.LOCKFILE_FAIL_IMMEDIATELY | + win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000, + pywintypes.OVERLAPPED()) + self._locked = True + return + except pywintypes.error as e: + if timeout == 0: + raise + + # If the error is not that the file is already + # in use, raise. + if e[0] != _Win32Opener.FILE_IN_USE_ERROR: + raise + + # We could not acquire the lock. Try again. + if (time.time() - start_time) >= timeout: + logger.warn('Could not lock %s in %s seconds' % ( + self._filename, timeout)) + if self._fh: + self._fh.close() + self._fh = open(self._filename, self._fallback_mode) + return + time.sleep(delay) + + def unlock_and_close(self): + """Close and unlock the file using the win32 primitive.""" + if self._locked: + try: + hfile = win32file._get_osfhandle(self._fh.fileno()) + win32file.UnlockFileEx(hfile, 0, -0x10000, + pywintypes.OVERLAPPED()) + except pywintypes.error as e: + if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR: + raise + self._locked = False + if self._fh: + self._fh.close() diff --git a/oauth2client/contrib/locked_file.py b/oauth2client/contrib/locked_file.py index 1028a7e..ab7de2b 100644 --- a/oauth2client/contrib/locked_file.py +++ b/oauth2client/contrib/locked_file.py @@ -173,164 +173,6 @@ class _PosixOpener(_Opener): return '%s.lock' % filename -try: - import fcntl - - class _FcntlOpener(_Opener): - """Open, lock, and unlock a file using fcntl.lockf.""" - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - CredentialsFileSymbolicLinkError: if the file is a symbolic - link. - """ - if self._locked: - raise AlreadyLockedException('File %s is already locked' % - self._filename) - start_time = time.time() - - validate_file(self._filename) - try: - self._fh = open(self._filename, self._mode) - except IOError as e: - # If we can't access with _mode, try _fallback_mode and - # don't lock. - if e.errno in (errno.EPERM, errno.EACCES): - self._fh = open(self._filename, self._fallback_mode) - return - - # We opened in _mode, try to lock the file. - while True: - try: - fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX) - self._locked = True - return - except IOError as e: - # If not retrying, then just pass on the error. - if timeout == 0: - raise - if e.errno != errno.EACCES: - raise - # We could not acquire the lock. Try again. - if (time.time() - start_time) >= timeout: - logger.warn('Could not lock %s in %s seconds', - self._filename, timeout) - if self._fh: - self._fh.close() - self._fh = open(self._filename, self._fallback_mode) - return - time.sleep(delay) - - def unlock_and_close(self): - """Close and unlock the file using the fcntl.lockf primitive.""" - if self._locked: - fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN) - self._locked = False - if self._fh: - self._fh.close() -except ImportError: - _FcntlOpener = None - - -try: - import pywintypes - import win32con - import win32file - - class _Win32Opener(_Opener): - """Open, lock, and unlock a file using windows primitives.""" - - # Error #33: - # 'The process cannot access the file because another process' - FILE_IN_USE_ERROR = 33 - - # Error #158: - # 'The segment is already unlocked.' - FILE_ALREADY_UNLOCKED_ERROR = 158 - - def open_and_lock(self, timeout, delay): - """Open the file and lock it. - - Args: - timeout: float, How long to try to lock for. - delay: float, How long to wait between retries - - Raises: - AlreadyLockedException: if the lock is already acquired. - IOError: if the open fails. - CredentialsFileSymbolicLinkError: if the file is a symbolic - link. - """ - if self._locked: - raise AlreadyLockedException('File %s is already locked' % - self._filename) - start_time = time.time() - - validate_file(self._filename) - try: - self._fh = open(self._filename, self._mode) - except IOError as e: - # If we can't access with _mode, try _fallback_mode - # and don't lock. - if e.errno == errno.EACCES: - self._fh = open(self._filename, self._fallback_mode) - return - - # We opened in _mode, try to lock the file. - while True: - try: - hfile = win32file._get_osfhandle(self._fh.fileno()) - win32file.LockFileEx( - hfile, - (win32con.LOCKFILE_FAIL_IMMEDIATELY | - win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000, - pywintypes.OVERLAPPED()) - self._locked = True - return - except pywintypes.error as e: - if timeout == 0: - raise - - # If the error is not that the file is already - # in use, raise. - if e[0] != _Win32Opener.FILE_IN_USE_ERROR: - raise - - # We could not acquire the lock. Try again. - if (time.time() - start_time) >= timeout: - logger.warn('Could not lock %s in %s seconds' % ( - self._filename, timeout)) - if self._fh: - self._fh.close() - self._fh = open(self._filename, self._fallback_mode) - return - time.sleep(delay) - - def unlock_and_close(self): - """Close and unlock the file using the win32 primitive.""" - if self._locked: - try: - hfile = win32file._get_osfhandle(self._fh.fileno()) - win32file.UnlockFileEx(hfile, 0, -0x10000, - pywintypes.OVERLAPPED()) - except pywintypes.error as e: - if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR: - raise - self._locked = False - if self._fh: - self._fh.close() -except ImportError: - _Win32Opener = None - - class LockedFile(object): """Represent a file that has exclusive access.""" @@ -347,10 +189,15 @@ class LockedFile(object): """ opener = None if not opener and use_native_locking: - if _Win32Opener: + try: + from oauth2client.contrib._win32_opener import _Win32Opener opener = _Win32Opener(filename, mode, fallback_mode) - if _FcntlOpener: - opener = _FcntlOpener(filename, mode, fallback_mode) + except ImportError: + try: + from oauth2client.contrib._fcntl_opener import _FcntlOpener + opener = _FcntlOpener(filename, mode, fallback_mode) + except ImportError: + pass if not opener: opener = _PosixOpener(filename, mode, fallback_mode) diff --git a/tests/contrib/test_locked_file.py b/tests/contrib/test_locked_file.py index dd70fbe..6a4be46 100644 --- a/tests/contrib/test_locked_file.py +++ b/tests/contrib/test_locked_file.py @@ -14,10 +14,10 @@ import errno import os +import sys import tempfile import mock -from six.moves import reload_module import unittest2 from oauth2client.contrib import locked_file @@ -85,7 +85,6 @@ class TestPosixOpener(TestOpener): # error gets re-raised, but the module lets the if statement fall # through. instance, _ = self._make_one() - fh_mock = mock.Mock() mock_open.side_effect = [IOError(errno.ENOENT, '')] instance.open_and_lock(1, 1) @@ -188,28 +187,33 @@ class TestLockedFile(unittest2.TestCase): @mock.patch('oauth2client.contrib.locked_file._PosixOpener') def test_ctor_minimal(self, opener_mock): - instance = locked_file.LockedFile( + locked_file.LockedFile( 'a_file', 'r+', 'r', use_native_locking=False) opener_mock.assert_called_with('a_file', 'r+', 'r') - @mock.patch('oauth2client.contrib.locked_file._Win32Opener') - def test_ctor_native_win32(self, opener_mock): - instance = locked_file.LockedFile( + @mock.patch.dict('sys.modules', { + 'oauth2client.contrib._win32_opener': mock.Mock()}) + def test_ctor_native_win32(self): + _win32_opener_mock = sys.modules['oauth2client.contrib._win32_opener'] + locked_file.LockedFile( 'a_file', 'r+', 'r', use_native_locking=True) - opener_mock.assert_called_with('a_file', 'r+', 'r') + _win32_opener_mock._Win32Opener.assert_called_with('a_file', 'r+', 'r') - @mock.patch('oauth2client.contrib.locked_file._FcntlOpener') - @mock.patch.object(locked_file, '_Win32Opener', None) - def test_ctor_native_fcntl(self, opener_mock): - instance = locked_file.LockedFile( + @mock.patch.dict('sys.modules', { + 'oauth2client.contrib._win32_opener': None, + 'oauth2client.contrib._fcntl_opener': mock.Mock()}) + def test_ctor_native_fcntl(self): + _fnctl_opener_mock = sys.modules['oauth2client.contrib._fcntl_opener'] + locked_file.LockedFile( 'a_file', 'r+', 'r', use_native_locking=True) - opener_mock.assert_called_with('a_file', 'r+', 'r') + _fnctl_opener_mock._FcntlOpener.assert_called_with('a_file', 'r+', 'r') @mock.patch('oauth2client.contrib.locked_file._PosixOpener') - @mock.patch.object(locked_file, '_Win32Opener', None) - @mock.patch.object(locked_file, '_FcntlOpener', None) + @mock.patch.dict('sys.modules', { + 'oauth2client.contrib._win32_opener': None, + 'oauth2client.contrib._fcntl_opener': None}) def test_ctor_native_posix_fallback(self, opener_mock): - instance = locked_file.LockedFile( + locked_file.LockedFile( 'a_file', 'r+', 'r', use_native_locking=True) opener_mock.assert_called_with('a_file', 'r+', 'r')