diff --git a/oauth2client/locked_file.py b/oauth2client/locked_file.py new file mode 100644 index 0000000..c78410c --- /dev/null +++ b/oauth2client/locked_file.py @@ -0,0 +1,254 @@ +# Copyright 2011 Google Inc. All Rights Reserved. + +"""Locked file interface that should work on Unix and Windows pythons. + +This module first tries to use fcntl locking to ensure serialized access +to a file, then falls back on a lock file if that is unavialable. + +Usage: + f = LockedFile('filename', 'r+b', 'rb') + f.open_and_lock() + if f.is_locked(): + print 'Acquired filename with r+b mode' + f.file_handle().write('locked data') + else: + print 'Aquired filename with rb mode' + f.unlock_and_close() +""" + +__author__ = 'cache@google.com (David T McWherter)' + +import errno +import logging +import os +import time + +logger = logging.getLogger(__name__) + + +class AlreadyLockedException(Exception): + """Trying to lock a file that has already been locked by the LockedFile.""" + pass + + +class _Opener(object): + """Base class for different locking primitives.""" + + def __init__(self, filename, mode, fallback_mode): + """Create an Opener. + + Args: + filename: string, The pathname of the file. + mode: string, The preferred mode to access the file with. + fallback_mode: string, The mode to use if locking fails. + """ + self._locked = False + self._filename = filename + self._mode = mode + self._fallback_mode = fallback_mode + self._fh = None + + def is_locked(self): + """Was the file locked.""" + return self._locked + + def file_handle(self): + """The file handle to the file. Valid only after opened.""" + return self._fh + + def filename(self): + """The filename that is being locked.""" + return self._filename + + def open_and_lock(self, timeout, delay): + """Open the file and lock it. + + Args: + timeout: float, How long to try to lock for. + delay: float, How long to wait between retries. + """ + pass + + def unlock_and_close(self): + """Unlock and close the file.""" + pass + + +class _PosixOpener(_Opener): + """Lock files using Posix advisory lock files.""" + + def open_and_lock(self, timeout, delay): + """Open the file and lock it. + + Tries to create a .lock file next to the file we're trying to open. + + Args: + timeout: float, How long to try to lock for. + delay: float, How long to wait between retries. + + Raises: + AlreadyLockedException: if the lock is already acquired. + IOError: if the open fails. + """ + if self._locked: + raise AlreadyLockedException('File %s is already locked' % + self._filename) + self._locked = False + + try: + self._fh = open(self._filename, self._mode) + except IOError, e: + # If we can't access with _mode, try _fallback_mode and don't lock. + if e.errno == errno.EACCES: + self._fh = open(self._filename, self._fallback_mode) + return + + lock_filename = self._posix_lockfile(self._filename) + start_time = time.time() + while True: + try: + self._lock_fd = os.open(lock_filename, + os.O_CREAT|os.O_EXCL|os.O_RDWR) + self._locked = True + break + + except OSError, e: + if e.errno != errno.EEXIST: + raise + if (time.time() - start_time) >= timeout: + logger.warn('Could not acquire lock %s in %s seconds' % ( + lock_filename, timeout)) + # Close the file and open in fallback_mode. + if self._fh: + self._fh.close() + self._fh = open(self._filename, self._fallback_mode) + return + time.sleep(delay) + + def unlock_and_close(self): + """Unlock a file by removing the .lock file, and close the handle.""" + if self._locked: + lock_filename = self._posix_lockfile(self._filename) + os.unlink(lock_filename) + os.close(self._lock_fd) + self._locked = False + self._lock_fd = None + if self._fh: + self._fh.close() + + def _posix_lockfile(self, filename): + """The name of the lock file to use for posix locking.""" + return '%s.lock' % filename + + +try: + import fcntl + class _FcntlOpener(_Opener): + """Open, lock, and unlock a file using fcntl.lockf.""" + + def open_and_lock(self, timeout, delay): + """Open the file and lock it. + + Args: + timeout: float, How long to try to lock for. + delay: float, How long to wait between retries + + Raises: + AlreadyLockedException: if the lock is already acquired. + IOError: if the open fails. + """ + if self._locked: + raise AlreadyLockedException('File %s is already locked' % + self._filename) + start_time = time.time() + + try: + self._fh = open(self._filename, self._mode) + except IOError, e: + # If we can't access with _mode, try _fallback_mode and don't lock. + if e.errno == errno.EACCES: + self._fh = open(self._filename, self._fallback_mode) + return + + # We opened in _mode, try to lock the file. + while True: + try: + fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX) + self._locked = True + return + except IOError, e: + # If not retrying, then just pass on the error. + if timeout == 0: + raise e + if e.errno != errno.EACCES: + raise e + # We could not acquire the lock. Try again. + if (time.time() - start_time) >= timeout: + logger.warn('Could not lock %s in %s seconds' % ( + self._filename, timeout)) + if self._fh: + self._fh.close() + self._fh = open(self._filename, self._fallback_mode) + return + time.sleep(delay) + + + def unlock_and_close(self): + """Close and unlock the file using the fcntl.lockf primitive.""" + if self._locked: + fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN) + self._locked = False + if self._fh: + self._fh.close() +except ImportError: + _FcntlOpener = None + + +class LockedFile(object): + """Represent a file that has exclusive access.""" + + def __init__(self, filename, mode, fallback_mode, use_fcntl=True): + """Construct a LockedFile. + + Args: + filename: string, The path of the file to open. + mode: string, The mode to try to open the file with. + fallback_mode: string, The mode to use if locking fails. + use_fcntl: string, Whether or not fcntl-based locking should be used. + """ + if not use_fcntl: + self._opener = _PosixOpener(filename, mode, fallback_mode) + else: + if _FcntlOpener: + self._opener = _FcntlOpener(filename, mode, fallback_mode) + else: + self._opener = _PosixOpener(filename, mode, fallback_mode) + + def filename(self): + """Return the filename we were constructed with.""" + return self._opener._filename + + def file_handle(self): + """Return the file_handle to the opened file.""" + return self._opener.file_handle() + + def is_locked(self): + """Return whether we successfully locked the file.""" + return self._opener.is_locked() + + def open_and_lock(self, timeout=0, delay=0.05): + """Open the file, trying to lock it. + + Args: + timeout: float, The number of seconds to try to acquire the lock. + delay: float, The number of seconds to wait between retry attempts. + + Raises: + AlreadyLockedException: if the lock is already acquired. + IOError: if the open fails. + """ + self._opener.open_and_lock(timeout, delay) + + def unlock_and_close(self): + """Unlock and close a file.""" + self._opener.unlock_and_close() diff --git a/oauth2client/multistore_file.py b/oauth2client/multistore_file.py index 1f756c7..60ac684 100644 --- a/oauth2client/multistore_file.py +++ b/oauth2client/multistore_file.py @@ -33,7 +33,6 @@ __author__ = 'jbeda@google.com (Joe Beda)' import base64 import errno -import fcntl import logging import os import threading @@ -41,6 +40,7 @@ import threading from anyjson import simplejson from client import Storage as BaseStorage from client import Credentials +from locked_file import LockedFile logger = logging.getLogger(__name__) @@ -94,9 +94,8 @@ class _MultiStore(object): This will create the file if necessary. """ - self._filename = filename + self._file = LockedFile(filename, 'r+b', 'rb') self._thread_lock = threading.Lock() - self._file_handle = None self._read_only = False self._warn_on_readonly = warn_on_readonly @@ -176,30 +175,24 @@ class _MultiStore(object): This method will not initialize the file. Instead it implements a simple version of "touch" to ensure the file has been created. """ - if not os.path.exists(self._filename): + if not os.path.exists(self._file.filename()): old_umask = os.umask(0177) try: - open(self._filename, 'a+b').close() + open(self._file.filename(), 'a+b').close() finally: os.umask(old_umask) def _lock(self): """Lock the entire multistore.""" self._thread_lock.acquire() - # Check to see if the file is writeable. - try: - self._file_handle = open(self._filename, 'r+b') - fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_EX) - except IOError, e: - if e.errno != errno.EACCES: - raise e - self._file_handle = open(self._filename, 'rb') + self._file.open_and_lock() + if not self._file.is_locked(): self._read_only = True if self._warn_on_readonly: logger.warn('The credentials file (%s) is not writable. Opening in ' 'read-only mode. Any refreshed credentials will only be ' - 'valid for this run.' % self._filename) - if os.path.getsize(self._filename) == 0: + 'valid for this run.' % self._file.filename()) + if os.path.getsize(self._file.filename()) == 0: logger.debug('Initializing empty multistore file') # The multistore is empty so write out an empty file. self._data = {} @@ -214,9 +207,7 @@ class _MultiStore(object): def _unlock(self): """Release the lock on the multistore.""" - if not self._read_only: - fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_UN) - self._file_handle.close() + self._file.unlock_and_close() self._thread_lock.release() def _locked_json_read(self): @@ -228,8 +219,8 @@ class _MultiStore(object): The contents of the multistore decoded as JSON. """ assert self._thread_lock.locked() - self._file_handle.seek(0) - return simplejson.load(self._file_handle) + self._file.file_handle().seek(0) + return simplejson.load(self._file.file_handle()) def _locked_json_write(self, data): """Write a JSON serializable data structure to the multistore. @@ -242,9 +233,9 @@ class _MultiStore(object): assert self._thread_lock.locked() if self._read_only: return - self._file_handle.seek(0) - simplejson.dump(data, self._file_handle, sort_keys=True, indent=2) - self._file_handle.truncate() + self._file.file_handle().seek(0) + simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2) + self._file.file_handle().truncate() def _refresh_data_cache(self): """Refresh the contents of the multistore.