Make oauth2client support Windows-friendly locking.

Reviewed in http://codereview.appspot.com/6265043/.

Fixes issue #138.

Index: oauth2client/locked_file.py
===================================================================
new file mode 100644

oauth2client/locked_file.py

@@ -0,0 +1,254 @@
+# Copyright 2011 Google Inc. All Rights Reserved.
+
+"""Locked file interface that should work on Unix and Windows pythons.
+
+This module first tries to use fcntl locking to ensure serialized access
+to a file, then falls back on a lock file if that is unavialable.
+
+Usage:
+    f = LockedFile('filename', 'r+b', 'rb')
+    f.open_and_lock()
+    if f.is_locked():
+      print 'Acquired filename with r+b mode'
+      f.file_handle().write('locked data')
+    else:
+      print 'Aquired filename with rb mode'
+    f.unlock_and_close()
+"""
+
+__author__ = 'cache@google.com (David T McWherter)'
+
+import errno
+import logging
+import os
+import time
+
+logger = logging.getLogger(__name__)
+
+
+class AlreadyLockedException(Exception):
+  """Trying to lock a file that has already been locked by the LockedFile."""
+  pass
+
+
+class _Opener(object):
+  """Base class for different locking primitives."""
+
+  def __init__(self, filename, mode, fallback_mode):
+    """Create an Opener.
+
+    Args:
+      filename: string, The pathname of the file.
+      mode: string, The preferred mode to access the file with.
+      fallback_mode: string, The mode to use if locking fails.
+    """
+    self._locked = False
+    self._filename = filename
+    self._mode = mode
+    self._fallback_mode = fallback_mode
+    self._fh = None
+
+  def is_locked(self):
+    """Was the file locked."""
+    return self._locked
+
+  def file_handle(self):
+    """The file handle to the file.  Valid only after opened."""
+    return self._fh
+
+  def filename(self):
+    """The filename that is being locked."""
+    return self._filename
+
+  def open_and_lock(self, timeout, delay):
+    """Open the file and lock it.
+
+    Args:
+      timeout: float, How long to try to lock for.
+      delay: float, How long to wait between retries.
+    """
+    pass
+
+  def unlock_and_close(self):
+    """Unlock and close the file."""
+    pass
+
+
+class _PosixOpener(_Opener):
+  """Lock files using Posix advisory lock files."""
+
+  def open_and_lock(self, timeout, delay):
+    """Open the file and lock it.
+
+    Tries to create a .lock file next to the file we're trying to open.
+
+    Args:
+      timeout: float, How long to try to lock for.
+      delay: float, How long to wait between retries.
+
+    Raises:
+      AlreadyLockedException: if the lock is already acquired.
+      IOError: if the open fails.
+    """
+    if self._locked:
+      raise AlreadyLockedException('File %s is already locked' %
+                                   self._filename)
+    self._locked = False
+
+    try:
+      self._fh = open(self._filename, self._mode)
+    except IOError, e:
+      # If we can't access with _mode, try _fallback_mode and don't lock.
+      if e.errno == errno.EACCES:
+        self._fh = open(self._filename, self._fallback_mode)
+        return
+
+    lock_filename = self._posix_lockfile(self._filename)
+    start_time = time.time()
+    while True:
+      try:
+        self._lock_fd = os.open(lock_filename,
+                                os.O_CREAT|os.O_EXCL|os.O_RDWR)
+        self._locked = True
+        break
+
+      except OSError, e:
+        if e.errno != errno.EEXIST:
+          raise
+        if (time.time() - start_time) >= timeout:
+          logger.warn('Could not acquire lock %s in %s seconds' % (
+              lock_filename, timeout))
+          # Close the file and open in fallback_mode.
+          if self._fh:
+            self._fh.close()
+          self._fh = open(self._filename, self._fallback_mode)
+          return
+        time.sleep(delay)
+
+  def unlock_and_close(self):
+    """Unlock a file by removing the .lock file, and close the handle."""
+    if self._locked:
+      lock_filename = self._posix_lockfile(self._filename)
+      os.unlink(lock_filename)
+      os.close(self._lock_fd)
+      self._locked = False
+      self._lock_fd = None
+    if self._fh:
+      self._fh.close()
+
+  def _posix_lockfile(self, filename):
+    """The name of the lock file to use for posix locking."""
+    return '%s.lock' % filename
+
+
+try:
+  import fcntl
+  class _FcntlOpener(_Opener):
+    """Open, lock, and unlock a file using fcntl.lockf."""
+
+    def open_and_lock(self, timeout, delay):
+      """Open the file and lock it.
+
+      Args:
+        timeout: float, How long to try to lock for.
+        delay: float, How long to wait between retries
+
+      Raises:
+        AlreadyLockedException: if the lock is already acquired.
+        IOError: if the open fails.
+      """
+      if self._locked:
+        raise AlreadyLockedException('File %s is already locked' %
+                                     self._filename)
+      start_time = time.time()
+
+      try:
+        self._fh = open(self._filename, self._mode)
+      except IOError, e:
+        # If we can't access with _mode, try _fallback_mode and don't lock.
+        if e.errno == errno.EACCES:
+          self._fh = open(self._filename, self._fallback_mode)
+          return
+
+      # We opened in _mode, try to lock the file.
+      while True:
+        try:
+          fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
+          self._locked = True
+          return
+        except IOError, e:
+          # If not retrying, then just pass on the error.
+          if timeout == 0:
+            raise e
+          if e.errno != errno.EACCES:
+            raise e
+          # We could not acquire the lock.  Try again.
+          if (time.time() - start_time) >= timeout:
+            logger.warn('Could not lock %s in %s seconds' % (
+                self._filename, timeout))
+            if self._fh:
+              self._fh.close()
+            self._fh = open(self._filename, self._fallback_mode)
+            return
+          time.sleep(delay)
+
+
+    def unlock_and_close(self):
+      """Close and unlock the file using the fcntl.lockf primitive."""
+      if self._locked:
+        fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
+      self._locked = False
+      if self._fh:
+        self._fh.close()
+except ImportError:
+  _FcntlOpener = None
+
+
+class LockedFile(object):
+  """Represent a file that has exclusive access."""
+
+  def __init__(self, filename, mode, fallback_mode, use_fcntl=True):
+    """Construct a LockedFile.
+
+    Args:
+      filename: string, The path of the file to open.
+      mode: string, The mode to try to open the file with.
+      fallback_mode: string, The mode to use if locking fails.
+      use_fcntl: string, Whether or not fcntl-based locking should be used.
+    """
+    if not use_fcntl:
+      self._opener = _PosixOpener(filename, mode, fallback_mode)
+    else:
+      if _FcntlOpener:
+        self._opener = _FcntlOpener(filename, mode, fallback_mode)
+      else:
+        self._opener = _PosixOpener(filename, mode, fallback_mode)
+
+  def filename(self):
+    """Return the filename we were constructed with."""
+    return self._opener._filename
+
+  def file_handle(self):
+    """Return the file_handle to the opened file."""
+    return self._opener.file_handle()
+
+  def is_locked(self):
+    """Return whether we successfully locked the file."""
+    return self._opener.is_locked()
+
+  def open_and_lock(self, timeout=0, delay=0.05):
+    """Open the file, trying to lock it.
+
+    Args:
+      timeout: float, The number of seconds to try to acquire the lock.
+      delay: float, The number of seconds to wait between retry attempts.
+
+    Raises:
+      AlreadyLockedException: if the lock is already acquired.
+      IOError: if the open fails.
+    """
+    self._opener.open_and_lock(timeout, delay)
+
+  def unlock_and_close(self):
+    """Unlock and close a file."""
+    self._opener.unlock_and_close()

oauth2client/multistore_file.py

@@ -33,7 +33,6 @@ __author__ = 'jbeda@google.com (Joe Beda)'
 
 import base64
 import errno
-import fcntl
 import logging
 import os
 import threading
@@ -41,6 +40,7 @@ import threading
 from anyjson import simplejson
 from client import Storage as BaseStorage
 from client import Credentials
+from locked_file import LockedFile
 
 logger = logging.getLogger(__name__)
 
@@ -94,9 +94,8 @@ class _MultiStore(object):
 
     This will create the file if necessary.
     """
-    self._filename = filename
+    self._file = LockedFile(filename, 'r+b', 'rb')
     self._thread_lock = threading.Lock()
-    self._file_handle = None
     self._read_only = False
     self._warn_on_readonly = warn_on_readonly
 
@@ -176,30 +175,24 @@ class _MultiStore(object):
     This method will not initialize the file. Instead it implements a
     simple version of "touch" to ensure the file has been created.
     """
-    if not os.path.exists(self._filename):
+    if not os.path.exists(self._file.filename()):
       old_umask = os.umask(0177)
       try:
-        open(self._filename, 'a+b').close()
+        open(self._file.filename(), 'a+b').close()
       finally:
         os.umask(old_umask)
 
   def _lock(self):
     """Lock the entire multistore."""
     self._thread_lock.acquire()
-    # Check to see if the file is writeable.
+    self._file.open_and_lock()
-    try:
+    if not self._file.is_locked():
-      self._file_handle = open(self._filename, 'r+b')
-      fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_EX)
-    except IOError, e:
-      if e.errno != errno.EACCES:
-        raise e
-      self._file_handle = open(self._filename, 'rb')
       self._read_only = True
       if self._warn_on_readonly:
         logger.warn('The credentials file (%s) is not writable. Opening in '
                     'read-only mode. Any refreshed credentials will only be '
-                    'valid for this run.' % self._filename)
+                    'valid for this run.' % self._file.filename())
-    if os.path.getsize(self._filename) == 0:
+    if os.path.getsize(self._file.filename()) == 0:
       logger.debug('Initializing empty multistore file')
       # The multistore is empty so write out an empty file.
       self._data = {}
@@ -214,9 +207,7 @@ class _MultiStore(object):
 
   def _unlock(self):
     """Release the lock on the multistore."""
-    if not self._read_only:
+    self._file.unlock_and_close()
-      fcntl.lockf(self._file_handle.fileno(), fcntl.LOCK_UN)
-    self._file_handle.close()
     self._thread_lock.release()
 
   def _locked_json_read(self):
@@ -228,8 +219,8 @@ class _MultiStore(object):
       The contents of the multistore decoded as JSON.
     """
     assert self._thread_lock.locked()
-    self._file_handle.seek(0)
+    self._file.file_handle().seek(0)
-    return simplejson.load(self._file_handle)
+    return simplejson.load(self._file.file_handle())
 
   def _locked_json_write(self, data):
     """Write a JSON serializable data structure to the multistore.
@@ -242,9 +233,9 @@ class _MultiStore(object):
     assert self._thread_lock.locked()
     if self._read_only:
       return
-    self._file_handle.seek(0)
+    self._file.file_handle().seek(0)
-    simplejson.dump(data, self._file_handle, sort_keys=True, indent=2)
+    simplejson.dump(data, self._file.file_handle(), sort_keys=True, indent=2)
-    self._file_handle.truncate()
+    self._file.file_handle().truncate()
 
   def _refresh_data_cache(self):
     """Refresh the contents of the multistore.