Remove contrib.multistore_file (#589)
Remove dependent modules as well. Resolves #470.
This commit is contained in:
Jon Wayne Parrott
committed by
GitHub
GitHub
parent
f04d5213d4
commit
0dc30bc033
@@ -1,7 +0,0 @@
|
||||
oauth2client.contrib.locked_file module
|
||||
=======================================
|
||||
|
||||
.. automodule:: oauth2client.contrib.locked_file
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -1,7 +0,0 @@
|
||||
oauth2client.contrib.multistore_file module
|
||||
===========================================
|
||||
|
||||
.. automodule:: oauth2client.contrib.multistore_file
|
||||
:members:
|
||||
:undoc-members:
|
||||
:show-inheritance:
|
||||
@@ -19,9 +19,7 @@ Submodules
|
||||
oauth2client.contrib.flask_util
|
||||
oauth2client.contrib.gce
|
||||
oauth2client.contrib.keyring_storage
|
||||
oauth2client.contrib.locked_file
|
||||
oauth2client.contrib.multiprocess_file_storage
|
||||
oauth2client.contrib.multistore_file
|
||||
oauth2client.contrib.sqlalchemy
|
||||
oauth2client.contrib.xsrfutil
|
||||
|
||||
|
||||
@@ -1,81 +0,0 @@
|
||||
# Copyright 2016 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import fcntl
|
||||
import time
|
||||
|
||||
from oauth2client import util
|
||||
from oauth2client.contrib import locked_file
|
||||
|
||||
|
||||
class _FcntlOpener(locked_file._Opener):
|
||||
"""Open, lock, and unlock a file using fcntl.lockf."""
|
||||
|
||||
def open_and_lock(self, timeout, delay):
|
||||
"""Open the file and lock it.
|
||||
|
||||
Args:
|
||||
timeout: float, How long to try to lock for.
|
||||
delay: float, How long to wait between retries
|
||||
|
||||
Raises:
|
||||
AlreadyLockedException: if the lock is already acquired.
|
||||
IOError: if the open fails.
|
||||
IOError: if the file is a symbolic link.
|
||||
"""
|
||||
if self._locked:
|
||||
raise locked_file.AlreadyLockedException(
|
||||
'File {0} is already locked'.format(self._filename))
|
||||
start_time = time.time()
|
||||
|
||||
util.validate_file(self._filename)
|
||||
try:
|
||||
self._fh = open(self._filename, self._mode)
|
||||
except IOError as e:
|
||||
# If we can't access with _mode, try _fallback_mode and
|
||||
# don't lock.
|
||||
if e.errno in (errno.EPERM, errno.EACCES):
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
|
||||
# We opened in _mode, try to lock the file.
|
||||
while True:
|
||||
try:
|
||||
fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
|
||||
self._locked = True
|
||||
return
|
||||
except IOError as e:
|
||||
# If not retrying, then just pass on the error.
|
||||
if timeout == 0:
|
||||
raise
|
||||
if e.errno != errno.EACCES:
|
||||
raise
|
||||
# We could not acquire the lock. Try again.
|
||||
if (time.time() - start_time) >= timeout:
|
||||
locked_file.logger.warn('Could not lock %s in %s seconds',
|
||||
self._filename, timeout)
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
time.sleep(delay)
|
||||
|
||||
def unlock_and_close(self):
|
||||
"""Close and unlock the file using the fcntl.lockf primitive."""
|
||||
if self._locked:
|
||||
fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
|
||||
self._locked = False
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
@@ -1,106 +0,0 @@
|
||||
# Copyright 2016 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import time
|
||||
|
||||
import pywintypes
|
||||
import win32con
|
||||
import win32file
|
||||
|
||||
from oauth2client import util
|
||||
from oauth2client.contrib import locked_file
|
||||
|
||||
|
||||
class _Win32Opener(locked_file._Opener):
|
||||
"""Open, lock, and unlock a file using windows primitives."""
|
||||
|
||||
# Error #33:
|
||||
# 'The process cannot access the file because another process'
|
||||
FILE_IN_USE_ERROR = 33
|
||||
|
||||
# Error #158:
|
||||
# 'The segment is already unlocked.'
|
||||
FILE_ALREADY_UNLOCKED_ERROR = 158
|
||||
|
||||
def open_and_lock(self, timeout, delay):
|
||||
"""Open the file and lock it.
|
||||
|
||||
Args:
|
||||
timeout: float, How long to try to lock for.
|
||||
delay: float, How long to wait between retries
|
||||
|
||||
Raises:
|
||||
AlreadyLockedException: if the lock is already acquired.
|
||||
IOError: if the open fails.
|
||||
IOError: if the file is a symbolic link.
|
||||
"""
|
||||
if self._locked:
|
||||
raise locked_file.AlreadyLockedException(
|
||||
'File {0} is already locked'.format(self._filename))
|
||||
start_time = time.time()
|
||||
|
||||
util.validate_file(self._filename)
|
||||
try:
|
||||
self._fh = open(self._filename, self._mode)
|
||||
except IOError as e:
|
||||
# If we can't access with _mode, try _fallback_mode
|
||||
# and don't lock.
|
||||
if e.errno == errno.EACCES:
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
|
||||
# We opened in _mode, try to lock the file.
|
||||
while True:
|
||||
try:
|
||||
hfile = win32file._get_osfhandle(self._fh.fileno())
|
||||
win32file.LockFileEx(
|
||||
hfile,
|
||||
(win32con.LOCKFILE_FAIL_IMMEDIATELY |
|
||||
win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
|
||||
pywintypes.OVERLAPPED())
|
||||
self._locked = True
|
||||
return
|
||||
except pywintypes.error as e:
|
||||
if timeout == 0:
|
||||
raise
|
||||
|
||||
# If the error is not that the file is already
|
||||
# in use, raise.
|
||||
if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
|
||||
raise
|
||||
|
||||
# We could not acquire the lock. Try again.
|
||||
if (time.time() - start_time) >= timeout:
|
||||
locked_file.logger.warn('Could not lock %s in %s seconds',
|
||||
self._filename, timeout)
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
time.sleep(delay)
|
||||
|
||||
def unlock_and_close(self):
|
||||
"""Close and unlock the file using the win32 primitive."""
|
||||
if self._locked:
|
||||
try:
|
||||
hfile = win32file._get_osfhandle(self._fh.fileno())
|
||||
win32file.UnlockFileEx(hfile, 0, -0x10000,
|
||||
pywintypes.OVERLAPPED())
|
||||
except pywintypes.error as e:
|
||||
if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
|
||||
raise
|
||||
self._locked = False
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
@@ -1,224 +0,0 @@
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Locked file interface that should work on Unix and Windows pythons.
|
||||
|
||||
This module first tries to use fcntl locking to ensure serialized access
|
||||
to a file, then falls back on a lock file if that is unavialable.
|
||||
|
||||
Usage::
|
||||
|
||||
f = LockedFile('filename', 'r+b', 'rb')
|
||||
f.open_and_lock()
|
||||
if f.is_locked():
|
||||
print('Acquired filename with r+b mode')
|
||||
f.file_handle().write('locked data')
|
||||
else:
|
||||
print('Acquired filename with rb mode')
|
||||
f.unlock_and_close()
|
||||
|
||||
"""
|
||||
|
||||
from __future__ import print_function
|
||||
|
||||
import errno
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
|
||||
from oauth2client import util
|
||||
|
||||
|
||||
__author__ = 'cache@google.com (David T McWherter)'
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class AlreadyLockedException(Exception):
|
||||
"""Trying to lock a file that has already been locked by the LockedFile."""
|
||||
pass
|
||||
|
||||
|
||||
class _Opener(object):
|
||||
"""Base class for different locking primitives."""
|
||||
|
||||
def __init__(self, filename, mode, fallback_mode):
|
||||
"""Create an Opener.
|
||||
|
||||
Args:
|
||||
filename: string, The pathname of the file.
|
||||
mode: string, The preferred mode to access the file with.
|
||||
fallback_mode: string, The mode to use if locking fails.
|
||||
"""
|
||||
self._locked = False
|
||||
self._filename = filename
|
||||
self._mode = mode
|
||||
self._fallback_mode = fallback_mode
|
||||
self._fh = None
|
||||
self._lock_fd = None
|
||||
|
||||
def is_locked(self):
|
||||
"""Was the file locked."""
|
||||
return self._locked
|
||||
|
||||
def file_handle(self):
|
||||
"""The file handle to the file. Valid only after opened."""
|
||||
return self._fh
|
||||
|
||||
def filename(self):
|
||||
"""The filename that is being locked."""
|
||||
return self._filename
|
||||
|
||||
def open_and_lock(self, timeout, delay):
|
||||
"""Open the file and lock it.
|
||||
|
||||
Args:
|
||||
timeout: float, How long to try to lock for.
|
||||
delay: float, How long to wait between retries.
|
||||
"""
|
||||
pass
|
||||
|
||||
def unlock_and_close(self):
|
||||
"""Unlock and close the file."""
|
||||
pass
|
||||
|
||||
|
||||
class _PosixOpener(_Opener):
|
||||
"""Lock files using Posix advisory lock files."""
|
||||
|
||||
def open_and_lock(self, timeout, delay):
|
||||
"""Open the file and lock it.
|
||||
|
||||
Tries to create a .lock file next to the file we're trying to open.
|
||||
|
||||
Args:
|
||||
timeout: float, How long to try to lock for.
|
||||
delay: float, How long to wait between retries.
|
||||
|
||||
Raises:
|
||||
AlreadyLockedException: if the lock is already acquired.
|
||||
IOError: if the open fails.
|
||||
IOError: if the file is a symbolic link.
|
||||
"""
|
||||
if self._locked:
|
||||
raise AlreadyLockedException(
|
||||
'File {0} is already locked'.format(self._filename))
|
||||
self._locked = False
|
||||
|
||||
util.validate_file(self._filename)
|
||||
try:
|
||||
self._fh = open(self._filename, self._mode)
|
||||
except IOError as e:
|
||||
# If we can't access with _mode, try _fallback_mode and don't lock.
|
||||
if e.errno == errno.EACCES:
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
|
||||
lock_filename = self._posix_lockfile(self._filename)
|
||||
start_time = time.time()
|
||||
while True:
|
||||
try:
|
||||
self._lock_fd = os.open(lock_filename,
|
||||
os.O_CREAT | os.O_EXCL | os.O_RDWR)
|
||||
self._locked = True
|
||||
break
|
||||
|
||||
except OSError as e:
|
||||
if e.errno != errno.EEXIST:
|
||||
raise
|
||||
if (time.time() - start_time) >= timeout:
|
||||
logger.warn('Could not acquire lock %s in %s seconds',
|
||||
lock_filename, timeout)
|
||||
# Close the file and open in fallback_mode.
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
self._fh = open(self._filename, self._fallback_mode)
|
||||
return
|
||||
time.sleep(delay)
|
||||
|
||||
def unlock_and_close(self):
|
||||
"""Unlock a file by removing the .lock file, and close the handle."""
|
||||
if self._locked:
|
||||
lock_filename = self._posix_lockfile(self._filename)
|
||||
os.close(self._lock_fd)
|
||||
os.unlink(lock_filename)
|
||||
self._locked = False
|
||||
self._lock_fd = None
|
||||
if self._fh:
|
||||
self._fh.close()
|
||||
|
||||
def _posix_lockfile(self, filename):
|
||||
"""The name of the lock file to use for posix locking."""
|
||||
return '{0}.lock'.format(filename)
|
||||
|
||||
|
||||
class LockedFile(object):
|
||||
"""Represent a file that has exclusive access."""
|
||||
|
||||
@util.positional(4)
|
||||
def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
|
||||
"""Construct a LockedFile.
|
||||
|
||||
Args:
|
||||
filename: string, The path of the file to open.
|
||||
mode: string, The mode to try to open the file with.
|
||||
fallback_mode: string, The mode to use if locking fails.
|
||||
use_native_locking: bool, Whether or not fcntl/win32 locking is
|
||||
used.
|
||||
"""
|
||||
opener = None
|
||||
if not opener and use_native_locking:
|
||||
try:
|
||||
from oauth2client.contrib._win32_opener import _Win32Opener
|
||||
opener = _Win32Opener(filename, mode, fallback_mode)
|
||||
except ImportError:
|
||||
try:
|
||||
from oauth2client.contrib._fcntl_opener import _FcntlOpener
|
||||
opener = _FcntlOpener(filename, mode, fallback_mode)
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
if not opener:
|
||||
opener = _PosixOpener(filename, mode, fallback_mode)
|
||||
|
||||
self._opener = opener
|
||||
|
||||
def filename(self):
|
||||
"""Return the filename we were constructed with."""
|
||||
return self._opener._filename
|
||||
|
||||
def file_handle(self):
|
||||
"""Return the file_handle to the opened file."""
|
||||
return self._opener.file_handle()
|
||||
|
||||
def is_locked(self):
|
||||
"""Return whether we successfully locked the file."""
|
||||
return self._opener.is_locked()
|
||||
|
||||
def open_and_lock(self, timeout=0, delay=0.05):
|
||||
"""Open the file, trying to lock it.
|
||||
|
||||
Args:
|
||||
timeout: float, The number of seconds to try to acquire the lock.
|
||||
delay: float, The number of seconds to wait between retry attempts.
|
||||
|
||||
Raises:
|
||||
AlreadyLockedException: if the lock is already acquired.
|
||||
IOError: if the open fails.
|
||||
"""
|
||||
self._opener.open_and_lock(timeout, delay)
|
||||
|
||||
def unlock_and_close(self):
|
||||
"""Unlock and close a file."""
|
||||
self._opener.unlock_and_close()
|
||||
@@ -1,505 +0,0 @@
|
||||
# Copyright 2014 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Multi-credential file store with lock support.
|
||||
|
||||
This module implements a JSON credential store where multiple
|
||||
credentials can be stored in one file. That file supports locking
|
||||
both in a single process and across processes.
|
||||
|
||||
The credential themselves are keyed off of:
|
||||
|
||||
* client_id
|
||||
* user_agent
|
||||
* scope
|
||||
|
||||
The format of the stored data is like so::
|
||||
|
||||
{
|
||||
'file_version': 1,
|
||||
'data': [
|
||||
{
|
||||
'key': {
|
||||
'clientId': '<client id>',
|
||||
'userAgent': '<user agent>',
|
||||
'scope': '<scope>'
|
||||
},
|
||||
'credential': {
|
||||
# JSON serialized Credentials.
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
"""
|
||||
|
||||
import errno
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import threading
|
||||
|
||||
from oauth2client import client
|
||||
from oauth2client import util
|
||||
from oauth2client.contrib import locked_file
|
||||
|
||||
__author__ = 'jbeda@google.com (Joe Beda)'
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
logger.warning(
|
||||
'The oauth2client.contrib.multistore_file module has been deprecated and '
|
||||
'will be removed in the next release of oauth2client. Please migrate to '
|
||||
'multiprocess_file_storage.')
|
||||
|
||||
# A dict from 'filename'->_MultiStore instances
|
||||
_multistores = {}
|
||||
_multistores_lock = threading.Lock()
|
||||
|
||||
|
||||
class Error(Exception):
|
||||
"""Base error for this module."""
|
||||
|
||||
|
||||
class NewerCredentialStoreError(Error):
|
||||
"""The credential store is a newer version than supported."""
|
||||
|
||||
|
||||
def _dict_to_tuple_key(dictionary):
|
||||
"""Converts a dictionary to a tuple that can be used as an immutable key.
|
||||
|
||||
The resulting key is always sorted so that logically equivalent
|
||||
dictionaries always produce an identical tuple for a key.
|
||||
|
||||
Args:
|
||||
dictionary: the dictionary to use as the key.
|
||||
|
||||
Returns:
|
||||
A tuple representing the dictionary in it's naturally sorted ordering.
|
||||
"""
|
||||
return tuple(sorted(dictionary.items()))
|
||||
|
||||
|
||||
@util.positional(4)
|
||||
def get_credential_storage(filename, client_id, user_agent, scope,
|
||||
warn_on_readonly=True):
|
||||
"""Get a Storage instance for a credential.
|
||||
|
||||
Args:
|
||||
filename: The JSON file storing a set of credentials
|
||||
client_id: The client_id for the credential
|
||||
user_agent: The user agent for the credential
|
||||
scope: string or iterable of strings, Scope(s) being requested
|
||||
warn_on_readonly: if True, log a warning if the store is readonly
|
||||
|
||||
Returns:
|
||||
An object derived from client.Storage for getting/setting the
|
||||
credential.
|
||||
"""
|
||||
# Recreate the legacy key with these specific parameters
|
||||
key = {'clientId': client_id, 'userAgent': user_agent,
|
||||
'scope': util.scopes_to_string(scope)}
|
||||
return get_credential_storage_custom_key(
|
||||
filename, key, warn_on_readonly=warn_on_readonly)
|
||||
|
||||
|
||||
@util.positional(2)
|
||||
def get_credential_storage_custom_string_key(filename, key_string,
|
||||
warn_on_readonly=True):
|
||||
"""Get a Storage instance for a credential using a single string as a key.
|
||||
|
||||
Allows you to provide a string as a custom key that will be used for
|
||||
credential storage and retrieval.
|
||||
|
||||
Args:
|
||||
filename: The JSON file storing a set of credentials
|
||||
key_string: A string to use as the key for storing this credential.
|
||||
warn_on_readonly: if True, log a warning if the store is readonly
|
||||
|
||||
Returns:
|
||||
An object derived from client.Storage for getting/setting the
|
||||
credential.
|
||||
"""
|
||||
# Create a key dictionary that can be used
|
||||
key_dict = {'key': key_string}
|
||||
return get_credential_storage_custom_key(
|
||||
filename, key_dict, warn_on_readonly=warn_on_readonly)
|
||||
|
||||
|
||||
@util.positional(2)
|
||||
def get_credential_storage_custom_key(filename, key_dict,
|
||||
warn_on_readonly=True):
|
||||
"""Get a Storage instance for a credential using a dictionary as a key.
|
||||
|
||||
Allows you to provide a dictionary as a custom key that will be used for
|
||||
credential storage and retrieval.
|
||||
|
||||
Args:
|
||||
filename: The JSON file storing a set of credentials
|
||||
key_dict: A dictionary to use as the key for storing this credential.
|
||||
There is no ordering of the keys in the dictionary. Logically
|
||||
equivalent dictionaries will produce equivalent storage keys.
|
||||
warn_on_readonly: if True, log a warning if the store is readonly
|
||||
|
||||
Returns:
|
||||
An object derived from client.Storage for getting/setting the
|
||||
credential.
|
||||
"""
|
||||
multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly)
|
||||
key = _dict_to_tuple_key(key_dict)
|
||||
return multistore._get_storage(key)
|
||||
|
||||
|
||||
@util.positional(1)
|
||||
def get_all_credential_keys(filename, warn_on_readonly=True):
|
||||
"""Gets all the registered credential keys in the given Multistore.
|
||||
|
||||
Args:
|
||||
filename: The JSON file storing a set of credentials
|
||||
warn_on_readonly: if True, log a warning if the store is readonly
|
||||
|
||||
Returns:
|
||||
A list of the credential keys present in the file. They are returned
|
||||
as dictionaries that can be passed into
|
||||
get_credential_storage_custom_key to get the actual credentials.
|
||||
"""
|
||||
multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly)
|
||||
multistore._lock()
|
||||
try:
|
||||
return multistore._get_all_credential_keys()
|
||||
finally:
|
||||
multistore._unlock()
|
||||
|
||||
|
||||
@util.positional(1)
|
||||
def _get_multistore(filename, warn_on_readonly=True):
|
||||
"""A helper method to initialize the multistore with proper locking.
|
||||
|
||||
Args:
|
||||
filename: The JSON file storing a set of credentials
|
||||
warn_on_readonly: if True, log a warning if the store is readonly
|
||||
|
||||
Returns:
|
||||
A multistore object
|
||||
"""
|
||||
filename = os.path.expanduser(filename)
|
||||
_multistores_lock.acquire()
|
||||
try:
|
||||
multistore = _multistores.setdefault(
|
||||
filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly))
|
||||
finally:
|
||||
_multistores_lock.release()
|
||||
return multistore
|
||||
|
||||
|
||||
class _MultiStore(object):
|
||||
"""A file backed store for multiple credentials."""
|
||||
|
||||
@util.positional(2)
|
||||
def __init__(self, filename, warn_on_readonly=True):
|
||||
"""Initialize the class.
|
||||
|
||||
This will create the file if necessary.
|
||||
"""
|
||||
self._file = locked_file.LockedFile(filename, 'r+', 'r')
|
||||
self._thread_lock = threading.Lock()
|
||||
self._read_only = False
|
||||
self._warn_on_readonly = warn_on_readonly
|
||||
|
||||
self._create_file_if_needed()
|
||||
|
||||
# Cache of deserialized store. This is only valid after the
|
||||
# _MultiStore is locked or _refresh_data_cache is called. This is
|
||||
# of the form of:
|
||||
#
|
||||
# ((key, value), (key, value)...) -> OAuth2Credential
|
||||
#
|
||||
# If this is None, then the store hasn't been read yet.
|
||||
self._data = None
|
||||
|
||||
class _Storage(client.Storage):
|
||||
"""A Storage object that can read/write a single credential."""
|
||||
|
||||
def __init__(self, multistore, key):
|
||||
self._multistore = multistore
|
||||
self._key = key
|
||||
|
||||
def acquire_lock(self):
|
||||
"""Acquires any lock necessary to access this Storage.
|
||||
|
||||
This lock is not reentrant.
|
||||
"""
|
||||
self._multistore._lock()
|
||||
|
||||
def release_lock(self):
|
||||
"""Release the Storage lock.
|
||||
|
||||
Trying to release a lock that isn't held will result in a
|
||||
RuntimeError.
|
||||
"""
|
||||
self._multistore._unlock()
|
||||
|
||||
def locked_get(self):
|
||||
"""Retrieve credential.
|
||||
|
||||
The Storage lock must be held when this is called.
|
||||
|
||||
Returns:
|
||||
oauth2client.client.Credentials
|
||||
"""
|
||||
credential = self._multistore._get_credential(self._key)
|
||||
if credential:
|
||||
credential.set_store(self)
|
||||
return credential
|
||||
|
||||
def locked_put(self, credentials):
|
||||
"""Write a credential.
|
||||
|
||||
The Storage lock must be held when this is called.
|
||||
|
||||
Args:
|
||||
credentials: Credentials, the credentials to store.
|
||||
"""
|
||||
self._multistore._update_credential(self._key, credentials)
|
||||
|
||||
def locked_delete(self):
|
||||
"""Delete a credential.
|
||||
|
||||
The Storage lock must be held when this is called.
|
||||
|
||||
Args:
|
||||
credentials: Credentials, the credentials to store.
|
||||
"""
|
||||
self._multistore._delete_credential(self._key)
|
||||
|
||||
def _create_file_if_needed(self):
|
||||
"""Create an empty file if necessary.
|
||||
|
||||
This method will not initialize the file. Instead it implements a
|
||||
simple version of "touch" to ensure the file has been created.
|
||||
"""
|
||||
if not os.path.exists(self._file.filename()):
|
||||
old_umask = os.umask(0o177)
|
||||
try:
|
||||
open(self._file.filename(), 'a+b').close()
|
||||
finally:
|
||||
os.umask(old_umask)
|
||||
|
||||
def _lock(self):
|
||||
"""Lock the entire multistore."""
|
||||
self._thread_lock.acquire()
|
||||
try:
|
||||
self._file.open_and_lock()
|
||||
except (IOError, OSError) as e:
|
||||
if e.errno == errno.ENOSYS:
|
||||
logger.warn('File system does not support locking the '
|
||||
'credentials file.')
|
||||
elif e.errno == errno.ENOLCK:
|
||||
logger.warn('File system is out of resources for writing the '
|
||||
'credentials file (is your disk full?).')
|
||||
elif e.errno == errno.EDEADLK:
|
||||
logger.warn('Lock contention on multistore file, opening '
|
||||
'in read-only mode.')
|
||||
elif e.errno == errno.EACCES:
|
||||
logger.warn('Cannot access credentials file.')
|
||||
else:
|
||||
raise
|
||||
if not self._file.is_locked():
|
||||
self._read_only = True
|
||||
if self._warn_on_readonly:
|
||||
logger.warn('The credentials file (%s) is not writable. '
|
||||
'Opening in read-only mode. Any refreshed '
|
||||
'credentials will only be '
|
||||
'valid for this run.', self._file.filename())
|
||||
|
||||
if os.path.getsize(self._file.filename()) == 0:
|
||||
logger.debug('Initializing empty multistore file')
|
||||
# The multistore is empty so write out an empty file.
|
||||
self._data = {}
|
||||
self._write()
|
||||
elif not self._read_only or self._data is None:
|
||||
# Only refresh the data if we are read/write or we haven't
|
||||
# cached the data yet. If we are readonly, we assume is isn't
|
||||
# changing out from under us and that we only have to read it
|
||||
# once. This prevents us from whacking any new access keys that
|
||||
# we have cached in memory but were unable to write out.
|
||||
self._refresh_data_cache()
|
||||
|
||||
def _unlock(self):
|
||||
"""Release the lock on the multistore."""
|
||||
self._file.unlock_and_close()
|
||||
self._thread_lock.release()
|
||||
|
||||
def _locked_json_read(self):
|
||||
"""Get the raw content of the multistore file.
|
||||
|
||||
The multistore must be locked when this is called.
|
||||
|
||||
Returns:
|
||||
The contents of the multistore decoded as JSON.
|
||||
"""
|
||||
assert self._thread_lock.locked()
|
||||
self._file.file_handle().seek(0)
|
||||
return json.load(self._file.file_handle())
|
||||
|
||||
def _locked_json_write(self, data):
|
||||
"""Write a JSON serializable data structure to the multistore.
|
||||
|
||||
The multistore must be locked when this is called.
|
||||
|
||||
Args:
|
||||
data: The data to be serialized and written.
|
||||
"""
|
||||
assert self._thread_lock.locked()
|
||||
if self._read_only:
|
||||
return
|
||||
self._file.file_handle().seek(0)
|
||||
json.dump(data, self._file.file_handle(),
|
||||
sort_keys=True, indent=2, separators=(',', ': '))
|
||||
self._file.file_handle().truncate()
|
||||
|
||||
def _refresh_data_cache(self):
|
||||
"""Refresh the contents of the multistore.
|
||||
|
||||
The multistore must be locked when this is called.
|
||||
|
||||
Raises:
|
||||
NewerCredentialStoreError: Raised when a newer client has written
|
||||
the store.
|
||||
"""
|
||||
self._data = {}
|
||||
try:
|
||||
raw_data = self._locked_json_read()
|
||||
except Exception:
|
||||
logger.warn('Credential data store could not be loaded. '
|
||||
'Will ignore and overwrite.')
|
||||
return
|
||||
|
||||
version = 0
|
||||
try:
|
||||
version = raw_data['file_version']
|
||||
except Exception:
|
||||
logger.warn('Missing version for credential data store. It may be '
|
||||
'corrupt or an old version. Overwriting.')
|
||||
if version > 1:
|
||||
raise NewerCredentialStoreError(
|
||||
'Credential file has file_version of {0}. '
|
||||
'Only file_version of 1 is supported.'.format(version))
|
||||
|
||||
credentials = []
|
||||
try:
|
||||
credentials = raw_data['data']
|
||||
except (TypeError, KeyError):
|
||||
pass
|
||||
|
||||
for cred_entry in credentials:
|
||||
try:
|
||||
key, credential = self._decode_credential_from_json(cred_entry)
|
||||
self._data[key] = credential
|
||||
except:
|
||||
# If something goes wrong loading a credential, just ignore it
|
||||
logger.info('Error decoding credential, skipping',
|
||||
exc_info=True)
|
||||
|
||||
def _decode_credential_from_json(self, cred_entry):
|
||||
"""Load a credential from our JSON serialization.
|
||||
|
||||
Args:
|
||||
cred_entry: A dict entry from the data member of our format
|
||||
|
||||
Returns:
|
||||
(key, cred) where the key is the key tuple and the cred is the
|
||||
OAuth2Credential object.
|
||||
"""
|
||||
raw_key = cred_entry['key']
|
||||
key = _dict_to_tuple_key(raw_key)
|
||||
credential = None
|
||||
credential = client.Credentials.new_from_json(
|
||||
json.dumps(cred_entry['credential']))
|
||||
return (key, credential)
|
||||
|
||||
def _write(self):
|
||||
"""Write the cached data back out.
|
||||
|
||||
The multistore must be locked.
|
||||
"""
|
||||
raw_data = {'file_version': 1}
|
||||
raw_creds = []
|
||||
raw_data['data'] = raw_creds
|
||||
for (cred_key, cred) in self._data.items():
|
||||
raw_key = dict(cred_key)
|
||||
raw_cred = json.loads(cred.to_json())
|
||||
raw_creds.append({'key': raw_key, 'credential': raw_cred})
|
||||
self._locked_json_write(raw_data)
|
||||
|
||||
def _get_all_credential_keys(self):
|
||||
"""Gets all the registered credential keys in the multistore.
|
||||
|
||||
Returns:
|
||||
A list of dictionaries corresponding to all the keys currently
|
||||
registered
|
||||
"""
|
||||
return [dict(key) for key in self._data.keys()]
|
||||
|
||||
def _get_credential(self, key):
|
||||
"""Get a credential from the multistore.
|
||||
|
||||
The multistore must be locked.
|
||||
|
||||
Args:
|
||||
key: The key used to retrieve the credential
|
||||
|
||||
Returns:
|
||||
The credential specified or None if not present
|
||||
"""
|
||||
return self._data.get(key, None)
|
||||
|
||||
def _update_credential(self, key, cred):
|
||||
"""Update a credential and write the multistore.
|
||||
|
||||
This must be called when the multistore is locked.
|
||||
|
||||
Args:
|
||||
key: The key used to retrieve the credential
|
||||
cred: The OAuth2Credential to update/set
|
||||
"""
|
||||
self._data[key] = cred
|
||||
self._write()
|
||||
|
||||
def _delete_credential(self, key):
|
||||
"""Delete a credential and write the multistore.
|
||||
|
||||
This must be called when the multistore is locked.
|
||||
|
||||
Args:
|
||||
key: The key used to retrieve the credential
|
||||
"""
|
||||
try:
|
||||
del self._data[key]
|
||||
except KeyError:
|
||||
pass
|
||||
self._write()
|
||||
|
||||
def _get_storage(self, key):
|
||||
"""Get a Storage object to get/set a credential.
|
||||
|
||||
This Storage is a 'view' into the multistore.
|
||||
|
||||
Args:
|
||||
key: The key used to retrieve the credential
|
||||
|
||||
Returns:
|
||||
A Storage object that can be used to get/set this cred
|
||||
"""
|
||||
return self._Storage(self, key)
|
||||
@@ -1,244 +0,0 @@
|
||||
# Copyright 2016 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import mock
|
||||
import unittest2
|
||||
|
||||
from oauth2client.contrib import locked_file
|
||||
|
||||
|
||||
class TestOpener(unittest2.TestCase):
|
||||
def _make_one(self):
|
||||
_filehandle, filename = tempfile.mkstemp()
|
||||
os.close(_filehandle)
|
||||
return locked_file._Opener(filename, 'r+', 'r'), filename
|
||||
|
||||
def test_ctor(self):
|
||||
instance, filename = self._make_one()
|
||||
self.assertFalse(instance._locked)
|
||||
self.assertEqual(instance._filename, filename)
|
||||
self.assertEqual(instance._mode, 'r+')
|
||||
self.assertEqual(instance._fallback_mode, 'r')
|
||||
self.assertIsNone(instance._fh)
|
||||
self.assertIsNone(instance._lock_fd)
|
||||
|
||||
def test_is_locked(self):
|
||||
instance, _ = self._make_one()
|
||||
self.assertFalse(instance.is_locked())
|
||||
instance._locked = True
|
||||
self.assertTrue(instance.is_locked())
|
||||
|
||||
def test_file_handle(self):
|
||||
instance, _ = self._make_one()
|
||||
self.assertIsNone(instance.file_handle())
|
||||
fh = mock.Mock()
|
||||
instance._fh = fh
|
||||
self.assertEqual(instance.file_handle(), fh)
|
||||
|
||||
def test_filename(self):
|
||||
instance, filename = self._make_one()
|
||||
self.assertEqual(instance.filename(), filename)
|
||||
|
||||
def test_open_and_lock(self):
|
||||
instance, _ = self._make_one()
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
def test_unlock_and_close(self):
|
||||
instance, _ = self._make_one()
|
||||
instance.unlock_and_close()
|
||||
|
||||
|
||||
class TestPosixOpener(TestOpener):
|
||||
def _make_one(self):
|
||||
_filehandle, filename = tempfile.mkstemp()
|
||||
os.close(_filehandle)
|
||||
return locked_file._PosixOpener(filename, 'r+', 'r'), filename
|
||||
|
||||
def test_relock_fail(self):
|
||||
instance, _ = self._make_one()
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
self.assertTrue(instance.is_locked())
|
||||
self.assertIsNotNone(instance.file_handle())
|
||||
with self.assertRaises(locked_file.AlreadyLockedException):
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
def test_lock_access_error_fallback_mode(self, mock_open):
|
||||
# NOTE: This is a bad case. The behavior here should be that the
|
||||
# error gets re-raised, but the module lets the if statement fall
|
||||
# through.
|
||||
instance, _ = self._make_one()
|
||||
mock_open.side_effect = [IOError(errno.ENOENT, '')]
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
self.assertIsNone(instance.file_handle())
|
||||
self.assertTrue(instance.is_locked())
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
def test_lock_non_access_error(self, mock_open):
|
||||
instance, _ = self._make_one()
|
||||
fh_mock = mock.Mock()
|
||||
mock_open.side_effect = [IOError(errno.EACCES, ''), fh_mock]
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
self.assertEqual(instance.file_handle(), fh_mock)
|
||||
self.assertFalse(instance.is_locked())
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
def test_lock_unexpected_error(self, mock_open):
|
||||
instance, _ = self._make_one()
|
||||
|
||||
with mock.patch('os.open') as mock_os_open:
|
||||
mock_os_open.side_effect = [OSError(errno.EPERM, '')]
|
||||
with self.assertRaises(OSError):
|
||||
instance.open_and_lock(1, 1)
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
@mock.patch('oauth2client.contrib.locked_file.logger')
|
||||
@mock.patch('time.time')
|
||||
def test_lock_timeout_error(self, mock_time, mock_logger, mock_open):
|
||||
instance, _ = self._make_one()
|
||||
# Make it seem like 10 seconds have passed between calls.
|
||||
mock_time.side_effect = [0, 10]
|
||||
|
||||
with mock.patch('os.open') as mock_os_open:
|
||||
# Raising EEXIST should cause it to try to retry locking.
|
||||
mock_os_open.side_effect = [OSError(errno.EEXIST, '')]
|
||||
instance.open_and_lock(1, 1)
|
||||
self.assertFalse(instance.is_locked())
|
||||
self.assertTrue(mock_logger.warn.called)
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
@mock.patch('oauth2client.contrib.locked_file.logger')
|
||||
@mock.patch('time.time')
|
||||
def test_lock_timeout_error_no_fh(self, mock_time, mock_logger, mock_open):
|
||||
instance, _ = self._make_one()
|
||||
# Make it seem like 10 seconds have passed between calls.
|
||||
mock_time.side_effect = [0, 10]
|
||||
# This will cause the retry loop to enter without a file handle.
|
||||
fh_mock = mock.Mock()
|
||||
mock_open.side_effect = [IOError(errno.ENOENT, ''), fh_mock]
|
||||
|
||||
with mock.patch('os.open') as mock_os_open:
|
||||
# Raising EEXIST should cause it to try to retry locking.
|
||||
mock_os_open.side_effect = [OSError(errno.EEXIST, '')]
|
||||
instance.open_and_lock(1, 1)
|
||||
self.assertFalse(instance.is_locked())
|
||||
self.assertTrue(mock_logger.warn.called)
|
||||
self.assertEqual(instance.file_handle(), fh_mock)
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
|
||||
@mock.patch('time.time')
|
||||
@mock.patch('time.sleep')
|
||||
def test_lock_retry_success(self, mock_sleep, mock_time, mock_open):
|
||||
instance, _ = self._make_one()
|
||||
# Make it seem like 1 second has passed between calls. Extra values
|
||||
# are needed by the logging module.
|
||||
mock_time.side_effect = [0, 1]
|
||||
|
||||
with mock.patch('os.open') as mock_os_open:
|
||||
# Raising EEXIST should cause it to try to retry locking.
|
||||
mock_os_open.side_effect = [
|
||||
OSError(errno.EEXIST, ''), mock.Mock()]
|
||||
instance.open_and_lock(10, 1)
|
||||
print(mock_os_open.call_args_list)
|
||||
self.assertTrue(instance.is_locked())
|
||||
mock_sleep.assert_called_with(1)
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file.os')
|
||||
def test_unlock(self, os_mock):
|
||||
instance, _ = self._make_one()
|
||||
instance._locked = True
|
||||
lock_fd_mock = instance._lock_fd = mock.Mock()
|
||||
instance._fh = mock.Mock()
|
||||
|
||||
instance.unlock_and_close()
|
||||
|
||||
self.assertFalse(instance.is_locked())
|
||||
os_mock.close.assert_called_once_with(lock_fd_mock)
|
||||
self.assertTrue(os_mock.unlink.called)
|
||||
self.assertTrue(instance._fh.close.called)
|
||||
|
||||
|
||||
class TestLockedFile(unittest2.TestCase):
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
|
||||
def _make_one(self, opener_ctor_mock):
|
||||
opener_mock = mock.Mock()
|
||||
opener_ctor_mock.return_value = opener_mock
|
||||
return locked_file.LockedFile(
|
||||
'a_file', 'r+', 'r', use_native_locking=False), opener_mock
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
|
||||
def test_ctor_minimal(self, opener_mock):
|
||||
locked_file.LockedFile(
|
||||
'a_file', 'r+', 'r', use_native_locking=False)
|
||||
opener_mock.assert_called_with('a_file', 'r+', 'r')
|
||||
|
||||
@mock.patch.dict('sys.modules', {
|
||||
'oauth2client.contrib._win32_opener': mock.Mock()})
|
||||
def test_ctor_native_win32(self):
|
||||
_win32_opener_mock = sys.modules['oauth2client.contrib._win32_opener']
|
||||
locked_file.LockedFile(
|
||||
'a_file', 'r+', 'r', use_native_locking=True)
|
||||
_win32_opener_mock._Win32Opener.assert_called_with('a_file', 'r+', 'r')
|
||||
|
||||
@mock.patch.dict('sys.modules', {
|
||||
'oauth2client.contrib._win32_opener': None,
|
||||
'oauth2client.contrib._fcntl_opener': mock.Mock()})
|
||||
def test_ctor_native_fcntl(self):
|
||||
_fnctl_opener_mock = sys.modules['oauth2client.contrib._fcntl_opener']
|
||||
locked_file.LockedFile(
|
||||
'a_file', 'r+', 'r', use_native_locking=True)
|
||||
_fnctl_opener_mock._FcntlOpener.assert_called_with('a_file', 'r+', 'r')
|
||||
|
||||
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
|
||||
@mock.patch.dict('sys.modules', {
|
||||
'oauth2client.contrib._win32_opener': None,
|
||||
'oauth2client.contrib._fcntl_opener': None})
|
||||
def test_ctor_native_posix_fallback(self, opener_mock):
|
||||
locked_file.LockedFile(
|
||||
'a_file', 'r+', 'r', use_native_locking=True)
|
||||
opener_mock.assert_called_with('a_file', 'r+', 'r')
|
||||
|
||||
def test_filename(self):
|
||||
instance, opener = self._make_one()
|
||||
opener._filename = 'some file'
|
||||
self.assertEqual(instance.filename(), 'some file')
|
||||
|
||||
def test_file_handle(self):
|
||||
instance, opener = self._make_one()
|
||||
self.assertEqual(instance.file_handle(), opener.file_handle())
|
||||
self.assertTrue(opener.file_handle.called)
|
||||
|
||||
def test_is_locked(self):
|
||||
instance, opener = self._make_one()
|
||||
self.assertEqual(instance.is_locked(), opener.is_locked())
|
||||
self.assertTrue(opener.is_locked.called)
|
||||
|
||||
def test_open_and_lock(self):
|
||||
instance, opener = self._make_one()
|
||||
instance.open_and_lock()
|
||||
opener.open_and_lock.assert_called_with(0, 0.05)
|
||||
|
||||
def test_unlock_and_close(self):
|
||||
instance, opener = self._make_one()
|
||||
instance.unlock_and_close()
|
||||
opener.unlock_and_close.assert_called_with()
|
||||
@@ -1,381 +0,0 @@
|
||||
# Copyright 2015 Google Inc. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Unit tests for oauth2client.multistore_file."""
|
||||
|
||||
import datetime
|
||||
import errno
|
||||
import os
|
||||
import stat
|
||||
import tempfile
|
||||
|
||||
import mock
|
||||
import unittest2
|
||||
|
||||
from oauth2client import client
|
||||
from oauth2client import util
|
||||
from oauth2client.contrib import multistore_file
|
||||
|
||||
_filehandle, FILENAME = tempfile.mkstemp('oauth2client_test.data')
|
||||
os.close(_filehandle)
|
||||
|
||||
|
||||
class _MockLockedFile(object):
|
||||
|
||||
def __init__(self, filename_str, error_class, error_code):
|
||||
self.filename_str = filename_str
|
||||
self.error_class = error_class
|
||||
self.error_code = error_code
|
||||
self.open_and_lock_called = False
|
||||
|
||||
def open_and_lock(self):
|
||||
self.open_and_lock_called = True
|
||||
raise self.error_class(self.error_code, '')
|
||||
|
||||
def is_locked(self):
|
||||
return False
|
||||
|
||||
def filename(self):
|
||||
return self.filename_str
|
||||
|
||||
|
||||
class Test__dict_to_tuple_key(unittest2.TestCase):
|
||||
|
||||
def test_key_conversions(self):
|
||||
key1, val1 = 'somekey', 'some value'
|
||||
key2, val2 = 'another', 'something else'
|
||||
key3, val3 = 'onemore', 'foo'
|
||||
test_dict = {
|
||||
key1: val1,
|
||||
key2: val2,
|
||||
key3: val3,
|
||||
}
|
||||
tuple_key = multistore_file._dict_to_tuple_key(test_dict)
|
||||
|
||||
# the resulting key should be naturally sorted
|
||||
expected_output = (
|
||||
(key2, val2),
|
||||
(key3, val3),
|
||||
(key1, val1),
|
||||
)
|
||||
self.assertTupleEqual(expected_output, tuple_key)
|
||||
# check we get the original dictionary back
|
||||
self.assertDictEqual(test_dict, dict(tuple_key))
|
||||
|
||||
|
||||
class MultistoreFileTests(unittest2.TestCase):
|
||||
|
||||
def tearDown(self):
|
||||
try:
|
||||
os.unlink(FILENAME)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def setUp(self):
|
||||
try:
|
||||
os.unlink(FILENAME)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _create_test_credentials(self, client_id='some_client_id',
|
||||
expiration=None):
|
||||
access_token = 'foo'
|
||||
client_secret = 'cOuDdkfjxxnv+'
|
||||
refresh_token = '1/0/a.df219fjls0'
|
||||
token_expiry = expiration or datetime.datetime.utcnow()
|
||||
token_uri = 'https://www.google.com/accounts/o8/oauth2/token'
|
||||
user_agent = 'refresh_checker/1.0'
|
||||
|
||||
credentials = client.OAuth2Credentials(
|
||||
access_token, client_id, client_secret,
|
||||
refresh_token, token_expiry, token_uri,
|
||||
user_agent)
|
||||
return credentials
|
||||
|
||||
def test_lock_file_raises_ioerror(self):
|
||||
filehandle, filename = tempfile.mkstemp()
|
||||
os.close(filehandle)
|
||||
|
||||
try:
|
||||
for error_code in (errno.EDEADLK, errno.ENOSYS, errno.ENOLCK,
|
||||
errno.EACCES):
|
||||
for error_class in (IOError, OSError):
|
||||
multistore = multistore_file._MultiStore(filename)
|
||||
multistore._file = _MockLockedFile(
|
||||
filename, error_class, error_code)
|
||||
# Should not raise though the underlying file class did.
|
||||
multistore._lock()
|
||||
self.assertTrue(multistore._file.open_and_lock_called)
|
||||
finally:
|
||||
os.unlink(filename)
|
||||
|
||||
def test_lock_file_raise_unexpected_error(self):
|
||||
filehandle, filename = tempfile.mkstemp()
|
||||
os.close(filehandle)
|
||||
|
||||
try:
|
||||
multistore = multistore_file._MultiStore(filename)
|
||||
multistore._file = _MockLockedFile(filename, IOError, errno.EBUSY)
|
||||
with self.assertRaises(IOError):
|
||||
multistore._lock()
|
||||
self.assertTrue(multistore._file.open_and_lock_called)
|
||||
finally:
|
||||
os.unlink(filename)
|
||||
|
||||
def test_read_only_file_fail_lock(self):
|
||||
credentials = self._create_test_credentials()
|
||||
|
||||
open(FILENAME, 'a+b').close()
|
||||
os.chmod(FILENAME, 0o400)
|
||||
|
||||
store = multistore_file.get_credential_storage(
|
||||
FILENAME,
|
||||
credentials.client_id,
|
||||
credentials.user_agent,
|
||||
['some-scope', 'some-other-scope'])
|
||||
|
||||
store.put(credentials)
|
||||
if os.name == 'posix': # pragma: NO COVER
|
||||
self.assertTrue(store._multistore._read_only)
|
||||
os.chmod(FILENAME, 0o600)
|
||||
|
||||
def test_read_only_file_fail_lock_no_warning(self):
|
||||
open(FILENAME, 'a+b').close()
|
||||
os.chmod(FILENAME, 0o400)
|
||||
|
||||
multistore = multistore_file._MultiStore(FILENAME)
|
||||
|
||||
with mock.patch.object(multistore_file.logger, 'warn') as mock_warn:
|
||||
multistore._warn_on_readonly = False
|
||||
multistore._lock()
|
||||
self.assertFalse(mock_warn.called)
|
||||
|
||||
def test_lock_skip_refresh(self):
|
||||
with open(FILENAME, 'w') as f:
|
||||
f.write('123')
|
||||
os.chmod(FILENAME, 0o400)
|
||||
|
||||
multistore = multistore_file._MultiStore(FILENAME)
|
||||
|
||||
refresh_patch = mock.patch.object(
|
||||
multistore, '_refresh_data_cache')
|
||||
|
||||
with refresh_patch as refresh_mock:
|
||||
multistore._data = {}
|
||||
multistore._lock()
|
||||
self.assertFalse(refresh_mock.called)
|
||||
|
||||
@unittest2.skipIf(not hasattr(os, 'symlink'), 'No symlink available')
|
||||
def test_multistore_no_symbolic_link_files(self):
|
||||
SYMFILENAME = FILENAME + 'sym'
|
||||
os.symlink(FILENAME, SYMFILENAME)
|
||||
store = multistore_file.get_credential_storage(
|
||||
SYMFILENAME,
|
||||
'some_client_id',
|
||||
'user-agent/1.0',
|
||||
['some-scope', 'some-other-scope'])
|
||||
try:
|
||||
with self.assertRaises(IOError):
|
||||
store.get()
|
||||
finally:
|
||||
os.unlink(SYMFILENAME)
|
||||
|
||||
def test_multistore_non_existent_file(self):
|
||||
store = multistore_file.get_credential_storage(
|
||||
FILENAME,
|
||||
'some_client_id',
|
||||
'user-agent/1.0',
|
||||
['some-scope', 'some-other-scope'])
|
||||
|
||||
credentials = store.get()
|
||||
self.assertIsNone(credentials)
|
||||
|
||||
def test_multistore_file(self):
|
||||
credentials = self._create_test_credentials()
|
||||
|
||||
store = multistore_file.get_credential_storage(
|
||||
FILENAME,
|
||||
credentials.client_id,
|
||||
credentials.user_agent,
|
||||
['some-scope', 'some-other-scope'])
|
||||
|
||||
# Save credentials
|
||||
store.put(credentials)
|
||||
credentials = store.get()
|
||||
|
||||
self.assertIsNotNone(credentials)
|
||||
self.assertEquals('foo', credentials.access_token)
|
||||
|
||||
# Delete credentials
|
||||
store.delete()
|
||||
credentials = store.get()
|
||||
|
||||
self.assertIsNone(credentials)
|
||||
|
||||
if os.name == 'posix': # pragma: NO COVER
|
||||
self.assertEquals(
|
||||
0o600, stat.S_IMODE(os.stat(FILENAME).st_mode))
|
||||
|
||||
def test_multistore_file_custom_key(self):
|
||||
credentials = self._create_test_credentials()
|
||||
|
||||
custom_key = {'myapp': 'testing', 'clientid': 'some client'}
|
||||
store = multistore_file.get_credential_storage_custom_key(
|
||||
FILENAME, custom_key)
|
||||
|
||||
store.put(credentials)
|
||||
stored_credentials = store.get()
|
||||
|
||||
self.assertIsNotNone(stored_credentials)
|
||||
self.assertEqual(credentials.access_token,
|
||||
stored_credentials.access_token)
|
||||
|
||||
store.delete()
|
||||
stored_credentials = store.get()
|
||||
|
||||
self.assertIsNone(stored_credentials)
|
||||
|
||||
def test_multistore_file_custom_string_key(self):
|
||||
credentials = self._create_test_credentials()
|
||||
|
||||
# store with string key
|
||||
store = multistore_file.get_credential_storage_custom_string_key(
|
||||
FILENAME, 'mykey')
|
||||
|
||||
store.put(credentials)
|
||||
stored_credentials = store.get()
|
||||
|
||||
self.assertIsNotNone(stored_credentials)
|
||||
self.assertEqual(credentials.access_token,
|
||||
stored_credentials.access_token)
|
||||
|
||||
# try retrieving with a dictionary
|
||||
multistore_file.get_credential_storage_custom_string_key(
|
||||
FILENAME, {'key': 'mykey'})
|
||||
stored_credentials = store.get()
|
||||
self.assertIsNotNone(stored_credentials)
|
||||
self.assertEqual(credentials.access_token,
|
||||
stored_credentials.access_token)
|
||||
|
||||
store.delete()
|
||||
stored_credentials = store.get()
|
||||
|
||||
self.assertIsNone(stored_credentials)
|
||||
|
||||
def test_multistore_file_backwards_compatibility(self):
|
||||
credentials = self._create_test_credentials()
|
||||
scopes = ['scope1', 'scope2']
|
||||
|
||||
# store the credentials using the legacy key method
|
||||
store = multistore_file.get_credential_storage(
|
||||
FILENAME, 'client_id', 'user_agent', scopes)
|
||||
store.put(credentials)
|
||||
|
||||
# retrieve the credentials using a custom key that matches the
|
||||
# legacy key
|
||||
key = {'clientId': 'client_id', 'userAgent': 'user_agent',
|
||||
'scope': util.scopes_to_string(scopes)}
|
||||
store = multistore_file.get_credential_storage_custom_key(
|
||||
FILENAME, key)
|
||||
stored_credentials = store.get()
|
||||
|
||||
self.assertEqual(credentials.access_token,
|
||||
stored_credentials.access_token)
|
||||
|
||||
def test_multistore_file_get_all_keys(self):
|
||||
# start with no keys
|
||||
keys = multistore_file.get_all_credential_keys(FILENAME)
|
||||
self.assertEquals([], keys)
|
||||
|
||||
# store credentials
|
||||
credentials = self._create_test_credentials(client_id='client1')
|
||||
custom_key = {'myapp': 'testing', 'clientid': 'client1'}
|
||||
store1 = multistore_file.get_credential_storage_custom_key(
|
||||
FILENAME, custom_key)
|
||||
store1.put(credentials)
|
||||
|
||||
keys = multistore_file.get_all_credential_keys(FILENAME)
|
||||
self.assertEquals([custom_key], keys)
|
||||
|
||||
# store more credentials
|
||||
credentials = self._create_test_credentials(client_id='client2')
|
||||
string_key = 'string_key'
|
||||
store2 = multistore_file.get_credential_storage_custom_string_key(
|
||||
FILENAME, string_key)
|
||||
store2.put(credentials)
|
||||
|
||||
keys = multistore_file.get_all_credential_keys(FILENAME)
|
||||
self.assertEquals(2, len(keys))
|
||||
self.assertTrue(custom_key in keys)
|
||||
self.assertTrue({'key': string_key} in keys)
|
||||
|
||||
# back to no keys
|
||||
store1.delete()
|
||||
store2.delete()
|
||||
keys = multistore_file.get_all_credential_keys(FILENAME)
|
||||
self.assertEquals([], keys)
|
||||
|
||||
def _refresh_data_cache_helper(self):
|
||||
multistore = multistore_file._MultiStore(FILENAME)
|
||||
json_patch = mock.patch.object(multistore, '_locked_json_read')
|
||||
|
||||
return multistore, json_patch
|
||||
|
||||
def test__refresh_data_cache_bad_json(self):
|
||||
multistore, json_patch = self._refresh_data_cache_helper()
|
||||
|
||||
with json_patch as json_mock:
|
||||
json_mock.side_effect = ValueError('')
|
||||
multistore._refresh_data_cache()
|
||||
self.assertTrue(json_mock.called)
|
||||
self.assertEqual(multistore._data, {})
|
||||
|
||||
def test__refresh_data_cache_bad_version(self):
|
||||
multistore, json_patch = self._refresh_data_cache_helper()
|
||||
|
||||
with json_patch as json_mock:
|
||||
json_mock.return_value = {}
|
||||
multistore._refresh_data_cache()
|
||||
self.assertTrue(json_mock.called)
|
||||
self.assertEqual(multistore._data, {})
|
||||
|
||||
def test__refresh_data_cache_newer_version(self):
|
||||
multistore, json_patch = self._refresh_data_cache_helper()
|
||||
|
||||
with json_patch as json_mock:
|
||||
json_mock.return_value = {'file_version': 5}
|
||||
with self.assertRaises(multistore_file.NewerCredentialStoreError):
|
||||
multistore._refresh_data_cache()
|
||||
self.assertTrue(json_mock.called)
|
||||
|
||||
def test__refresh_data_cache_bad_credentials(self):
|
||||
multistore, json_patch = self._refresh_data_cache_helper()
|
||||
|
||||
with json_patch as json_mock:
|
||||
json_mock.return_value = {
|
||||
'file_version': 1,
|
||||
'data': [
|
||||
{'lol': 'this is a bad credential object.'}
|
||||
]}
|
||||
multistore._refresh_data_cache()
|
||||
self.assertTrue(json_mock.called)
|
||||
self.assertEqual(multistore._data, {})
|
||||
|
||||
def test__delete_credential_nonexistent(self):
|
||||
multistore = multistore_file._MultiStore(FILENAME)
|
||||
|
||||
with mock.patch.object(multistore, '_write') as write_mock:
|
||||
multistore._data = {}
|
||||
multistore._delete_credential('nonexistent_key')
|
||||
self.assertTrue(write_mock.called)
|
||||
Reference in New Issue
Block a user