openstack/deb-python-oauth2client
Code Issues Proposed changes

Remove contrib.multistore_file (#589)

Browse Source 
Remove dependent modules as well. Resolves #470.
This commit is contained in:
Jon Wayne Parrott
2016-08-04 12:24:45 -07:00
committed by GitHub
parent f04d5213d4
commit 0dc30bc033
9 changed files with 0 additions and 1557 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
docs/source/oauth2client.contrib.locked_file.rst
View File

@@ -1,7 +0,0 @@
oauth2client.contrib.locked_file module
=======================================
.. automodule:: oauth2client.contrib.locked_file
:members:
:undoc-members:
:show-inheritance:

7
docs/source/oauth2client.contrib.multistore_file.rst
View File

@@ -1,7 +0,0 @@
oauth2client.contrib.multistore_file module
===========================================
.. automodule:: oauth2client.contrib.multistore_file
:members:
:undoc-members:
:show-inheritance:

2
docs/source/oauth2client.contrib.rst
View File

@@ -19,9 +19,7 @@ Submodules
oauth2client.contrib.flask_util oauth2client.contrib.flask_util
oauth2client.contrib.gce oauth2client.contrib.gce
oauth2client.contrib.keyring_storage oauth2client.contrib.keyring_storage
oauth2client.contrib.locked_file
oauth2client.contrib.multiprocess_file_storage oauth2client.contrib.multiprocess_file_storage
oauth2client.contrib.multistore_file
oauth2client.contrib.sqlalchemy oauth2client.contrib.sqlalchemy
oauth2client.contrib.xsrfutil oauth2client.contrib.xsrfutil

81
oauth2client/contrib/_fcntl_opener.py
View File

@@ -1,81 +0,0 @@
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import errno
import fcntl
import time
from oauth2client import util
from oauth2client.contrib import locked_file
class _FcntlOpener(locked_file._Opener):
"""Open, lock, and unlock a file using fcntl.lockf."""
def open_and_lock(self, timeout, delay):
"""Open the file and lock it.
Args:
timeout: float, How long to try to lock for.
delay: float, How long to wait between retries
Raises:
AlreadyLockedException: if the lock is already acquired.
IOError: if the open fails.
IOError: if the file is a symbolic link.
"""
if self._locked:
raise locked_file.AlreadyLockedException(
'File {0} is already locked'.format(self._filename))
start_time = time.time()
util.validate_file(self._filename)
try:
self._fh = open(self._filename, self._mode)
except IOError as e:
# If we can't access with _mode, try _fallback_mode and
# don't lock.
if e.errno in (errno.EPERM, errno.EACCES):
self._fh = open(self._filename, self._fallback_mode)
return
# We opened in _mode, try to lock the file.
while True:
try:
fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX)
self._locked = True
return
except IOError as e:
# If not retrying, then just pass on the error.
if timeout == 0:
raise
if e.errno != errno.EACCES:
raise
# We could not acquire the lock. Try again.
if (time.time() - start_time) >= timeout:
locked_file.logger.warn('Could not lock %s in %s seconds',
self._filename, timeout)
if self._fh:
self._fh.close()
self._fh = open(self._filename, self._fallback_mode)
return
time.sleep(delay)
def unlock_and_close(self):
"""Close and unlock the file using the fcntl.lockf primitive."""
if self._locked:
fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN)
self._locked = False
if self._fh:
self._fh.close()

106
oauth2client/contrib/_win32_opener.py
View File

@@ -1,106 +0,0 @@
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import errno
import time
import pywintypes
import win32con
import win32file
from oauth2client import util
from oauth2client.contrib import locked_file
class _Win32Opener(locked_file._Opener):
"""Open, lock, and unlock a file using windows primitives."""
# Error #33:
# 'The process cannot access the file because another process'
FILE_IN_USE_ERROR = 33
# Error #158:
# 'The segment is already unlocked.'
FILE_ALREADY_UNLOCKED_ERROR = 158
def open_and_lock(self, timeout, delay):
"""Open the file and lock it.
Args:
timeout: float, How long to try to lock for.
delay: float, How long to wait between retries
Raises:
AlreadyLockedException: if the lock is already acquired.
IOError: if the open fails.
IOError: if the file is a symbolic link.
"""
if self._locked:
raise locked_file.AlreadyLockedException(
'File {0} is already locked'.format(self._filename))
start_time = time.time()
util.validate_file(self._filename)
try:
self._fh = open(self._filename, self._mode)
except IOError as e:
# If we can't access with _mode, try _fallback_mode
# and don't lock.
if e.errno == errno.EACCES:
self._fh = open(self._filename, self._fallback_mode)
return
# We opened in _mode, try to lock the file.
while True:
try:
hfile = win32file._get_osfhandle(self._fh.fileno())
win32file.LockFileEx(
hfile,
(win32con.LOCKFILE_FAIL_IMMEDIATELY |
win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000,
pywintypes.OVERLAPPED())
self._locked = True
return
except pywintypes.error as e:
if timeout == 0:
raise
# If the error is not that the file is already
# in use, raise.
if e[0] != _Win32Opener.FILE_IN_USE_ERROR:
raise
# We could not acquire the lock. Try again.
if (time.time() - start_time) >= timeout:
locked_file.logger.warn('Could not lock %s in %s seconds',
self._filename, timeout)
if self._fh:
self._fh.close()
self._fh = open(self._filename, self._fallback_mode)
return
time.sleep(delay)
def unlock_and_close(self):
"""Close and unlock the file using the win32 primitive."""
if self._locked:
try:
hfile = win32file._get_osfhandle(self._fh.fileno())
win32file.UnlockFileEx(hfile, 0, -0x10000,
pywintypes.OVERLAPPED())
except pywintypes.error as e:
if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR:
raise
self._locked = False
if self._fh:
self._fh.close()

224
oauth2client/contrib/locked_file.py
View File

@@ -1,224 +0,0 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Locked file interface that should work on Unix and Windows pythons.
This module first tries to use fcntl locking to ensure serialized access
to a file, then falls back on a lock file if that is unavialable.
Usage::
f = LockedFile('filename', 'r+b', 'rb')
f.open_and_lock()
if f.is_locked():
print('Acquired filename with r+b mode')
f.file_handle().write('locked data')
else:
print('Acquired filename with rb mode')
f.unlock_and_close()
"""
from __future__ import print_function
import errno
import logging
import os
import time
from oauth2client import util
__author__ = 'cache@google.com (David T McWherter)'
logger = logging.getLogger(__name__)
class AlreadyLockedException(Exception):
"""Trying to lock a file that has already been locked by the LockedFile."""
pass
class _Opener(object):
"""Base class for different locking primitives."""
def __init__(self, filename, mode, fallback_mode):
"""Create an Opener.
Args:
filename: string, The pathname of the file.
mode: string, The preferred mode to access the file with.
fallback_mode: string, The mode to use if locking fails.
"""
self._locked = False
self._filename = filename
self._mode = mode
self._fallback_mode = fallback_mode
self._fh = None
self._lock_fd = None
def is_locked(self):
"""Was the file locked."""
return self._locked
def file_handle(self):
"""The file handle to the file. Valid only after opened."""
return self._fh
def filename(self):
"""The filename that is being locked."""
return self._filename
def open_and_lock(self, timeout, delay):
"""Open the file and lock it.
Args:
timeout: float, How long to try to lock for.
delay: float, How long to wait between retries.
"""
pass
def unlock_and_close(self):
"""Unlock and close the file."""
pass
class _PosixOpener(_Opener):
"""Lock files using Posix advisory lock files."""
def open_and_lock(self, timeout, delay):
"""Open the file and lock it.
Tries to create a .lock file next to the file we're trying to open.
Args:
timeout: float, How long to try to lock for.
delay: float, How long to wait between retries.
Raises:
AlreadyLockedException: if the lock is already acquired.
IOError: if the open fails.
IOError: if the file is a symbolic link.
"""
if self._locked:
raise AlreadyLockedException(
'File {0} is already locked'.format(self._filename))
self._locked = False
util.validate_file(self._filename)
try:
self._fh = open(self._filename, self._mode)
except IOError as e:
# If we can't access with _mode, try _fallback_mode and don't lock.
if e.errno == errno.EACCES:
self._fh = open(self._filename, self._fallback_mode)
return
lock_filename = self._posix_lockfile(self._filename)
start_time = time.time()
while True:
try:
self._lock_fd = os.open(lock_filename,
os.O_CREAT | os.O_EXCL | os.O_RDWR)
self._locked = True
break
except OSError as e:
if e.errno != errno.EEXIST:
raise
if (time.time() - start_time) >= timeout:
logger.warn('Could not acquire lock %s in %s seconds',
lock_filename, timeout)
# Close the file and open in fallback_mode.
if self._fh:
self._fh.close()
self._fh = open(self._filename, self._fallback_mode)
return
time.sleep(delay)
def unlock_and_close(self):
"""Unlock a file by removing the .lock file, and close the handle."""
if self._locked:
lock_filename = self._posix_lockfile(self._filename)
os.close(self._lock_fd)
os.unlink(lock_filename)
self._locked = False
self._lock_fd = None
if self._fh:
self._fh.close()
def _posix_lockfile(self, filename):
"""The name of the lock file to use for posix locking."""
return '{0}.lock'.format(filename)
class LockedFile(object):
"""Represent a file that has exclusive access."""
@util.positional(4)
def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
"""Construct a LockedFile.
Args:
filename: string, The path of the file to open.
mode: string, The mode to try to open the file with.
fallback_mode: string, The mode to use if locking fails.
use_native_locking: bool, Whether or not fcntl/win32 locking is
used.
"""
opener = None
if not opener and use_native_locking:
try:
from oauth2client.contrib._win32_opener import _Win32Opener
opener = _Win32Opener(filename, mode, fallback_mode)
except ImportError:
try:
from oauth2client.contrib._fcntl_opener import _FcntlOpener
opener = _FcntlOpener(filename, mode, fallback_mode)
except ImportError:
pass
if not opener:
opener = _PosixOpener(filename, mode, fallback_mode)
self._opener = opener
def filename(self):
"""Return the filename we were constructed with."""
return self._opener._filename
def file_handle(self):
"""Return the file_handle to the opened file."""
return self._opener.file_handle()
def is_locked(self):
"""Return whether we successfully locked the file."""
return self._opener.is_locked()
def open_and_lock(self, timeout=0, delay=0.05):
"""Open the file, trying to lock it.
Args:
timeout: float, The number of seconds to try to acquire the lock.
delay: float, The number of seconds to wait between retry attempts.
Raises:
AlreadyLockedException: if the lock is already acquired.
IOError: if the open fails.
"""
self._opener.open_and_lock(timeout, delay)
def unlock_and_close(self):
"""Unlock and close a file."""
self._opener.unlock_and_close()

505
oauth2client/contrib/multistore_file.py
View File

@@ -1,505 +0,0 @@
# Copyright 2014 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Multi-credential file store with lock support.
This module implements a JSON credential store where multiple
credentials can be stored in one file. That file supports locking
both in a single process and across processes.
The credential themselves are keyed off of:
* client_id
* user_agent
* scope
The format of the stored data is like so::
{
'file_version': 1,
'data': [
{
'key': {
'clientId': '<client id>',
'userAgent': '<user agent>',
'scope': '<scope>'
},
'credential': {
# JSON serialized Credentials.
}
}
]
}
"""
import errno
import json
import logging
import os
import threading
from oauth2client import client
from oauth2client import util
from oauth2client.contrib import locked_file
__author__ = 'jbeda@google.com (Joe Beda)'
logger = logging.getLogger(__name__)
logger.warning(
'The oauth2client.contrib.multistore_file module has been deprecated and '
'will be removed in the next release of oauth2client. Please migrate to '
'multiprocess_file_storage.')
# A dict from 'filename'->_MultiStore instances
_multistores = {}
_multistores_lock = threading.Lock()
class Error(Exception):
"""Base error for this module."""
class NewerCredentialStoreError(Error):
"""The credential store is a newer version than supported."""
def _dict_to_tuple_key(dictionary):
"""Converts a dictionary to a tuple that can be used as an immutable key.
The resulting key is always sorted so that logically equivalent
dictionaries always produce an identical tuple for a key.
Args:
dictionary: the dictionary to use as the key.
Returns:
A tuple representing the dictionary in it's naturally sorted ordering.
"""
return tuple(sorted(dictionary.items()))
@util.positional(4)
def get_credential_storage(filename, client_id, user_agent, scope,
warn_on_readonly=True):
"""Get a Storage instance for a credential.
Args:
filename: The JSON file storing a set of credentials
client_id: The client_id for the credential
user_agent: The user agent for the credential
scope: string or iterable of strings, Scope(s) being requested
warn_on_readonly: if True, log a warning if the store is readonly
Returns:
An object derived from client.Storage for getting/setting the
credential.
"""
# Recreate the legacy key with these specific parameters
key = {'clientId': client_id, 'userAgent': user_agent,
'scope': util.scopes_to_string(scope)}
return get_credential_storage_custom_key(
filename, key, warn_on_readonly=warn_on_readonly)
@util.positional(2)
def get_credential_storage_custom_string_key(filename, key_string,
warn_on_readonly=True):
"""Get a Storage instance for a credential using a single string as a key.
Allows you to provide a string as a custom key that will be used for
credential storage and retrieval.
Args:
filename: The JSON file storing a set of credentials
key_string: A string to use as the key for storing this credential.
warn_on_readonly: if True, log a warning if the store is readonly
Returns:
An object derived from client.Storage for getting/setting the
credential.
"""
# Create a key dictionary that can be used
key_dict = {'key': key_string}
return get_credential_storage_custom_key(
filename, key_dict, warn_on_readonly=warn_on_readonly)
@util.positional(2)
def get_credential_storage_custom_key(filename, key_dict,
warn_on_readonly=True):
"""Get a Storage instance for a credential using a dictionary as a key.
Allows you to provide a dictionary as a custom key that will be used for
credential storage and retrieval.
Args:
filename: The JSON file storing a set of credentials
key_dict: A dictionary to use as the key for storing this credential.
There is no ordering of the keys in the dictionary. Logically
equivalent dictionaries will produce equivalent storage keys.
warn_on_readonly: if True, log a warning if the store is readonly
Returns:
An object derived from client.Storage for getting/setting the
credential.
"""
multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly)
key = _dict_to_tuple_key(key_dict)
return multistore._get_storage(key)
@util.positional(1)
def get_all_credential_keys(filename, warn_on_readonly=True):
"""Gets all the registered credential keys in the given Multistore.
Args:
filename: The JSON file storing a set of credentials
warn_on_readonly: if True, log a warning if the store is readonly
Returns:
A list of the credential keys present in the file. They are returned
as dictionaries that can be passed into
get_credential_storage_custom_key to get the actual credentials.
"""
multistore = _get_multistore(filename, warn_on_readonly=warn_on_readonly)
multistore._lock()
try:
return multistore._get_all_credential_keys()
finally:
multistore._unlock()
@util.positional(1)
def _get_multistore(filename, warn_on_readonly=True):
"""A helper method to initialize the multistore with proper locking.
Args:
filename: The JSON file storing a set of credentials
warn_on_readonly: if True, log a warning if the store is readonly
Returns:
A multistore object
"""
filename = os.path.expanduser(filename)
_multistores_lock.acquire()
try:
multistore = _multistores.setdefault(
filename, _MultiStore(filename, warn_on_readonly=warn_on_readonly))
finally:
_multistores_lock.release()
return multistore
class _MultiStore(object):
"""A file backed store for multiple credentials."""
@util.positional(2)
def __init__(self, filename, warn_on_readonly=True):
"""Initialize the class.
This will create the file if necessary.
"""
self._file = locked_file.LockedFile(filename, 'r+', 'r')
self._thread_lock = threading.Lock()
self._read_only = False
self._warn_on_readonly = warn_on_readonly
self._create_file_if_needed()
# Cache of deserialized store. This is only valid after the
# _MultiStore is locked or _refresh_data_cache is called. This is
# of the form of:
#
# ((key, value), (key, value)...) -> OAuth2Credential
#
# If this is None, then the store hasn't been read yet.
self._data = None
class _Storage(client.Storage):
"""A Storage object that can read/write a single credential."""
def __init__(self, multistore, key):
self._multistore = multistore
self._key = key
def acquire_lock(self):
"""Acquires any lock necessary to access this Storage.
This lock is not reentrant.
"""
self._multistore._lock()
def release_lock(self):
"""Release the Storage lock.
Trying to release a lock that isn't held will result in a
RuntimeError.
"""
self._multistore._unlock()
def locked_get(self):
"""Retrieve credential.
The Storage lock must be held when this is called.
Returns:
oauth2client.client.Credentials
"""
credential = self._multistore._get_credential(self._key)
if credential:
credential.set_store(self)
return credential
def locked_put(self, credentials):
"""Write a credential.
The Storage lock must be held when this is called.
Args:
credentials: Credentials, the credentials to store.
"""
self._multistore._update_credential(self._key, credentials)
def locked_delete(self):
"""Delete a credential.
The Storage lock must be held when this is called.
Args:
credentials: Credentials, the credentials to store.
"""
self._multistore._delete_credential(self._key)
def _create_file_if_needed(self):
"""Create an empty file if necessary.
This method will not initialize the file. Instead it implements a
simple version of "touch" to ensure the file has been created.
"""
if not os.path.exists(self._file.filename()):
old_umask = os.umask(0o177)
try:
open(self._file.filename(), 'a+b').close()
finally:
os.umask(old_umask)
def _lock(self):
"""Lock the entire multistore."""
self._thread_lock.acquire()
try:
self._file.open_and_lock()
except (IOError, OSError) as e:
if e.errno == errno.ENOSYS:
logger.warn('File system does not support locking the '
'credentials file.')
elif e.errno == errno.ENOLCK:
logger.warn('File system is out of resources for writing the '
'credentials file (is your disk full?).')
elif e.errno == errno.EDEADLK:
logger.warn('Lock contention on multistore file, opening '
'in read-only mode.')
elif e.errno == errno.EACCES:
logger.warn('Cannot access credentials file.')
else:
raise
if not self._file.is_locked():
self._read_only = True
if self._warn_on_readonly:
logger.warn('The credentials file (%s) is not writable. '
'Opening in read-only mode. Any refreshed '
'credentials will only be '
'valid for this run.', self._file.filename())
if os.path.getsize(self._file.filename()) == 0:
logger.debug('Initializing empty multistore file')
# The multistore is empty so write out an empty file.
self._data = {}
self._write()
elif not self._read_only or self._data is None:
# Only refresh the data if we are read/write or we haven't
# cached the data yet. If we are readonly, we assume is isn't
# changing out from under us and that we only have to read it
# once. This prevents us from whacking any new access keys that
# we have cached in memory but were unable to write out.
self._refresh_data_cache()
def _unlock(self):
"""Release the lock on the multistore."""
self._file.unlock_and_close()
self._thread_lock.release()
def _locked_json_read(self):
"""Get the raw content of the multistore file.
The multistore must be locked when this is called.
Returns:
The contents of the multistore decoded as JSON.
"""
assert self._thread_lock.locked()
self._file.file_handle().seek(0)
return json.load(self._file.file_handle())
def _locked_json_write(self, data):
"""Write a JSON serializable data structure to the multistore.
The multistore must be locked when this is called.
Args:
data: The data to be serialized and written.
"""
assert self._thread_lock.locked()
if self._read_only:
return
self._file.file_handle().seek(0)
json.dump(data, self._file.file_handle(),
sort_keys=True, indent=2, separators=(',', ': '))
self._file.file_handle().truncate()
def _refresh_data_cache(self):
"""Refresh the contents of the multistore.
The multistore must be locked when this is called.
Raises:
NewerCredentialStoreError: Raised when a newer client has written
the store.
"""
self._data = {}
try:
raw_data = self._locked_json_read()
except Exception:
logger.warn('Credential data store could not be loaded. '
'Will ignore and overwrite.')
return
version = 0
try:
version = raw_data['file_version']
except Exception:
logger.warn('Missing version for credential data store. It may be '
'corrupt or an old version. Overwriting.')
if version > 1:
raise NewerCredentialStoreError(
'Credential file has file_version of {0}. '
'Only file_version of 1 is supported.'.format(version))
credentials = []
try:
credentials = raw_data['data']
except (TypeError, KeyError):
pass
for cred_entry in credentials:
try:
key, credential = self._decode_credential_from_json(cred_entry)
self._data[key] = credential
except:
# If something goes wrong loading a credential, just ignore it
logger.info('Error decoding credential, skipping',
exc_info=True)
def _decode_credential_from_json(self, cred_entry):
"""Load a credential from our JSON serialization.
Args:
cred_entry: A dict entry from the data member of our format
Returns:
(key, cred) where the key is the key tuple and the cred is the
OAuth2Credential object.
"""
raw_key = cred_entry['key']
key = _dict_to_tuple_key(raw_key)
credential = None
credential = client.Credentials.new_from_json(
json.dumps(cred_entry['credential']))
return (key, credential)
def _write(self):
"""Write the cached data back out.
The multistore must be locked.
"""
raw_data = {'file_version': 1}
raw_creds = []
raw_data['data'] = raw_creds
for (cred_key, cred) in self._data.items():
raw_key = dict(cred_key)
raw_cred = json.loads(cred.to_json())
raw_creds.append({'key': raw_key, 'credential': raw_cred})
self._locked_json_write(raw_data)
def _get_all_credential_keys(self):
"""Gets all the registered credential keys in the multistore.
Returns:
A list of dictionaries corresponding to all the keys currently
registered
"""
return [dict(key) for key in self._data.keys()]
def _get_credential(self, key):
"""Get a credential from the multistore.
The multistore must be locked.
Args:
key: The key used to retrieve the credential
Returns:
The credential specified or None if not present
"""
return self._data.get(key, None)
def _update_credential(self, key, cred):
"""Update a credential and write the multistore.
This must be called when the multistore is locked.
Args:
key: The key used to retrieve the credential
cred: The OAuth2Credential to update/set
"""
self._data[key] = cred
self._write()
def _delete_credential(self, key):
"""Delete a credential and write the multistore.
This must be called when the multistore is locked.
Args:
key: The key used to retrieve the credential
"""
try:
del self._data[key]
except KeyError:
pass
self._write()
def _get_storage(self, key):
"""Get a Storage object to get/set a credential.
This Storage is a 'view' into the multistore.
Args:
key: The key used to retrieve the credential
Returns:
A Storage object that can be used to get/set this cred
"""
return self._Storage(self, key)

244
tests/contrib/test_locked_file.py
View File

@@ -1,244 +0,0 @@
# Copyright 2016 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import errno
import os
import sys
import tempfile
import mock
import unittest2
from oauth2client.contrib import locked_file
class TestOpener(unittest2.TestCase):
def _make_one(self):
_filehandle, filename = tempfile.mkstemp()
os.close(_filehandle)
return locked_file._Opener(filename, 'r+', 'r'), filename
def test_ctor(self):
instance, filename = self._make_one()
self.assertFalse(instance._locked)
self.assertEqual(instance._filename, filename)
self.assertEqual(instance._mode, 'r+')
self.assertEqual(instance._fallback_mode, 'r')
self.assertIsNone(instance._fh)
self.assertIsNone(instance._lock_fd)
def test_is_locked(self):
instance, _ = self._make_one()
self.assertFalse(instance.is_locked())
instance._locked = True
self.assertTrue(instance.is_locked())
def test_file_handle(self):
instance, _ = self._make_one()
self.assertIsNone(instance.file_handle())
fh = mock.Mock()
instance._fh = fh
self.assertEqual(instance.file_handle(), fh)
def test_filename(self):
instance, filename = self._make_one()
self.assertEqual(instance.filename(), filename)
def test_open_and_lock(self):
instance, _ = self._make_one()
instance.open_and_lock(1, 1)
def test_unlock_and_close(self):
instance, _ = self._make_one()
instance.unlock_and_close()
class TestPosixOpener(TestOpener):
def _make_one(self):
_filehandle, filename = tempfile.mkstemp()
os.close(_filehandle)
return locked_file._PosixOpener(filename, 'r+', 'r'), filename
def test_relock_fail(self):
instance, _ = self._make_one()
instance.open_and_lock(1, 1)
self.assertTrue(instance.is_locked())
self.assertIsNotNone(instance.file_handle())
with self.assertRaises(locked_file.AlreadyLockedException):
instance.open_and_lock(1, 1)
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
def test_lock_access_error_fallback_mode(self, mock_open):
# NOTE: This is a bad case. The behavior here should be that the
# error gets re-raised, but the module lets the if statement fall
# through.
instance, _ = self._make_one()
mock_open.side_effect = [IOError(errno.ENOENT, '')]
instance.open_and_lock(1, 1)
self.assertIsNone(instance.file_handle())
self.assertTrue(instance.is_locked())
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
def test_lock_non_access_error(self, mock_open):
instance, _ = self._make_one()
fh_mock = mock.Mock()
mock_open.side_effect = [IOError(errno.EACCES, ''), fh_mock]
instance.open_and_lock(1, 1)
self.assertEqual(instance.file_handle(), fh_mock)
self.assertFalse(instance.is_locked())
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
def test_lock_unexpected_error(self, mock_open):
instance, _ = self._make_one()
with mock.patch('os.open') as mock_os_open:
mock_os_open.side_effect = [OSError(errno.EPERM, '')]
with self.assertRaises(OSError):
instance.open_and_lock(1, 1)
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
@mock.patch('oauth2client.contrib.locked_file.logger')
@mock.patch('time.time')
def test_lock_timeout_error(self, mock_time, mock_logger, mock_open):
instance, _ = self._make_one()
# Make it seem like 10 seconds have passed between calls.
mock_time.side_effect = [0, 10]
with mock.patch('os.open') as mock_os_open:
# Raising EEXIST should cause it to try to retry locking.
mock_os_open.side_effect = [OSError(errno.EEXIST, '')]
instance.open_and_lock(1, 1)
self.assertFalse(instance.is_locked())
self.assertTrue(mock_logger.warn.called)
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
@mock.patch('oauth2client.contrib.locked_file.logger')
@mock.patch('time.time')
def test_lock_timeout_error_no_fh(self, mock_time, mock_logger, mock_open):
instance, _ = self._make_one()
# Make it seem like 10 seconds have passed between calls.
mock_time.side_effect = [0, 10]
# This will cause the retry loop to enter without a file handle.
fh_mock = mock.Mock()
mock_open.side_effect = [IOError(errno.ENOENT, ''), fh_mock]
with mock.patch('os.open') as mock_os_open:
# Raising EEXIST should cause it to try to retry locking.
mock_os_open.side_effect = [OSError(errno.EEXIST, '')]
instance.open_and_lock(1, 1)
self.assertFalse(instance.is_locked())
self.assertTrue(mock_logger.warn.called)
self.assertEqual(instance.file_handle(), fh_mock)
@mock.patch('oauth2client.contrib.locked_file.open', create=True)
@mock.patch('time.time')
@mock.patch('time.sleep')
def test_lock_retry_success(self, mock_sleep, mock_time, mock_open):
instance, _ = self._make_one()
# Make it seem like 1 second has passed between calls. Extra values
# are needed by the logging module.
mock_time.side_effect = [0, 1]
with mock.patch('os.open') as mock_os_open:
# Raising EEXIST should cause it to try to retry locking.
mock_os_open.side_effect = [
OSError(errno.EEXIST, ''), mock.Mock()]
instance.open_and_lock(10, 1)
print(mock_os_open.call_args_list)
self.assertTrue(instance.is_locked())
mock_sleep.assert_called_with(1)
@mock.patch('oauth2client.contrib.locked_file.os')
def test_unlock(self, os_mock):
instance, _ = self._make_one()
instance._locked = True
lock_fd_mock = instance._lock_fd = mock.Mock()
instance._fh = mock.Mock()
instance.unlock_and_close()
self.assertFalse(instance.is_locked())
os_mock.close.assert_called_once_with(lock_fd_mock)
self.assertTrue(os_mock.unlink.called)
self.assertTrue(instance._fh.close.called)
class TestLockedFile(unittest2.TestCase):
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
def _make_one(self, opener_ctor_mock):
opener_mock = mock.Mock()
opener_ctor_mock.return_value = opener_mock
return locked_file.LockedFile(
'a_file', 'r+', 'r', use_native_locking=False), opener_mock
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
def test_ctor_minimal(self, opener_mock):
locked_file.LockedFile(
'a_file', 'r+', 'r', use_native_locking=False)
opener_mock.assert_called_with('a_file', 'r+', 'r')
@mock.patch.dict('sys.modules', {
'oauth2client.contrib._win32_opener': mock.Mock()})
def test_ctor_native_win32(self):
_win32_opener_mock = sys.modules['oauth2client.contrib._win32_opener']
locked_file.LockedFile(
'a_file', 'r+', 'r', use_native_locking=True)
_win32_opener_mock._Win32Opener.assert_called_with('a_file', 'r+', 'r')
@mock.patch.dict('sys.modules', {
'oauth2client.contrib._win32_opener': None,
'oauth2client.contrib._fcntl_opener': mock.Mock()})
def test_ctor_native_fcntl(self):
_fnctl_opener_mock = sys.modules['oauth2client.contrib._fcntl_opener']
locked_file.LockedFile(
'a_file', 'r+', 'r', use_native_locking=True)
_fnctl_opener_mock._FcntlOpener.assert_called_with('a_file', 'r+', 'r')
@mock.patch('oauth2client.contrib.locked_file._PosixOpener')
@mock.patch.dict('sys.modules', {
'oauth2client.contrib._win32_opener': None,
'oauth2client.contrib._fcntl_opener': None})
def test_ctor_native_posix_fallback(self, opener_mock):
locked_file.LockedFile(
'a_file', 'r+', 'r', use_native_locking=True)
opener_mock.assert_called_with('a_file', 'r+', 'r')
def test_filename(self):
instance, opener = self._make_one()
opener._filename = 'some file'
self.assertEqual(instance.filename(), 'some file')
def test_file_handle(self):
instance, opener = self._make_one()
self.assertEqual(instance.file_handle(), opener.file_handle())
self.assertTrue(opener.file_handle.called)
def test_is_locked(self):
instance, opener = self._make_one()
self.assertEqual(instance.is_locked(), opener.is_locked())
self.assertTrue(opener.is_locked.called)
def test_open_and_lock(self):
instance, opener = self._make_one()
instance.open_and_lock()
opener.open_and_lock.assert_called_with(0, 0.05)
def test_unlock_and_close(self):
instance, opener = self._make_one()
instance.unlock_and_close()
opener.unlock_and_close.assert_called_with()

381
tests/contrib/test_multistore_file.py
View File

@@ -1,381 +0,0 @@
# Copyright 2015 Google Inc. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Unit tests for oauth2client.multistore_file."""
import datetime
import errno
import os
import stat
import tempfile
import mock
import unittest2
from oauth2client import client
from oauth2client import util
from oauth2client.contrib import multistore_file
_filehandle, FILENAME = tempfile.mkstemp('oauth2client_test.data')
os.close(_filehandle)
class _MockLockedFile(object):
def __init__(self, filename_str, error_class, error_code):
self.filename_str = filename_str
self.error_class = error_class
self.error_code = error_code
self.open_and_lock_called = False
def open_and_lock(self):
self.open_and_lock_called = True
raise self.error_class(self.error_code, '')
def is_locked(self):
return False
def filename(self):
return self.filename_str
class Test__dict_to_tuple_key(unittest2.TestCase):
def test_key_conversions(self):
key1, val1 = 'somekey', 'some value'
key2, val2 = 'another', 'something else'
key3, val3 = 'onemore', 'foo'
test_dict = {
key1: val1,
key2: val2,
key3: val3,
}
tuple_key = multistore_file._dict_to_tuple_key(test_dict)
# the resulting key should be naturally sorted
expected_output = (
(key2, val2),
(key3, val3),
(key1, val1),
)
self.assertTupleEqual(expected_output, tuple_key)
# check we get the original dictionary back
self.assertDictEqual(test_dict, dict(tuple_key))
class MultistoreFileTests(unittest2.TestCase):
def tearDown(self):
try:
os.unlink(FILENAME)
except OSError:
pass
def setUp(self):
try:
os.unlink(FILENAME)
except OSError:
pass
def _create_test_credentials(self, client_id='some_client_id',
expiration=None):
access_token = 'foo'
client_secret = 'cOuDdkfjxxnv+'
refresh_token = '1/0/a.df219fjls0'
token_expiry = expiration or datetime.datetime.utcnow()
token_uri = 'https://www.google.com/accounts/o8/oauth2/token'
user_agent = 'refresh_checker/1.0'
credentials = client.OAuth2Credentials(
access_token, client_id, client_secret,
refresh_token, token_expiry, token_uri,
user_agent)
return credentials
def test_lock_file_raises_ioerror(self):
filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
try:
for error_code in (errno.EDEADLK, errno.ENOSYS, errno.ENOLCK,
errno.EACCES):
for error_class in (IOError, OSError):
multistore = multistore_file._MultiStore(filename)
multistore._file = _MockLockedFile(
filename, error_class, error_code)
# Should not raise though the underlying file class did.
multistore._lock()
self.assertTrue(multistore._file.open_and_lock_called)
finally:
os.unlink(filename)
def test_lock_file_raise_unexpected_error(self):
filehandle, filename = tempfile.mkstemp()
os.close(filehandle)
try:
multistore = multistore_file._MultiStore(filename)
multistore._file = _MockLockedFile(filename, IOError, errno.EBUSY)
with self.assertRaises(IOError):
multistore._lock()
self.assertTrue(multistore._file.open_and_lock_called)
finally:
os.unlink(filename)
def test_read_only_file_fail_lock(self):
credentials = self._create_test_credentials()
open(FILENAME, 'a+b').close()
os.chmod(FILENAME, 0o400)
store = multistore_file.get_credential_storage(
FILENAME,
credentials.client_id,
credentials.user_agent,
['some-scope', 'some-other-scope'])
store.put(credentials)
if os.name == 'posix': # pragma: NO COVER
self.assertTrue(store._multistore._read_only)
os.chmod(FILENAME, 0o600)
def test_read_only_file_fail_lock_no_warning(self):
open(FILENAME, 'a+b').close()
os.chmod(FILENAME, 0o400)
multistore = multistore_file._MultiStore(FILENAME)
with mock.patch.object(multistore_file.logger, 'warn') as mock_warn:
multistore._warn_on_readonly = False
multistore._lock()
self.assertFalse(mock_warn.called)
def test_lock_skip_refresh(self):
with open(FILENAME, 'w') as f:
f.write('123')
os.chmod(FILENAME, 0o400)
multistore = multistore_file._MultiStore(FILENAME)
refresh_patch = mock.patch.object(
multistore, '_refresh_data_cache')
with refresh_patch as refresh_mock:
multistore._data = {}
multistore._lock()
self.assertFalse(refresh_mock.called)
@unittest2.skipIf(not hasattr(os, 'symlink'), 'No symlink available')
def test_multistore_no_symbolic_link_files(self):
SYMFILENAME = FILENAME + 'sym'
os.symlink(FILENAME, SYMFILENAME)
store = multistore_file.get_credential_storage(
SYMFILENAME,
'some_client_id',
'user-agent/1.0',
['some-scope', 'some-other-scope'])
try:
with self.assertRaises(IOError):
store.get()
finally:
os.unlink(SYMFILENAME)
def test_multistore_non_existent_file(self):
store = multistore_file.get_credential_storage(
FILENAME,
'some_client_id',
'user-agent/1.0',
['some-scope', 'some-other-scope'])
credentials = store.get()
self.assertIsNone(credentials)
def test_multistore_file(self):
credentials = self._create_test_credentials()
store = multistore_file.get_credential_storage(
FILENAME,
credentials.client_id,
credentials.user_agent,
['some-scope', 'some-other-scope'])
# Save credentials
store.put(credentials)
credentials = store.get()
self.assertIsNotNone(credentials)
self.assertEquals('foo', credentials.access_token)
# Delete credentials
store.delete()
credentials = store.get()
self.assertIsNone(credentials)
if os.name == 'posix': # pragma: NO COVER
self.assertEquals(
0o600, stat.S_IMODE(os.stat(FILENAME).st_mode))
def test_multistore_file_custom_key(self):
credentials = self._create_test_credentials()
custom_key = {'myapp': 'testing', 'clientid': 'some client'}
store = multistore_file.get_credential_storage_custom_key(
FILENAME, custom_key)
store.put(credentials)
stored_credentials = store.get()
self.assertIsNotNone(stored_credentials)
self.assertEqual(credentials.access_token,
stored_credentials.access_token)
store.delete()
stored_credentials = store.get()
self.assertIsNone(stored_credentials)
def test_multistore_file_custom_string_key(self):
credentials = self._create_test_credentials()
# store with string key
store = multistore_file.get_credential_storage_custom_string_key(
FILENAME, 'mykey')
store.put(credentials)
stored_credentials = store.get()
self.assertIsNotNone(stored_credentials)
self.assertEqual(credentials.access_token,
stored_credentials.access_token)
# try retrieving with a dictionary
multistore_file.get_credential_storage_custom_string_key(
FILENAME, {'key': 'mykey'})
stored_credentials = store.get()
self.assertIsNotNone(stored_credentials)
self.assertEqual(credentials.access_token,
stored_credentials.access_token)
store.delete()
stored_credentials = store.get()
self.assertIsNone(stored_credentials)
def test_multistore_file_backwards_compatibility(self):
credentials = self._create_test_credentials()
scopes = ['scope1', 'scope2']
# store the credentials using the legacy key method
store = multistore_file.get_credential_storage(
FILENAME, 'client_id', 'user_agent', scopes)
store.put(credentials)
# retrieve the credentials using a custom key that matches the
# legacy key
key = {'clientId': 'client_id', 'userAgent': 'user_agent',
'scope': util.scopes_to_string(scopes)}
store = multistore_file.get_credential_storage_custom_key(
FILENAME, key)
stored_credentials = store.get()
self.assertEqual(credentials.access_token,
stored_credentials.access_token)
def test_multistore_file_get_all_keys(self):
# start with no keys
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
# store credentials
credentials = self._create_test_credentials(client_id='client1')
custom_key = {'myapp': 'testing', 'clientid': 'client1'}
store1 = multistore_file.get_credential_storage_custom_key(
FILENAME, custom_key)
store1.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([custom_key], keys)
# store more credentials
credentials = self._create_test_credentials(client_id='client2')
string_key = 'string_key'
store2 = multistore_file.get_credential_storage_custom_string_key(
FILENAME, string_key)
store2.put(credentials)
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals(2, len(keys))
self.assertTrue(custom_key in keys)
self.assertTrue({'key': string_key} in keys)
# back to no keys
store1.delete()
store2.delete()
keys = multistore_file.get_all_credential_keys(FILENAME)
self.assertEquals([], keys)
def _refresh_data_cache_helper(self):
multistore = multistore_file._MultiStore(FILENAME)
json_patch = mock.patch.object(multistore, '_locked_json_read')
return multistore, json_patch
def test__refresh_data_cache_bad_json(self):
multistore, json_patch = self._refresh_data_cache_helper()
with json_patch as json_mock:
json_mock.side_effect = ValueError('')
multistore._refresh_data_cache()
self.assertTrue(json_mock.called)
self.assertEqual(multistore._data, {})
def test__refresh_data_cache_bad_version(self):
multistore, json_patch = self._refresh_data_cache_helper()
with json_patch as json_mock:
json_mock.return_value = {}
multistore._refresh_data_cache()
self.assertTrue(json_mock.called)
self.assertEqual(multistore._data, {})
def test__refresh_data_cache_newer_version(self):
multistore, json_patch = self._refresh_data_cache_helper()
with json_patch as json_mock:
json_mock.return_value = {'file_version': 5}
with self.assertRaises(multistore_file.NewerCredentialStoreError):
multistore._refresh_data_cache()
self.assertTrue(json_mock.called)
def test__refresh_data_cache_bad_credentials(self):
multistore, json_patch = self._refresh_data_cache_helper()
with json_patch as json_mock:
json_mock.return_value = {
'file_version': 1,
'data': [
{'lol': 'this is a bad credential object.'}
]}
multistore._refresh_data_cache()
self.assertTrue(json_mock.called)
self.assertEqual(multistore._data, {})
def test__delete_credential_nonexistent(self):
multistore = multistore_file._MultiStore(FILENAME)
with mock.patch.object(multistore, '_write') as write_mock:
multistore._data = {}
multistore._delete_credential('nonexistent_key')
self.assertTrue(write_mock.called)