openstack
/
zaqar
Code Issues Proposed changes

style: Fix flake8 and hacking errors 

In the interest of conforming to the de-facto coding standard for
OpenStack projects, this patch enables all checks for flake8 and
hacking, plus fixes everything reported.

Also, several docstrings were edited for clarity and grammar.

Change-Id: If85c4e420811b6df8eb55de1b5f07a0a8acfceff
Implements: blueprint grizzly-debt
This commit is contained in:
kgriffs 2013-05-06 13:57:18 -04:00
parent ddb24ff5b1
commit a556f392c5
23 changed files with 129 additions and 168 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
HACKING.rst
View File

@ -124,10 +124,9 @@ Docstrings that span more than one line should look like this:
Example::
"""
Start the docstring on the line following the opening triple-double-quote
"""Single-line summary, right after the opening triple-double-quote.
If you are going to describe parameters and return values, use Sphinx, the
If you are going to describe parameters and return values, use Sphinx; the
appropriate syntax is as follows.
:param foo: the foo parameter

3
marconi/__init__.py
View File

@ -22,7 +22,8 @@ try:
except NameError:
import gettext
gettext.install("marconi", unicode=1)
from marconi.bootstrap import Bootstrap # NOQA
import marconi.bootstrap
Bootstrap = marconi.bootstrap.Bootstrap
import marconi.version

7
marconi/bootstrap.py
View File

@ -25,11 +25,10 @@ cfg = config.namespace('drivers').from_options(
class Bootstrap(object):
"""
Defines the Marconi Bootstrap
"""Defines the Marconi bootstrapper.
The bootstrap loads up drivers per a given configuration, and manages their
lifetimes.
The bootstrap loads up drivers per a given configuration, and
manages their lifetimes.
"""
def __init__(self, config_file=None, cli_args=None):

24
marconi/common/config.py
View File

@ -13,8 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Decentralized configuration module.
"""Decentralized configuration module.
A config variable `foo` is a read-only property accessible through
@ -75,8 +74,7 @@ def _init():
conf = cfg.ConfigOpts()
def namespace(name, title=None):
"""
Create a config namespace.
"""Create a config namespace.
:param name: the section name appears in the .ini file
:param title: an optional description
@ -87,8 +85,7 @@ def _init():
conf.register_group(grp)
def from_options(**opts):
"""
Define options under the associated namespace.
"""Define options under the associated namespace.
:returns: ConfigProxy of the associated namespace
"""
@ -105,16 +102,14 @@ def _init():
return Obj(from_options=from_options)
def project(name=None):
"""
Access the global namespace.
"""Access the global namespace.
:param name: the name of the project
:returns: a global option object
"""
def from_options(**opts):
"""
Define options under the global namespace.
"""Define options under the global namespace.
:returns: ConfigProxy of the global namespace
"""
@ -170,11 +165,12 @@ namespace, project = _init()
def _make_opt(name, default):
"""
Create an oslo.config option with the type deduce from the %default
value of an option %name.
"""Create an oslo.config option with type deduction
A default value of None is deduced to Opt. MultiStrOpt is not supported.
The type for the option is deduced from the %default value given
for that option. A default value of None is deduced to Opt.
Note: MultiStrOpt is not supported.
:param name: the name of the option in a valid Python identifier
:param default: the default value of the option, or (default, description)

12
marconi/storage/__init__.py
View File

@ -1,7 +1,11 @@
"""Marconi Storage Drivers"""
from marconi.storage.base import ClaimBase # NOQA
from marconi.storage.base import DriverBase # NOQA
from marconi.storage.base import MessageBase # NOQA
from marconi.storage.base import QueueBase # NOQA
from marconi.storage import base
from marconi.storage import exceptions # NOQA
# Hoist classes into package namespace
ClaimBase = base.ClaimBase
DriverBase = base.DriverBase
MessageBase = base.MessageBase
QueueBase = base.QueueBase

76
marconi/storage/base.py
View File

@ -23,29 +23,22 @@ class DriverBase:
@abc.abstractproperty
def queue_controller(self):
"""
Returns storage's queues controller
"""
"""Returns storage's queues controller."""
raise NotImplementedError
@abc.abstractproperty
def message_controller(self):
"""
Returns storage's messages controller
"""
"""Returns storage's messages controller."""
raise NotImplementedError
@abc.abstractproperty
def claim_controller(self):
"""
Returns storage's claims controller
"""
"""Returns storage's claims controller."""
raise NotImplementedError
class ControllerBase(object):
"""
Top level class for controllers.
"""Top-level class for controllers.
:param driver: Instance of the driver
instantiating this controller.
@ -56,14 +49,13 @@ class ControllerBase(object):
class QueueBase(ControllerBase):
"""
This class is responsible of managing
queues which means handling their CRUD
operations, monitoring and interactions.
"""This class is responsible for managing queues.
Storages' implementations of this class
should be capable of handling high work
loads and huge number of queues.
Queue operations include CRUD, monitoring, etc.
Storage driver implementations of this class should
be capable of handling high workloads and huge
numbers of queues.
"""
__metaclass__ = abc.ABCMeta
@ -71,8 +63,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def list(self, project=None, marker=None,
limit=10, detailed=False):
"""
Base method for listing queues.
"""Base method for listing queues.
:param project: Project id
:param marker: The last queue name
@ -86,8 +77,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def get(self, name, project=None):
"""
Base method for queue retrieval.
"""Base method for queue retrieval.
:param name: The queue name
:param project: Project id
@ -99,9 +89,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def upsert(self, name, metadata, project=None):
"""
This methods handles both creates and updates
operations for queues.
"""This method handles both create and update operations.
:param name: The queue name
:param metadata: Arbitrary metadata
@ -114,8 +102,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def delete(self, name, project=None):
"""
Base method for queue deletion.
"""Base method for deleting a queue.
:param name: The queue name
:param project: Project id
@ -124,8 +111,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def stats(self, name, project=None):
"""
Base method for queue stats.
"""Base method for queue stats.
:param name: The queue name
:param project: Project id
@ -136,8 +122,7 @@ class QueueBase(ControllerBase):
@abc.abstractmethod
def actions(self, name, project=None, marker=None, limit=10):
"""
Base method for queue actions.
"""Base method for queue actions.
:param name: Queue name
:param project: Project id
@ -149,18 +134,14 @@ class QueueBase(ControllerBase):
class MessageBase(ControllerBase):
"""
This class is responsible for managing
messages CRUD.
"""
"""This class is responsible for managing message CRUD."""
__metaclass__ = abc.ABCMeta
@abc.abstractmethod
def list(self, queue, project=None, marker=None,
limit=10, echo=False, client_uuid=None):
"""
Base message list method
"""Base method for listing messages.
:param queue: Name of the queue to get the
message from.
@ -179,8 +160,7 @@ class MessageBase(ControllerBase):
raise NotImplementedError
def get(self, queue, message_id, project=None):
"""
Base message get method
"""Base method for getting a message.
:param queue: Name of the queue to get the
message from.
@ -194,8 +174,7 @@ class MessageBase(ControllerBase):
@abc.abstractmethod
def post(self, queue, messages, client_uuid, project=None):
"""
Base message post method
"""Base method for posting one or more messages.
Implementations of this method should guarantee
and preserve the order, in the returned list, of
@ -213,8 +192,7 @@ class MessageBase(ControllerBase):
@abc.abstractmethod
def delete(self, queue, message_id, project=None, claim=None):
"""
Base message delete method
"""Base method for deleting a single message.
:param queue: Name of the queue to post
message to.
@ -234,8 +212,7 @@ class ClaimBase(ControllerBase):
@abc.abstractmethod
def get(self, queue, claim_id, project=None):
"""
Base claim get method
"""Base method for getting a claim.
:param queue: Name of the queue this
claim belongs to.
@ -249,8 +226,7 @@ class ClaimBase(ControllerBase):
@abc.abstractmethod
def create(self, queue, metadata, project=None, limit=10):
"""
Base claim create method
"""Base method for creating a claim.
:param queue: Name of the queue this
claim belongs to.
@ -266,8 +242,7 @@ class ClaimBase(ControllerBase):
@abc.abstractmethod
def update(self, queue, claim_id, metadata, project=None):
"""
Base claim update method
"""Base method for updating a claim.
:param queue: Name of the queue this
claim belongs to.
@ -280,8 +255,7 @@ class ClaimBase(ControllerBase):
@abc.abstractmethod
def delete(self, queue, claim_id, project=None):
"""
Base claim delete method
"""Base method for deleting a claim.
:param queue: Name of the queue this
claim belongs to.

6
marconi/storage/mongodb/__init__.py
View File

@ -1,3 +1,7 @@
"""MongoDB Storage Driver for Marconi"""
from marconi.storage.mongodb.base import Driver # NOQA
from marconi.storage.mongodb import driver
# Hoist classes into package namespace
Driver = driver.Driver

20
marconi/storage/mongodb/controllers.py
View File

@ -13,10 +13,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Mongodb storage controllers implementation
"""Implements Mongodb storage controllers.
Fields Mapping:
Field Mappings:
In order to reduce the disk / memory space used,
fields name will be, most of the time, the first
letter of their long name. Fields mapping will be
@ -34,7 +33,8 @@ from marconi.storage.mongodb import utils
class QueueController(storage.QueueBase):
"""
"""Implements queue resource operations using MongoDB.
Queues:
Name Field
----------------
@ -87,8 +87,7 @@ class QueueController(storage.QueueBase):
return queue
def get_id(self, name, project=None):
"""
Just like `get` method but returns the queue's id
"""Just like the `get` method, but only returns the queue's id
:returns: Queue's `ObjectId`
"""
@ -133,7 +132,8 @@ class QueueController(storage.QueueBase):
class MessageController(storage.MessageBase):
"""
"""Implements message resource operations using MongoDB.
Messages:
Name Field
----------------
@ -374,7 +374,8 @@ class MessageController(storage.MessageBase):
class ClaimController(storage.ClaimBase):
"""
"""Implements claim resource operations using MongoDB.
No dedicated collection is being used
for claims.
@ -440,7 +441,8 @@ class ClaimController(storage.ClaimBase):
return (claim, messages)
def create(self, queue, metadata, project=None, limit=10):
"""
"""Creates a claim.
This implementation was done in a best-effort fashion.
In order to create a claim we need to get a list
of messages that can be claimed. Once we have that

9
marconi/storage/mongodb/base.py → marconi/storage/mongodb/driver.py
View File

@ -13,9 +13,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Mongodb storage driver implementation
"""
"""Mongodb storage driver implementation."""
import pymongo
import pymongo.errors
@ -39,10 +37,7 @@ class Driver(storage.DriverBase):
@property
def db(self):
"""
Property for lazy instantiation of
mongodb's database.
"""
"""Property for lazy instantiation of mongodb's database."""
if not self._database:
if cfg.uri and 'replicaSet' in cfg.uri:
conn = pymongo.MongoReplicaSetClient(cfg.uri)

15
marconi/storage/mongodb/utils.py
View File

@ -20,14 +20,14 @@ from marconi.openstack.common import timeutils
def to_oid(obj):
"""
Creates a new ObjectId based on the input
and raises ValueError whenever a TypeError
or InvalidId error is raised by the ObjectID
class.
"""Creates a new ObjectId based on the input.
Raises ValueError when TypeError or InvalidId
is raised by the ObjectID class.
:param obj: Anything that can be passed as an
input to `objectid.ObjectId`
:raises: ValueError
"""
try:
return objectid.ObjectId(obj)
@ -37,10 +37,7 @@ def to_oid(obj):
def oid_utc(oid):
"""
Creates a non-tz-aware datetime based on
the incoming objectid's datetime information.
"""
"""Converts an ObjectId to a non-tz-aware datetime."""
try:
return timeutils.normalize_time(oid.generation_time)
except AttributeError:

5
marconi/storage/sqlite/__init__.py
View File

@ -4,4 +4,7 @@ In-memory reference Storage Driver for Marconi.
Useful for automated testing and for prototyping storage driver concepts.
"""
from marconi.storage.sqlite.driver import Driver # NOQA
from marconi.storage.sqlite import driver
# Hoist classes into package namespace
Driver = driver.Driver

22
marconi/storage/sqlite/driver.py
View File

@ -39,9 +39,7 @@ class Driver(storage.DriverBase):
@staticmethod
def pack(o):
"""
Convert a Python variable to a SQlite variable
with the customized type `DOCUMENT`.
"""Converts a Python variable to a custom SQlite `DOCUMENT`.
:param o: a Python str, unicode, int, long, float, bool, None
or a dict or list of %o
@ -52,8 +50,7 @@ class Driver(storage.DriverBase):
msgpack.loads(s, encoding='utf-8'))
def run(self, sql, *args):
"""
Perform a SQL query.
"""Performs a SQL query.
:param sql: a query string with the '?' placeholders
:param args: the arguments to substitute the placeholders
@ -61,8 +58,7 @@ class Driver(storage.DriverBase):
return self.__db.execute(sql, args)
def run_multiple(self, sql, it):
"""
Iteratively perform multiple SQL queries.
"""Iteratively perform multiple SQL queries.
:param sql: a query string with the '?' placeholders
:param it: an iterator which yields a sequence of arguments to
@ -71,8 +67,7 @@ class Driver(storage.DriverBase):
self.__db.executemany(sql, it)
def get(self, sql, *args):
"""
Get one entry from the query result.
"""Runs %sql and returns the first entry in the results.
:param sql: a query string with the '?' placeholders
:param args: the arguments to substitute the placeholders
@ -86,18 +81,13 @@ class Driver(storage.DriverBase):
@property
def affected(self):
"""
Check whether a row is affected in
the last operation.
"""
"""Checks whether a row is affected in the last operation."""
assert self.__db.rowcount in (0, 1)
return self.__db.rowcount == 1
@property
def lastrowid(self):
"""
Get last inserted row id.
"""
"""Returns the last inserted row id."""
return self.__db.lastrowid
@contextlib.contextmanager

29
marconi/tests/storage/base.py
View File

@ -17,6 +17,7 @@
from marconi import storage
from marconi.storage import exceptions
from marconi.tests import util as testing
from marconi.tests.util import helpers
class ControllerBaseTest(testing.TestBase):
@ -40,9 +41,7 @@ class ControllerBaseTest(testing.TestBase):
class QueueControllerTest(ControllerBaseTest):
"""
Queue Controller base tests
"""
"""Queue Controller base tests."""
controller_base_class = storage.QueueBase
def setUp(self):
@ -109,8 +108,7 @@ class QueueControllerTest(ControllerBaseTest):
class MessageControllerTest(ControllerBaseTest):
"""
Message Controller base tests
"""Message Controller base tests.
NOTE(flaper87): Implementations of this class should
override the tearDown method in order
@ -203,7 +201,7 @@ class MessageControllerTest(ControllerBaseTest):
[msg1, msg2] = msgs
# A wrong claim does not ensure the message deletion
with testing.expected(storage.exceptions.NotPermitted):
with helpers.expected(storage.exceptions.NotPermitted):
self.controller.delete(self.queue_name, msg1["id"],
project=self.project,
claim=another_cid)
@ -213,7 +211,7 @@ class MessageControllerTest(ControllerBaseTest):
project=self.project,
claim=cid)
with testing.expected(storage.exceptions.DoesNotExist):
with helpers.expected(storage.exceptions.DoesNotExist):
self.controller.get(self.queue_name, msg1["id"],
project=self.project)
@ -226,7 +224,7 @@ class MessageControllerTest(ControllerBaseTest):
self.claim_controller.delete(self.queue_name, cid,
project=self.project)
with testing.expected(storage.exceptions.NotPermitted):
with helpers.expected(storage.exceptions.NotPermitted):
self.controller.delete(self.queue_name, msg2["id"],
project=self.project,
claim=cid)
@ -238,7 +236,7 @@ class MessageControllerTest(ControllerBaseTest):
project=self.project,
client_uuid='my_uuid')
with testing.expected(storage.exceptions.DoesNotExist):
with helpers.expected(storage.exceptions.DoesNotExist):
self.controller.get(self.queue_name, msgid,
project=self.project)
@ -257,7 +255,7 @@ class MessageControllerTest(ControllerBaseTest):
self.assertEquals(len(msgs), 0)
with testing.expected(exceptions.DoesNotExist):
with helpers.expected(exceptions.DoesNotExist):
self.controller.get('unused', 'illformed', '480924')
def test_illformed_claim(self):
@ -267,15 +265,14 @@ class MessageControllerTest(ControllerBaseTest):
project='480924',
client_uuid='unused')
with testing.expected(exceptions.NotPermitted):
with helpers.expected(exceptions.NotPermitted):
self.controller.delete('unused', msgid,
project='480924',
claim='illformed')
class ClaimControllerTest(ControllerBaseTest):
"""
Claim Controller base tests
"""Claim Controller base tests.
NOTE(flaper87): Implementations of this class should
override the tearDown method in order
@ -357,11 +354,11 @@ class ClaimControllerTest(ControllerBaseTest):
claim_id, messages = self.controller.create(self.queue_name, meta,
project=self.project)
with testing.expected(storage.exceptions.DoesNotExist):
with helpers.expected(storage.exceptions.DoesNotExist):
self.controller.get(self.queue_name, claim_id,
project=self.project)
with testing.expected(storage.exceptions.DoesNotExist):
with helpers.expected(storage.exceptions.DoesNotExist):
self.controller.update(self.queue_name, claim_id,
meta, project=self.project)
@ -371,7 +368,7 @@ class ClaimControllerTest(ControllerBaseTest):
self.queue_controller.upsert('unused', {}, '480924')
self.controller.delete('unused', 'illformed', '480924')
with testing.expected(exceptions.DoesNotExist):
with helpers.expected(exceptions.DoesNotExist):
self.controller.update('unused', 'illformed',
{'ttl': 40}, '480924')

8
marconi/tests/storage/test_impl_mongodb.py
View File

@ -111,10 +111,10 @@ class MongodbClaimTests(base.ClaimControllerTest):
self.load_conf("wsgi_mongodb.conf")
def test_claim_doesnt_exist(self):
"""
Tests that methods raise an exception when
the claim doesn't exists and / or
has expired.
"""Verifies that operations fail on expired/missing claims.
Methods should raise an exception when the claim doesn't
exists and/or has expired.
"""
epoch = '000000000000000000000000'
self.assertRaises(storage.exceptions.ClaimDoesNotExist,

3
marconi/tests/test_config.py
View File

@ -15,6 +15,7 @@
from marconi.common import config
from marconi.tests import util as testing
from marconi.tests.util import helpers
cfg_handle = config.project()
@ -35,5 +36,5 @@ class TestConfig(testing.TestBase):
def test_wrong_type(self):
ns = config.namespace('local')
with testing.expected(config.cfg.Error):
with helpers.expected(config.cfg.Error):
ns.from_options(opt={})

6
marconi/tests/util/__init__.py
View File

@ -1,4 +1,6 @@
"""Test utilities"""
from marconi.tests.util.base import TestBase # NOQA
from marconi.tests.util.helpers import expected # NOQA
from marconi.tests.util import base
TestBase = base.TestBase

9
marconi/tests/util/base.py
View File

@ -23,8 +23,7 @@ cfg = config.project()
class TestBase(testtools.TestCase):
"""
Child class of testtools.TestCase for testing Marconi
"""Child class of testtools.TestCase for testing Marconi.
Inherit from this and write your test methods. If the child class defines
a prepare(self) method, this method will be called before executing each
@ -32,8 +31,7 @@ class TestBase(testtools.TestCase):
"""
def conf_path(self, filename):
"""
Returns the full path to the specified Marconi conf file
"""Returns the full path to the specified Marconi conf file.
:param filename: Name of the conf file to find (e.g.,
"wsgi_memory.conf")
@ -43,8 +41,7 @@ class TestBase(testtools.TestCase):
return os.path.join(parent, 'etc', filename)
def load_conf(self, filename):
"""
Loads `filename` configuration file.
"""Loads `filename` configuration file.
:param filename: Name of the conf file to find (e.g.,
"wsgi_memory.conf")

6
marconi/transport/__init__.py
View File

@ -1,7 +1,11 @@
"""Marconi Transport Drivers"""
from marconi.transport import base
MAX_QUEUE_METADATA_SIZE = 64 * 1024
"""Maximum metadata size per queue when serialized as JSON"""
from marconi.transport.base import DriverBase # NOQA
# Hoist into package namespace
DriverBase = base.DriverBase

5
marconi/transport/base.py
View File

@ -23,8 +23,5 @@ class DriverBase:
@abc.abstractmethod
def listen():
"""
Called to start listening for client requests when Marconi is
ran in self-hosting mode.
"""
"""Start listening for client requests (self-hosting mode)."""
raise NotImplementedError

16
marconi/transport/helpers.py
View File

@ -21,8 +21,7 @@ class MalformedJSON(Exception):
def read_json(stream):
"""
Like json.load, but raises an exception upon failure.
"""Like json.load, but raises an exception upon failure.
:param stream: a file-like object
"""
@ -34,19 +33,16 @@ def read_json(stream):
def to_json(obj):
"""
Like json.dumps, but outputs a UTF-8 encoded string.
"""Like json.dumps, but outputs a UTF-8 encoded string.
:param obj: a JSON-serializable object
"""
return json.dumps(obj, ensure_ascii=False)
def purge(d):
"""
Remove entries with a value of None from a dict by
returning a purged copy.
def purge(src):
"""Returns a copy of a dict, excluding any keys set to `None`.
:param d: a dictionary object
:param src: a dictionary-like object to copy
"""
return dict([(k, v) for k, v in d.items() if v is not None])
return dict([(k, v) for k, v in src.items() if v is not None])

6
marconi/transport/wsgi/__init__.py
View File

@ -1,7 +1,11 @@
"""WSGI Transport Driver"""
from marconi.transport.wsgi import claims # NOQA
from marconi.transport.wsgi.driver import Driver # NOQA
from marconi.transport.wsgi import driver
from marconi.transport.wsgi import messages # NOQA
from marconi.transport.wsgi import queues # NOQA
from marconi.transport.wsgi import stats # NOQA
# Hoist into package namespace
Driver = driver.Driver

4
tools/install_venv.py
View File

@ -22,7 +22,7 @@
import os
import sys
import install_venv_common as install_venv
import marconi.tools.install_venv_common as install_venv
def print_help(venv, root):
@ -61,7 +61,7 @@ def main(argv):
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'Marconi'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
py_version, project)
options = install.parse_args(argv)
install.check_python_version()
install.check_dependencies()

1
tox.ini
View File

@ -27,6 +27,5 @@ setenv = NOSE_WITH_COVERAGE=1
commands = {posargs}
[flake8]
ignore = E128,H302,H304,H404
builtins = _,__MARCONI_SETUP__
exclude = .venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*