style: Enable pep8 warnings/errors and fix ensuing chaos

Change-Id: I8dc2ccc148357019d822bc7e747608abc713e9e0

marconi/common/config.py

@@ -168,12 +168,12 @@ def _make_opt(name, default):
     """
 
     deduction = {
-          str: cfg.StrOpt,
+        str: cfg.StrOpt,
-         bool: cfg.BoolOpt,
+        bool: cfg.BoolOpt,
-          int: cfg.IntOpt,
+        int: cfg.IntOpt,
-         long: cfg.IntOpt,
+        long: cfg.IntOpt,
         float: cfg.FloatOpt,
-         list: cfg.ListOpt,
+        list: cfg.ListOpt,
     }
 
     if type(default) is tuple:

marconi/storage/sqlite/controllers.py

@@ -37,39 +37,51 @@ class Queue(base.QueueBase):
 
     def list(self, tenant):
         records = self.driver._run('''select name, metadata from Queues where
-                tenant = ?''', tenant)
+                                      tenant = ?''', tenant)
+
         for k, v in records:
             yield {'name': k, 'metadata': v}
 
     def get(self, name, tenant):
+        sql = '''select metadata from Queues where
+                 tenant = ? and name = ?'''
+
         try:
-            return json.loads(
+            return json.loads(self.driver._get(sql, tenant, name)[0])
-                self.driver._get('''select metadata from Queues where
-                    tenant = ? and name = ?''', tenant, name)[0])
         except TypeError:
             msg = (_("Queue %(name)s does not exist for tenant %(tenant)s")
-                    % dict(name=name, tenant=tenant))
+                   % dict(name=name, tenant=tenant))
+
             raise exceptions.DoesNotExist(msg)
 
     def upsert(self, name, metadata, tenant):
         with self.driver:
-            rc = self.driver._get('''select metadata from Queues where
+            sql_select = '''select metadata from Queues where
-                    tenant = ? and name = ?''', tenant, name) is None
+                            tenant = ? and name = ?'''
-            self.driver._run('''replace into Queues values
+            previous_record = self.driver._get(sql_select, tenant, name)
-                    (null, ?, ?, ?)''', tenant, name,
+
-                    json.dumps(metadata, ensure_ascii=False))
+            sql_replace = '''replace into Queues
-            return rc
+                             values (null, ?, ?, ?)'''
+            doc = json.dumps(metadata, ensure_ascii=False)
+            self.driver._run(sql_replace, tenant, name, doc)
+
+            return previous_record is None
 
     def delete(self, name, tenant):
         self.driver._run('''delete from Queues where
-                tenant = ? and name = ?''', tenant, name)
+                            tenant = ? and name = ?''',
+                         tenant, name)
 
     def stats(self, name, tenant):
-        return {'messages': self.driver._get('''select count(id)
+        sql = '''select count(id)
-                from Messages where
+                 from Messages where
-                qid = (select id from Queues where
+                 qid = (select id from Queues where
-                       tenant = ? and name = ?)''', tenant, name)[0],
+                       tenant = ? and name = ?)'''
-                'actions': 0}
+
+        return {
+            'messages': self.driver._get(sql, tenant, name)[0],
+            'actions': 0,
+        }
 
     def actions(self, name, tenant, marker=None, limit=10):
         pass
@@ -78,15 +90,17 @@ class Queue(base.QueueBase):
 class Message(base.MessageBase):
     def __init__(self, driver):
         self.driver = driver
-        self.driver._run('''create table if not exists Messages (
+        self.driver._run('''
-        id INTEGER,
+            create table if not exists Messages (
-        qid INTEGER,
+                id INTEGER,
-        ttl INTEGER,
+                qid INTEGER,
-        content TEXT,
+                ttl INTEGER,
-        created DATETIME,
+                content TEXT,
-        PRIMARY KEY(id),
+                created DATETIME,
-        FOREIGN KEY(qid) references Queues(id) on delete cascade
+                PRIMARY KEY(id),
-        )''')
+                FOREIGN KEY(qid) references Queues(id) on delete cascade
+            )
+        ''')
 
     def get(self, queue, tenant=None, message_id=None,
             marker=None, echo=False, client_uuid=None):
@@ -96,28 +110,34 @@ class Message(base.MessageBase):
         with self.driver:
             try:
                 qid, = self.driver._get('''select id from Queues where
-                        tenant = ? and name = ?''', tenant, queue)
+                                           tenant = ? and name = ?''',
+                                        tenant, queue)
             except TypeError:
                 msg = (_("Queue %(name)s does not exist for tenant %(tenant)s")
-                        % dict(name=queue, tenant=tenant))
+                       % dict(name=queue, tenant=tenant))
+
                 raise exceptions.DoesNotExist(msg)
 
             # executemany() sets lastrowid to None, so no matter we manually
             # generate the IDs or not, we still need to query for it.
             try:
                 unused, = self.driver._get('''select id + 1 from Messages
-                        where id = (select max(id) from Messages)''')
+                                              where id = (select max(id)
+                                              from Messages)''')
             except TypeError:
                 unused, = 1001,
 
             def it(newid):
                 for m in messages:
                     yield (newid, qid, m['ttl'],
-                            json.dumps(m, ensure_ascii=False))
+                           json.dumps(m, ensure_ascii=False))
+
                     newid += 1
 
             self.driver._run_multiple('''insert into Messages values
-                    (?, ?, ?, ?, datetime())''', it(unused))
+                                         (?, ?, ?, ?, datetime())''',
+                                      it(unused))
+
         return [str(x) for x in range(unused, unused + len(messages))]
 
     def delete(self, queue, message_id, tenant=None, claim=None):

marconi/storage/sqlite/driver.py

@@ -22,7 +22,7 @@ from marconi.storage.sqlite import controllers
 
 
 cfg = config.namespace('drivers:storage:sqlite').from_options(
-        database=':memory:')
+    database=':memory:')
 
 
 class Driver(storage.DriverBase):

marconi/tests/test_config.py

@@ -21,8 +21,8 @@ from marconi.tests import util as testing
 
 cfg_handle = config.project()
 cfg = cfg_handle.from_options(
-        without_help=3,
+    without_help=3,
-        with_help=(None, "nonsense"))
+    with_help=(None, "nonsense"))
 
 
 class TestConfig(testing.TestBase):

marconi/tests/test_sqlite.py

@@ -1,13 +1,13 @@
 # Copyright (c) 2013 Rackspace, Inc.
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
+# Licensed under the Apache License, Version 2.0 (the 'License');
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
 #    http://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
+# distributed under the License is distributed on an 'AS IS' BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
 # implied.
 # See the License for the specific language governing permissions and
@@ -28,21 +28,25 @@ class TestSqlite(testing.TestBase):
         q = storage.queue_controller
         q.upsert('fizbit', {'_message_ttl': 40}, '480924')
         m = storage.message_controller
+
         d = [
-                {"body": {
+            {
-                    "event": "BackupStarted",
+                'body': {
-                    "backupId": "c378813c-3f0b-11e2-ad92-7823d2b0f3ce"
+                    'event': 'BackupStarted',
+                    'backupId': 'c378813c-3f0b-11e2-ad92-7823d2b0f3ce',
                 },
-                'ttl': 30
+                'ttl': 30,
-                },
+            },
-                {"body": {
+            {
-                "event": "BackupProgress",
+                'body': {
-                "currentBytes": "0",
+                    'event': 'BackupProgress',
-                "totalBytes": "99614720"
+                    'currentBytes': '0',
+                    'totalBytes': '99614720',
                 },
                 'ttl': 10
-                }
+            },
-            ]
+        ]
+
         n = q.stats('fizbit', '480924')['messages']
         l1 = m.post('fizbit', d, '480924')
         l2 = m.post('fizbit', d, '480924')

tools/run_pep8.sh

@@ -5,9 +5,10 @@ set -e
 python tools/hacking.py --doctest
 
 # Until all these issues get fixed, ignore.
-PEP8='python tools/hacking.py --ignore=E12,E711,E721,E712,N303,N403,N404'
+PEP8='python tools/hacking.py --ignore=N404'
 
 EXCLUDE='--exclude=.venv,.git,.tox,dist,doc,*openstack/common*,*lib/python*'
+EXCLUDE+=',./tools'
 EXCLUDE+=',*egg,build'
 ${PEP8} ${EXCLUDE} .
 

tox.ini

@@ -19,8 +19,6 @@ downloadcache = ~/cache/pip
 
 [testenv:pep8]
 # deps = pep8==1.3.3
-# commands =
-#  pep8 --ignore=E125,E126,E711,E712 --repeat --show-source --exclude=.venv,.tox,dist,doc,openstack .
 commands = {toxinidir}/tools/run_pep8.sh
 
 [testenv:cover]