Browse Source

Setup tox.ini for unit and pep8 testing

This patch includes fixes to all the pep8 warnings.

Change-Id: I4f87e1428258824509d2d83879312a1f0dff0a8a
tags/1.2.0
Miguel Angel Ajo 1 year ago
parent
commit
5193d0010e
10 changed files with 335 additions and 88 deletions
  1. 127
    0
      .pylintrc
  2. 21
    0
      .zuul.yaml
  3. 17
    20
      oslogmerger/oslogmerger.py
  4. 2
    1
      oslogmerger/probes/netprobe.py
  5. 2
    0
      requirements.txt
  6. 35
    1
      setup.cfg
  7. 3
    66
      setup.py
  8. 11
    0
      test-requirements.txt
  9. 58
    0
      tools/coding-checks.sh
  10. 59
    0
      tox.ini

+ 127
- 0
.pylintrc View File

@@ -0,0 +1,127 @@
1
+# The format of this file isn't really documented; just use --generate-rcfile
2
+[MASTER]
3
+# Add <file or directory> to the black list. It should be a base name, not a
4
+# path. You may set this option multiple times.
5
+ignore=.git,tests
6
+
7
+[MESSAGES CONTROL]
8
+# NOTE(gus): This is a long list.  A number of these are important and
9
+# should be re-enabled once the offending code is fixed (or marked
10
+# with a local disable)
11
+disable=
12
+# "F" Fatal errors that prevent further processing
13
+ import-error,
14
+# "I" Informational noise
15
+ locally-disabled,
16
+# "E" Error for important programming issues (likely bugs)
17
+ access-member-before-definition,
18
+ anomalous-backslash-in-string,
19
+ no-member,
20
+ no-method-argument,
21
+ no-self-argument,
22
+ not-an-iterable,
23
+# "W" Warnings for stylistic problems or minor programming issues
24
+ abstract-method,
25
+ arguments-differ,
26
+ attribute-defined-outside-init,
27
+ bad-builtin,
28
+ bad-indentation,
29
+ broad-except,
30
+ dangerous-default-value,
31
+ deprecated-lambda,
32
+ expression-not-assigned,
33
+ fixme,
34
+ global-statement,
35
+ literal-comparison,
36
+ no-init,
37
+ non-parent-init-called,
38
+ not-callable,
39
+ protected-access,
40
+ redefined-builtin,
41
+ redefined-outer-name,
42
+ signature-differs,
43
+ star-args,
44
+ super-init-not-called,
45
+ super-on-old-class,
46
+ unpacking-non-sequence,
47
+ unused-argument,
48
+ unused-import,
49
+ unused-variable,
50
+ useless-super-delegation,
51
+ nonstandard-exception,
52
+# "C" Coding convention violations
53
+ bad-continuation,
54
+ consider-iterating-dictionary,
55
+ consider-using-enumerate,
56
+ invalid-name,
57
+ len-as-condition,
58
+ misplaced-comparison-constant,
59
+ missing-docstring,
60
+ singleton-comparison,
61
+ superfluous-parens,
62
+ ungrouped-imports,
63
+ wrong-import-order,
64
+# "R" Refactor recommendations
65
+ abstract-class-little-used,
66
+ abstract-class-not-used,
67
+ consider-merging-isinstance,
68
+ consider-using-ternary,
69
+ duplicate-code,
70
+ interface-not-implemented,
71
+ no-else-return,
72
+ no-self-use,
73
+ redefined-argument-from-local,
74
+ simplifiable-if-statement,
75
+ too-few-public-methods,
76
+ too-many-ancestors,
77
+ too-many-arguments,
78
+ too-many-branches,
79
+ too-many-instance-attributes,
80
+ too-many-lines,
81
+ too-many-locals,
82
+ too-many-nested-blocks,
83
+ too-many-public-methods,
84
+ too-many-return-statements,
85
+ too-many-statements
86
+
87
+[BASIC]
88
+# Variable names can be 1 to 31 characters long, with lowercase and underscores
89
+variable-rgx=[a-z_][a-z0-9_]{0,30}$
90
+
91
+# Argument names can be 2 to 31 characters long, with lowercase and underscores
92
+argument-rgx=[a-z_][a-z0-9_]{1,30}$
93
+
94
+# Method names should be at least 3 characters long
95
+# and be lowercased with underscores
96
+method-rgx=([a-z_][a-z0-9_]{2,}|setUp|tearDown)$
97
+
98
+# Don't require docstrings on tests.
99
+no-docstring-rgx=((__.*__)|([tT]est.*)|setUp|tearDown)$
100
+
101
+[FORMAT]
102
+# Maximum number of characters on a single line.
103
+max-line-length=79
104
+
105
+[VARIABLES]
106
+# List of additional names supposed to be defined in builtins. Remember that
107
+# you should avoid to define new builtins when possible.
108
+# _ is used by our localization
109
+additional-builtins=_
110
+
111
+[CLASSES]
112
+# List of interface methods to ignore, separated by a comma.
113
+ignore-iface-methods=
114
+
115
+[IMPORTS]
116
+# Deprecated modules which should not be used, separated by a comma
117
+deprecated-modules=
118
+# should use oslo_serialization.jsonutils
119
+ json
120
+
121
+[TYPECHECK]
122
+# List of module names for which member attributes should not be checked
123
+ignored-modules=six.moves,_MovedItems
124
+
125
+[REPORTS]
126
+# Tells whether to display a full report or only the messages
127
+reports=no

+ 21
- 0
.zuul.yaml View File

@@ -0,0 +1,21 @@
1
+# Copyright 2018 Red Hat, Inc.
2
+#
3
+# Licensed under the Apache License, Version 2.0 (the "License");
4
+# you may not use this file except in compliance with the License.
5
+# You may obtain a copy of the License at
6
+#
7
+#   http://www.apache.org/licenses/LICENSE-2.0
8
+#
9
+# Unless required by applicable law or agreed to in writing, software
10
+# distributed under the License is distributed on an "AS IS" BASIS,
11
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+# See the License for the specific language governing permissions and
13
+# limitations under the License.
14
+
15
+- project:
16
+    check:
17
+      jobs:
18
+        - tox-pep8
19
+    gate:
20
+      jobs:
21
+        - tox-pep8

+ 17
- 20
oslogmerger/oslogmerger.py View File

@@ -1,8 +1,6 @@
1 1
 from __future__ import print_function
2 2
 import argparse
3 3
 from datetime import datetime, timedelta
4
-import dateutil.parser
5
-import dateutil.tz
6 4
 import hashlib
7 5
 import heapq
8 6
 import itertools
@@ -12,9 +10,11 @@ import sys
12 10
 import tempfile
13 11
 import time
14 12
 
15
-from six.moves.urllib.parse import urlparse
13
+import dateutil.parser
14
+import dateutil.tz
15
+from six.moves.urllib.request import urlopen
16 16
 
17
-__version__ = '1.1.0'
17
+__version__ = '1.1.1'
18 18
 
19 19
 EXTRALINES_PADDING = " " * 40
20 20
 CACHE_DIR = "%s/oslogmerger-cache/" % tempfile.gettempdir()
@@ -62,7 +62,6 @@ FILE_MAP = {
62 62
     'lbaas-agent': 'LBAAS',
63 63
     'metadata-agent': 'META',
64 64
     'metering-agent': 'MTR',
65
-    'openvswitch-agent': 'VSWI',
66 65
     'server': 'API',
67 66
     'linuxbridge-agent': 'SVC',
68 67
     'netprobe': 'NET',
@@ -77,7 +76,6 @@ FILE_MAP = {
77 76
     'ceilometer-dbsync': 'DBSY',
78 77
     'central': 'CENT',
79 78
     'collector': 'COLL',
80
-    'compute': 'CPT',
81 79
 }
82 80
 
83 81
 
@@ -175,14 +173,14 @@ class LibvirtdParser(LogParser):
175 173
     This parser handles libvirtd.log and libvirt domain logs. Domain logs
176 174
     contain a mixture of libvirt and qemu logs, hence the 2 log formats.
177 175
     """
178
-    LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) '         # Date
179
-                         '(\d{2}):(\d{2}):(\d{2})\.(\d{3})' # Time
180
-                         '('                                #
181
-                          '([+-])(\d{2})(\d{2})'            # Timezone
182
-                         '):\s*')                           #
176
+    LIBVIRT = re.compile('(\d{4})-(\d{2})-(\d{2}) '  # Date
177
+                         '(\d{2}):(\d{2}):(\d{2})\.(\d{3})'  # Time
178
+                         '('
179
+                         '([+-])(\d{2})(\d{2})'  # Timezone
180
+                         '):\s*')
183 181
 
184
-    QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T'            # Date
185
-                      '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*') # Time
182
+    QEMU = re.compile('(\d{4})-(\d{2})-(\d{2})T'  # Date
183
+                      '(\d{2}):(\d{2}):(\d{2})\.(\d+)Z\s*')  # Time
186 184
 
187 185
     def parse_line(self, line):
188 186
         m = self.LIBVIRT.match(line)
@@ -248,11 +246,11 @@ class RawSyslog(LogParser):
248 246
     # manually anyway.
249 247
     HEADER = re.compile('<\d+>\d+\s'
250 248
                         '('
251
-                         '(\d{4})-(\d{2})-(\d{2})T'       # Date
252
-                         '(\d{2}):(\d{2}):(\d{2})\.(\d+)' # Time
253
-                         '('                              #
254
-                          '([+-])(\d{2}):(\d{2})'         # Timezone
255
-                         ')'                              #
249
+                        '(\d{4})-(\d{2})-(\d{2})T'  # Date
250
+                        '(\d{2}):(\d{2}):(\d{2})\.(\d+)'  # Time
251
+                        '('
252
+                        '([+-])(\d{2}):(\d{2})'  # Timezone
253
+                        ')'
256 254
                         ')\s*')
257 255
 
258 256
     def parse_line(self, line):
@@ -324,7 +322,7 @@ class TSLogParser(LogParser):
324 322
     def parse_line(self, line):
325 323
         end, timestamp = self._read_timestamp(line)
326 324
         dt = self.start_date + timedelta(seconds=timestamp)
327
-        dt = dt.replace(tzinfo = self.cfg.default_tz)
325
+        dt = dt.replace(tzinfo=self.cfg.default_tz)
328 326
         return dt, line[:end + 1], line[end + 1:]
329 327
 
330 328
 
@@ -453,7 +451,6 @@ DETECTED_LOG_TYPES = [
453 451
 
454 452
 
455 453
 def process_logs(cfg):
456
-    filename_alias = {}
457 454
     logs = []
458 455
 
459 456
     paths_aliases = {}

+ 2
- 1
oslogmerger/probes/netprobe.py View File

@@ -88,7 +88,7 @@ def spawn_tcpdump(interface, netns=None,
88 88
         reads, writes, excs = select.select([tcpdump.stdout], [], [])
89 89
         try:
90 90
             out = reads[0].readline()
91
-        except:
91
+        except Exception:
92 92
             continue
93 93
         if out == '':
94 94
             break
@@ -187,5 +187,6 @@ def main():
187 187
     output = open(args.output_file, 'w', 0)
188 188
     scan_loop(args)
189 189
 
190
+
190 191
 if __name__ == '__main__':
191 192
     main()

+ 2
- 0
requirements.txt View File

@@ -0,0 +1,2 @@
1
+python-dateutil
2
+six

+ 35
- 1
setup.cfg View File

@@ -1 +1,35 @@
1
-[bdist_wheel]
1
+[metadata]
2
+name = os-log-merger
3
+summary = OpenStack Log merge tool
4
+description-file =
5
+    README.rst
6
+author = OpenStack
7
+author-email = openstack-dev@lists.openstack.org
8
+home-page = https://github.com/openstack/os-log-merger
9
+classifier =
10
+    Environment :: OpenStack
11
+    Intended Audience :: Developers
12
+    Intended Audience :: System Administrators
13
+    License :: OSI Approved :: Apache Software License
14
+    Operating System :: POSIX :: Linux
15
+    Programming Language :: Python
16
+    Programming Language :: Python :: 2
17
+    Programming Language :: Python :: 2.7
18
+    Programming Language :: Python :: 3
19
+    Programming Language :: Python :: 3.5
20
+    Topic:: Utilities
21
+keywords = OpenStack log merger
22
+license = Apache Software License
23
+
24
+[files]
25
+packages =
26
+    oslogmerger
27
+
28
+[entry_points]
29
+console_scripts =
30
+    os-log-merger=oslogmerger.oslogmerger:main
31
+    oslogmerger=oslogmerger.oslogmerger:main
32
+    netprobe=oslogmerger.probes.netprobe:main
33
+
34
+[wheel]
35
+universal = 1

+ 3
- 66
setup.py View File

@@ -1,71 +1,8 @@
1
-from setuptools import setup, find_packages
2
-from codecs import open
3
-from os import path
1
+from setuptools import setup
4 2
 from oslogmerger.oslogmerger import __version__
5 3
 
6
-here = path.abspath(path.dirname(__file__))
7
-
8
-# Get the long description from the README file
9
-with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
10
-    long_description = f.read()
11
-
12 4
 setup(
13
-    name='os-log-merger',
14
-
15 5
     version=__version__,
16
-    description='OpenStack Log merge tool',
17
-    long_description=long_description,
18
-
19
-    # The project's main homepage.
20
-    url='https://github.com/mangelajo/os-log-merger',
21
-
22
-    # Author details
23
-    author='Miguel Angel Ajo, Gorka Eguileor',
24
-    author_email='majopela@redhat.com, geguileo@redhat.com',
25
-
26
-    # Choose your license
27
-    license='Apache Software License',
28
-
29
-    # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
30
-    classifiers=[
31
-        # How mature is this project? Common values are
32
-        #   3 - Alpha
33
-        #   4 - Beta
34
-        #   5 - Production/Stable
35
-        'Development Status :: 3 - Alpha',
36
-
37
-        # Indicate who your project is intended for
38
-        'Intended Audience :: Developers',
39
-
40
-        # Pick your license as you wish (should match "license" above)
41
-        'License :: OSI Approved :: Apache Software License',
42
-        'Topic :: Utilities',
43
-        'Environment :: OpenStack',
44
-
45
-        # Specify the Python versions you support here. In particular, ensure
46
-        # that you indicate whether you support Python 2, Python 3 or both.
47
-        'Programming Language :: Python :: 2.7',
48
-    ],
49
-
50
-    # What does your project relate to?
51
-    keywords='OpenStack log merger',
52
-
53
-    # You can just specify the packages manually here if your project is
54
-    # simple. Or you can use find_packages().
55
-    packages=find_packages(exclude=['contrib', 'docs', 'tests']),
56
-
57
-    py_modules=['oslogmerger'],
58
-    install_requires=[],
59
-    # extras_require={
60
-    #    'dev': ['check-manifest'],
61
-    #    'test': ['coverage'],
62
-    # },
63
-
64
-    entry_points={
65
-        'console_scripts': [
66
-            'os-log-merger=oslogmerger.oslogmerger:main',
67
-            'oslogmerger=oslogmerger.oslogmerger:main',
68
-	    'netprobe=oslogmerger.probes.netprobe:main',
69
-        ],
70
-    },
6
+    setup_requires=['pbr>=2.0.0'],
7
+    pbr=True,
71 8
 )

+ 11
- 0
test-requirements.txt View File

@@ -0,0 +1,11 @@
1
+# The order of packages is significant, because pip processes them in the order
2
+# of appearance. Changing the order has an impact on the overall integration
3
+# process, which may cause wedges in the gate later.
4
+flake8
5
+flake8-import-order==0.12 # LGPLv3
6
+mock>=2.0.0 # BSD
7
+python-subunit>=0.0.18 # Apache-2.0/BSD
8
+testtools>=1.4.0 # MIT
9
+oslotest>=1.10.0 # Apache-2.0
10
+os-testr>=1.0.0 # Apache-2.0
11
+pylint==1.4.5 # GPLv2

+ 58
- 0
tools/coding-checks.sh View File

@@ -0,0 +1,58 @@
1
+#!/bin/sh
2
+
3
+set -eu
4
+
5
+usage () {
6
+    echo "Usage: $0 [OPTION]..."
7
+    echo "Run os-log-mergers's coding check(s)"
8
+    echo ""
9
+    echo "  -Y, --pylint [<basecommit>] Run pylint check on the entire oslogmerger module or just files changed in basecommit (e.g. HEAD~1)"
10
+    echo "  -h, --help                  Print this usage message"
11
+    echo
12
+    exit 0
13
+}
14
+
15
+process_options () {
16
+    i=1
17
+    while [ $i -le $# ]; do
18
+        eval opt=\$$i
19
+        case $opt in
20
+            -h|--help) usage;;
21
+            -Y|--pylint) pylint=1;;
22
+            *) scriptargs="$scriptargs $opt"
23
+        esac
24
+        i=$((i+1))
25
+    done
26
+}
27
+
28
+run_pylint () {
29
+    local target="${scriptargs:-all}"
30
+
31
+    if [ "$target" = "all" ]; then
32
+        files="oslogmerger"
33
+    else
34
+        case "$target" in
35
+            *HEAD~[0-9]*) files=$(git diff --diff-filter=AM --name-only $target -- "*.py");;
36
+            *) echo "$target is an unrecognized basecommit"; exit 1;;
37
+        esac
38
+    fi
39
+
40
+    echo "Running pylint..."
41
+    echo "You can speed this up by running it on 'HEAD~[0-9]' (e.g. HEAD~1, this change only)..."
42
+    if [ -n "${files}" ]; then
43
+        pylint --rcfile=.pylintrc --output-format=colorized ${files}
44
+    else
45
+        echo "No python changes in this commit, pylint check not required."
46
+        exit 0
47
+    fi
48
+}
49
+
50
+scriptargs=
51
+pylint=1
52
+
53
+process_options $@
54
+
55
+if [ $pylint -eq 1 ]; then
56
+    run_pylint
57
+    exit 0
58
+fi

+ 59
- 0
tox.ini View File

@@ -0,0 +1,59 @@
1
+[tox]
2
+envlist = py35,pep8
3
+minversion = 2.3.2
4
+skipsdist = True
5
+
6
+[testenv]
7
+setenv = VIRTUAL_ENV={envdir}
8
+         PYTHONWARNINGS=default::DeprecationWarning
9
+passenv = TRACE_FAILONLY GENERATE_HASHES http_proxy HTTP_PROXY https_proxy HTTPS_PROXY no_proxy NO_PROXY
10
+usedevelop = True
11
+install_command =
12
+    pip install -U -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
13
+deps = -r{toxinidir}/requirements.txt
14
+       -r{toxinidir}/test-requirements.txt
15
+whitelist_externals = sh
16
+commands =
17
+  {toxinidir}/tools/ostestr_compat_shim.sh {posargs}
18
+# there is also secret magic in ostestr which lets you run in a fail only
19
+# mode. To do this define the TRACE_FAILONLY environmental variable.
20
+
21
+[testenv:common]
22
+# Fake job to define environment variables shared between dsvm/non-dsvm jobs
23
+setenv = OS_TEST_TIMEOUT={env:OS_TEST_TIMEOUT:180}
24
+commands = false
25
+
26
+[testenv:pep8]
27
+basepython = python2.7
28
+deps =
29
+  {[testenv]deps}
30
+commands=
31
+  flake8
32
+  sh ./tools/coding-checks.sh --pylint '{posargs}'
33
+whitelist_externals =
34
+  sh
35
+  bash
36
+
37
+[testenv:venv]
38
+commands = {posargs}
39
+
40
+[flake8]
41
+# E125 continuation line does not distinguish itself from next logical line
42
+# E126 continuation line over-indented for hanging indent
43
+# E128 continuation line under-indented for visual indent
44
+# E129 visually indented line with same indent as next logical line
45
+# E265 block comment should start with '# '
46
+# H404 multi line docstring should start with a summary
47
+# H405 multi line docstring summary not separated with an empty line
48
+# N530 direct neutron imports not allowed
49
+# TODO(ihrachys) figure out what to do with N534 and N536
50
+# N534 Untranslated exception message
51
+# N536 Use assertIsNone rather than assertEqual to check for None values
52
+ignore = E125,E126,E128,E129,E265,H404,H405,N530,N534,N536
53
+# H106: Don't put vim configuration in source files
54
+# H203: Use assertIs(Not)None to check for None
55
+# H904: Delay string interpolations at logging calls
56
+enable-extensions=H106,H203,H904
57
+show-source = true
58
+exclude = ./.*,build,dist,doc
59
+import-order-style = pep8

Loading…
Cancel
Save