From cedadccc6f9129f4466ba30eac0e83c554562813 Mon Sep 17 00:00:00 2001 From: Andreas Jaeger Date: Sun, 29 Mar 2020 17:51:44 +0200 Subject: [PATCH] Update hacking for Python3 The repo is Python 3 now, so update hacking to version 3.0 which supports Python 3. Fix problems found. Update local hacking checks for new flake8. Change-Id: I6396403d0a62f5403fc5b7fb04b6ce790c332c84 --- docker/collector/health_check.py | 2 +- docker/forwarder/health_check.py | 2 +- docker/statsd/health_check.py | 2 +- monasca_agent/collector/checks_d/couchbase.py | 8 ++++---- monasca_agent/collector/checks_d/cpu.py | 2 +- monasca_agent/collector/checks_d/hdfs.py | 1 + .../collector/checks_d/supervisord.py | 3 ++- monasca_agent/collector/checks_d/vcenter.py | 2 +- monasca_agent/collector/daemon.py | 1 + monasca_agent/common/daemon.py | 2 +- monasca_agent/common/util.py | 4 ++-- monasca_agent/forwarder/api/monasca_api.py | 8 ++++---- monasca_agent/forwarder/daemon.py | 1 + monasca_agent/hacking/checks.py | 20 +++++++++---------- monasca_setup/detection/plugins/haproxy.py | 4 ++-- monasca_setup/detection/plugins/keystone.py | 2 +- monasca_setup/detection/plugins/mon.py | 2 +- test-requirements.txt | 2 +- tox.ini | 15 +++++++++++--- 19 files changed, 48 insertions(+), 35 deletions(-) diff --git a/docker/collector/health_check.py b/docker/collector/health_check.py index 8e83b3fe..c49c843f 100755 --- a/docker/collector/health_check.py +++ b/docker/collector/health_check.py @@ -20,7 +20,7 @@ def main(): """Health check for Monasca-agent collector.""" - # TODO wait for health check endpoint ... + # TODO(Dobroslaw Zybort) wait for health check endpoint ... return 0 diff --git a/docker/forwarder/health_check.py b/docker/forwarder/health_check.py index 54fdfa39..17a7c7d7 100644 --- a/docker/forwarder/health_check.py +++ b/docker/forwarder/health_check.py @@ -20,7 +20,7 @@ def main(): """Health check for Monasca-agent forwarder.""" - # TODO wait for health check endpoint ... + # TODO(Dobroslaw Zybort) wait for health check endpoint ... return 0 diff --git a/docker/statsd/health_check.py b/docker/statsd/health_check.py index d5de7dc1..93a1ea5d 100644 --- a/docker/statsd/health_check.py +++ b/docker/statsd/health_check.py @@ -20,7 +20,7 @@ def main(): """Health check for Monasca-agent statsd.""" - # TODO wait for health check endpoint ... + # TODO(Michał Piotrowski) wait for health check endpoint ... return 0 diff --git a/monasca_agent/collector/checks_d/couchbase.py b/monasca_agent/collector/checks_d/couchbase.py index 7ef10a98..cac42f82 100644 --- a/monasca_agent/collector/checks_d/couchbase.py +++ b/monasca_agent/collector/checks_d/couchbase.py @@ -139,15 +139,15 @@ class Couchbase(AgentCheck): # Returns input if non-camelCase variable is detected. def camel_case_to_joined_lower(self, variable): # replace non-word with _ - converted_variable = re.sub('\W+', '_', variable) + converted_variable = re.sub(r'\W+', '_', variable) # insert _ in front of capital letters and lowercase the string - converted_variable = re.sub('([A-Z])', '_\g<1>', converted_variable).lower() + converted_variable = re.sub(r'([A-Z])', r'_\g<1>', converted_variable).lower() # remove duplicate _ - converted_variable = re.sub('_+', '_', converted_variable) + converted_variable = re.sub(r'_+', '_', converted_variable) # handle special case of starting/ending underscores - converted_variable = re.sub('^_|_$', '', converted_variable) + converted_variable = re.sub(r'^_|_$', '', converted_variable) return converted_variable diff --git a/monasca_agent/collector/checks_d/cpu.py b/monasca_agent/collector/checks_d/cpu.py index 3eaef2d9..ff9dd245 100644 --- a/monasca_agent/collector/checks_d/cpu.py +++ b/monasca_agent/collector/checks_d/cpu.py @@ -85,7 +85,7 @@ class Cpu(checks.AgentCheck): 'lscpu', stdout=subprocess.PIPE, stderr=subprocess.PIPE) lscpu_output = lscpu_command.communicate()[0].decode( encoding='UTF-8') - cpu_freq_output = re.search("(CPU MHz:.*?(\d+\.\d+)\n)", lscpu_output) + cpu_freq_output = re.search(r"(CPU MHz:.*?(\d+\.\d+)\n)", lscpu_output) cpu_freq = float(cpu_freq_output.group(2)) data['cpu.frequency_mhz'] = cpu_freq except Exception: diff --git a/monasca_agent/collector/checks_d/hdfs.py b/monasca_agent/collector/checks_d/hdfs.py index c5103c5f..43592a64 100644 --- a/monasca_agent/collector/checks_d/hdfs.py +++ b/monasca_agent/collector/checks_d/hdfs.py @@ -55,6 +55,7 @@ class HDFSCheck(AgentCheck): self.gauge('hdfs.missing_blocks', stats['missing_blocks'], dimensions=dimensions) self.gauge('hdfs.corrupt_blocks', stats['corrupt_blocks'], dimensions=dimensions) + if __name__ == '__main__': check, instances = HDFSCheck.from_yaml('./hdfs.yaml') for instance in instances: diff --git a/monasca_agent/collector/checks_d/supervisord.py b/monasca_agent/collector/checks_d/supervisord.py index 39482038..48f2ad18 100644 --- a/monasca_agent/collector/checks_d/supervisord.py +++ b/monasca_agent/collector/checks_d/supervisord.py @@ -67,6 +67,7 @@ PROCESS_TAG = 'supervisord_process' def _format_time(x): return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(x)) + SERVER_SERVICE_CHECK = 'supervisord.can_connect' PROCESS_SERVICE_CHECK = 'supervisord.process.status' PROCESS_UP_TIME_CHECK = 'supervisord.process.uptime' @@ -101,7 +102,7 @@ class Supervisord(checks.AgentCheck): 'An error occurred while reading process information: %s %s' % (error.faultCode, error.faultString) ) - except socket.error as error: + except socket.error: host = instance.get('host', DEFAULT_HOST) port = instance.get('port', DEFAULT_PORT) sock = instance.get('socket') diff --git a/monasca_agent/collector/checks_d/vcenter.py b/monasca_agent/collector/checks_d/vcenter.py index 22026758..26f4ef02 100644 --- a/monasca_agent/collector/checks_d/vcenter.py +++ b/monasca_agent/collector/checks_d/vcenter.py @@ -617,7 +617,7 @@ class VcenterOperations(object): instance = perf_metric_series_csv.id.instance if (instance is not None and len(instance) > 0 and - instance is not "*"): + instance != "*"): name += "." + instance perf_result[name] = perf_metric_series_csv.value diff --git a/monasca_agent/collector/daemon.py b/monasca_agent/collector/daemon.py index 3db2fa72..f846783e 100644 --- a/monasca_agent/collector/daemon.py +++ b/monasca_agent/collector/daemon.py @@ -339,6 +339,7 @@ def run_check(check): check.get_metrics(prettyprint=True) print("#" * 80 + "\n\n") + if __name__ == '__main__': try: sys.exit(main()) diff --git a/monasca_agent/common/daemon.py b/monasca_agent/common/daemon.py index 418f2e03..8ccdb766 100644 --- a/monasca_agent/common/daemon.py +++ b/monasca_agent/common/daemon.py @@ -149,7 +149,7 @@ class Daemon(object): fp.write(str(pid)) fp.close() os.chmod(self.pidfile, 0o644) - except Exception as e: + except Exception: msg = "Unable to write pidfile: %s" % self.pidfile log.exception(msg) sys.stderr.write(msg + "\n") diff --git a/monasca_agent/common/util.py b/monasca_agent/common/util.py index 1052d4de..bc9b553b 100644 --- a/monasca_agent/common/util.py +++ b/monasca_agent/common/util.py @@ -41,7 +41,7 @@ log = logging.getLogger(__name__) VALID_HOSTNAME_RFC_1123_PATTERN = re.compile( r"^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*" - "([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$") + r"([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$") MAX_HOSTNAME_LEN = 255 LOGGING_MAX_BYTES = 5 * 1024 * 1024 @@ -571,7 +571,7 @@ def load_check_directory(): try: c = check_class(check_name, init_config=init_config, agent_config=agent_config, instances=instances) - except TypeError as e: + except TypeError: # Backwards compatibility for checks which don't support the # instances argument in the constructor. c = check_class(check_name, init_config=init_config, diff --git a/monasca_agent/forwarder/api/monasca_api.py b/monasca_agent/forwarder/api/monasca_api.py index 836331c3..54bcdfc3 100644 --- a/monasca_agent/forwarder/api/monasca_api.py +++ b/monasca_agent/forwarder/api/monasca_api.py @@ -153,10 +153,10 @@ class MonascaAPI(object): self._mon_client.metrics.create(**kwargs) return True except exceptions.ClientException as ex: - log.exception("ClientException: error sending " - "message to monasca-api.") - self._failure_reason = ('Error sending message to ' - 'the Monasca API: {0}').format(str(ex)) + log.exception("ClientException: error sending " + "message to monasca-api.") + self._failure_reason = ('Error sending message to ' + 'the Monasca API: {0}').format(str(ex)) except Exception: log.exception("Error sending message to Monasca API.") self._failure_reason = 'The Monasca API is DOWN or unreachable' diff --git a/monasca_agent/forwarder/daemon.py b/monasca_agent/forwarder/daemon.py index 91476315..01b866b0 100644 --- a/monasca_agent/forwarder/daemon.py +++ b/monasca_agent/forwarder/daemon.py @@ -244,5 +244,6 @@ def main(): return -1 return 0 + if __name__ == "__main__": sys.exit(main()) diff --git a/monasca_agent/hacking/checks.py b/monasca_agent/hacking/checks.py index 81b0334f..e095298d 100644 --- a/monasca_agent/hacking/checks.py +++ b/monasca_agent/hacking/checks.py @@ -11,8 +11,11 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. + import re +from hacking import core + assert_no_xrange_re = re.compile(r"\s*xrange\s*\(") assert_True = re.compile(r".*assertEqual\(True, .*\)") assert_None = re.compile(r".*assertEqual\(None, .*\)") @@ -23,17 +26,20 @@ no_log_warn = re.compile(r".*LOG.warn\(.*\)") mutable_default_args = re.compile(r"^\s*def .+\((.+=\{\}|.+=\[\])") +@core.flake8ext def no_mutable_default_args(logical_line): msg = "M001: Method's default argument shouldn't be mutable!" if mutable_default_args.match(logical_line): yield (0, msg) +@core.flake8ext def no_xrange(logical_line): if assert_no_xrange_re.match(logical_line): yield (0, "M002: Do not use xrange().") +@core.flake8ext def validate_assertTrue(logical_line): if re.match(assert_True, logical_line): msg = ("M003: Unit tests should use assertTrue(value) instead" @@ -41,6 +47,7 @@ def validate_assertTrue(logical_line): yield (0, msg) +@core.flake8ext def validate_assertIsNone(logical_line): if re.match(assert_None, logical_line): msg = ("M004: Unit tests should use assertIsNone(value) instead" @@ -48,12 +55,14 @@ def validate_assertIsNone(logical_line): yield (0, msg) +@core.flake8ext def no_log_warn_check(logical_line): if re.match(no_log_warn, logical_line): msg = ("M005: LOG.warn is deprecated, please use LOG.warning!") yield (0, msg) +@core.flake8ext def validate_assertIsNotNone(logical_line): if re.match(assert_Not_Equal, logical_line) or \ re.match(assert_Is_Not, logical_line): @@ -63,18 +72,9 @@ def validate_assertIsNotNone(logical_line): yield (0, msg) +@core.flake8ext def assert_raisesRegexp(logical_line): res = assert_raises_regexp.search(logical_line) if res: yield (0, "M007: assertRaisesRegex must be used instead " "of assertRaisesRegexp") - - -def factory(register): - register(no_mutable_default_args) - register(no_xrange) - register(validate_assertTrue) - register(validate_assertIsNone) - register(no_log_warn_check) - register(validate_assertIsNotNone) - register(assert_raisesRegexp) diff --git a/monasca_setup/detection/plugins/haproxy.py b/monasca_setup/detection/plugins/haproxy.py index 5239387d..e54be5ac 100644 --- a/monasca_setup/detection/plugins/haproxy.py +++ b/monasca_setup/detection/plugins/haproxy.py @@ -60,7 +60,7 @@ class HAProxy(monasca_setup.detection.Plugin): password = None for line in proxy_cfg.splitlines(): if line.startswith('listen'): - listen_match = re.search('^listen.*stats\S*\s(.*)', line) + listen_match = re.search(r'^listen.*stats\S*\s(.*)', line) if listen_match is None: continue listen_socket = listen_match.group(1).split(':', 1) @@ -71,7 +71,7 @@ class HAProxy(monasca_setup.detection.Plugin): port = listen_socket[1].strip() url = 'http://{0}:{1}'.format(host, port) if url is not None and line.strip().startswith('stats auth'): - auth = re.search('stats auth\s(.*)', line).group(1).split(':') + auth = re.search(r'stats auth\s(.*)', line).group(1).split(':') user = auth[0].strip() password = auth[1].strip() diff --git a/monasca_setup/detection/plugins/keystone.py b/monasca_setup/detection/plugins/keystone.py index b00ab266..b4a1f5d4 100644 --- a/monasca_setup/detection/plugins/keystone.py +++ b/monasca_setup/detection/plugins/keystone.py @@ -30,7 +30,7 @@ class Keystone(monasca_setup.detection.ServicePlugin): 'keystone-main', 'keystone-all'], 'service_api_url': 'http://localhost:35357/v3', - 'search_pattern': '.*v3\..*' + 'search_pattern': r'.*v3\..*' } super(Keystone, self).__init__(service_params) diff --git a/monasca_setup/detection/plugins/mon.py b/monasca_setup/detection/plugins/mon.py index 34bb26b7..b00bab11 100644 --- a/monasca_setup/detection/plugins/mon.py +++ b/monasca_setup/detection/plugins/mon.py @@ -362,7 +362,7 @@ class _DropwizardJavaHelper(object): * monasca-persister [**Java**] """ - YAML_PATTERN = re.compile('.*\.ya?ml', re.IGNORECASE) + YAML_PATTERN = re.compile(r'.*\.ya?ml', re.IGNORECASE) def __init__(self, cmdline=None): self._cmdline = cmdline diff --git a/test-requirements.txt b/test-requirements.txt index 33a12711..3623f4d4 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,7 +1,7 @@ # The order of packages is significant, because pip processes them in the order # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. -hacking>=1.1.0,<1.2.0 # Apache-2.0 +hacking>=3.0,<3.1.0 # Apache-2.0 bandit!=1.6.0,>=1.1.0 # Apache-2.0 mock>=2.0.0 # BSD coverage!=4.4,>=4.0 # Apache-2.0 diff --git a/tox.ini b/tox.ini index bf6df93e..8e5dd276 100644 --- a/tox.ini +++ b/tox.ini @@ -98,9 +98,18 @@ max-complexity = 30 # C901 MongoDb.check function in monasca_agent/collector/checks_d/mongo.py # is too complex. Need to be simplified with less if, for loops and then # C901 can be removed from here. -ignore = C901,E402,H405 +# W504 line break after binary operator +ignore = C901,E402,H405,W504 show-source = True exclude=.venv,.git,.tox,dist,*egg,build,tests,tests_to_fix -[hacking] -local-check-factory = monasca_agent.hacking.checks.factory +[flake8:local-plugins] +extension = + M001 = checks:no_mutable_default_args + M002 = checks:no_xrange + M003 = checks:validate_assertTrue + M004 = checks:validate_assertIsNone + M005 = checks:no_log_warn_check + M006 = checks:validate_assertIsNotNone + M007 = checks:assert_raisesRegexp +paths = ./monasca_agent/hacking