From abcde8950fee3e638a934eca3e79e25f5ba61d08 Mon Sep 17 00:00:00 2001 From: Maciej Kwiek Date: Wed, 13 May 2015 11:36:11 +0200 Subject: [PATCH] Add url_access_checker to network checker This change introduces new entry point for network_checker, which allows to check if it is possible to access urls give to the command. Usage: urlchecker check ... It will be used to check repository connectivity from slave nodes. python-requests is added to nailgun-net-check package requirements in both rpm spec and debian `control` file. Change-Id: Idc04f74ad7364dee452e9151391654f828e2342d Partial-Bug: #1439686 --- debian/control | 2 +- network_checker/dhcp_checker/cli.py | 24 ++------- .../fuel_network_checker/__init__.py | 0 .../fuel_network_checker/base_app.py | 43 ++++++++++++++++ network_checker/requirements.txt | 1 + network_checker/setup.py | 7 ++- .../url_access_checker/__init__.py | 0 network_checker/url_access_checker/api.py | 50 +++++++++++++++++++ network_checker/url_access_checker/cli.py | 39 +++++++++++++++ .../url_access_checker/commands.py | 40 +++++++++++++++ network_checker/url_access_checker/errors.py | 17 +++++++ .../url_access_checker/tests/unit/__init__.py | 0 .../tests/unit/test_commands.py | 47 +++++++++++++++++ specs/nailgun.spec | 1 + 14 files changed, 248 insertions(+), 23 deletions(-) create mode 100644 network_checker/fuel_network_checker/__init__.py create mode 100644 network_checker/fuel_network_checker/base_app.py create mode 100644 network_checker/url_access_checker/__init__.py create mode 100644 network_checker/url_access_checker/api.py create mode 100644 network_checker/url_access_checker/cli.py create mode 100644 network_checker/url_access_checker/commands.py create mode 100644 network_checker/url_access_checker/errors.py create mode 100644 network_checker/url_access_checker/tests/unit/__init__.py create mode 100644 network_checker/url_access_checker/tests/unit/test_commands.py diff --git a/debian/control b/debian/control index 7b798bd16f..af1ab13161 100644 --- a/debian/control +++ b/debian/control @@ -31,6 +31,6 @@ Description: Package: nailgun-net-check Architecture: all -Depends: ${misc:Depends}, ${python:Depends}, python-pypcap, vlan, python-scapy, cliff-tablib, python-stevedore, python-daemonize, python-yaml, tcpdump +Depends: ${misc:Depends}, ${python:Depends}, python-pypcap, vlan, python-scapy, cliff-tablib, python-stevedore, python-daemonize, python-yaml, tcpdump, python-requests Description: NailGun client net-check . diff --git a/network_checker/dhcp_checker/cli.py b/network_checker/dhcp_checker/cli.py index b60a3632f4..4b97ffdeb9 100644 --- a/network_checker/dhcp_checker/cli.py +++ b/network_checker/dhcp_checker/cli.py @@ -13,18 +13,19 @@ # under the License. import logging -from logging import handlers import os import sys # fixed in cmd2 >=0.6.6 os.environ['EDITOR'] = '/usr/bin/nano' -from cliff.app import App from cliff.commandmanager import CommandManager +from fuel_network_checker import base_app -class DhcpApp(App): + +class DhcpApp(base_app.BaseApp): DEFAULT_VERBOSE_LEVEL = 0 + LOG_FILENAME = '/var/log/dhcp_checker.log' def __init__(self): super(DhcpApp, self).__init__( @@ -35,23 +36,6 @@ class DhcpApp(App): def configure_logging(self): super(DhcpApp, self).configure_logging() - logger = logging.getLogger() - logger.setLevel(logging.DEBUG) - formatter = logging.Formatter( - '%(asctime)s %(levelname)s (%(module)s) %(message)s', - "%Y-%m-%d %H:%M:%S") - - stream_handler = logging.StreamHandler() - stream_handler.setLevel(logging.ERROR) - stream_handler.setFormatter(formatter) - - file_handler = handlers.TimedRotatingFileHandler( - '/var/log/dhcp_checker.log') - file_handler.setLevel(logging.DEBUG) - file_handler.setFormatter(formatter) - - logger.addHandler(stream_handler) - logger.addHandler(file_handler) # set scapy logger level only to ERROR # due to a lot of spam diff --git a/network_checker/fuel_network_checker/__init__.py b/network_checker/fuel_network_checker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/network_checker/fuel_network_checker/base_app.py b/network_checker/fuel_network_checker/base_app.py new file mode 100644 index 0000000000..0fe0c75234 --- /dev/null +++ b/network_checker/fuel_network_checker/base_app.py @@ -0,0 +1,43 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +import logging +from logging import handlers + +from cliff.app import App + + +class BaseApp(App): + DEFAULT_VERBOSE_LEVEL = 0 + LOG_FILENAME = '' # This needs to be redefined in child class + + def configure_logging(self): + super(BaseApp, self).configure_logging() + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s %(levelname)s (%(module)s) %(message)s', + "%Y-%m-%d %H:%M:%S") + + stream_handler = logging.StreamHandler() + stream_handler.setLevel(logging.ERROR) + stream_handler.setFormatter(formatter) + + file_handler = handlers.TimedRotatingFileHandler( + self.LOG_FILENAME) + file_handler.setLevel(logging.DEBUG) + file_handler.setFormatter(formatter) + + logger.addHandler(stream_handler) + logger.addHandler(file_handler) diff --git a/network_checker/requirements.txt b/network_checker/requirements.txt index e965c0566e..0a677e20ec 100644 --- a/network_checker/requirements.txt +++ b/network_checker/requirements.txt @@ -5,3 +5,4 @@ pypcap==1.1.1 stevedore daemonize pyyaml +requests diff --git a/network_checker/setup.py b/network_checker/setup.py index 79835b5f83..2d34f418b3 100644 --- a/network_checker/setup.py +++ b/network_checker/setup.py @@ -34,16 +34,19 @@ setuptools.setup( 'net_probe.py = network_checker.net_check.api:main', 'fuel-netcheck = network_checker.cli:main', 'dhcpcheck = dhcp_checker.cli:main', + 'urlaccesscheck = url_access_checker.cli:main', ], 'dhcp.check': [ 'discover = dhcp_checker.commands:ListDhcpServers', 'request = dhcp_checker.commands:ListDhcpAssignment', 'vlans = dhcp_checker.commands:DhcpWithVlansCheck' - ], 'network_checker': [ 'multicast = network_checker.multicast.api:MulticastChecker', 'simple = network_checker.tests.simple:SimpleChecker' - ] + ], + 'urlaccesscheck': [ + 'check = url_access_checker.commands:CheckUrls' + ], }, ) diff --git a/network_checker/url_access_checker/__init__.py b/network_checker/url_access_checker/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/network_checker/url_access_checker/api.py b/network_checker/url_access_checker/api.py new file mode 100644 index 0000000000..2038d44cbd --- /dev/null +++ b/network_checker/url_access_checker/api.py @@ -0,0 +1,50 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import socket + +import requests + +import url_access_checker.errors as errors + + +def check_urls(urls): + responses = map(_get_response_tuple, urls) + failed_responses = filter(lambda x: x[0], responses) + + if failed_responses: + raise errors.UrlNotAvailable(json.dumps( + {'failed_urls': map(lambda r: r[1], failed_responses)})) + + +def _get_response_tuple(url): + """Return a tuple which contains a result of url test + + Arguments: + url -- a string containing url for testing + + Result tuple content: + result[0] -- boolean value, True if the url is deemed failed + result[1] -- unchange url argument + """ + try: + response = requests.get(url) + return (response.status_code != 200, url) + except (requests.exceptions.ConnectionError, + requests.exceptions.Timeout, + requests.exceptions.HTTPError, + ValueError, + socket.timeout): + return (True, url) diff --git a/network_checker/url_access_checker/cli.py b/network_checker/url_access_checker/cli.py new file mode 100644 index 0000000000..21199c4a2b --- /dev/null +++ b/network_checker/url_access_checker/cli.py @@ -0,0 +1,39 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from cliff.commandmanager import CommandManager + +from fuel_network_checker import base_app + + +class UrlAccessCheckApp(base_app.BaseApp): + LOG_FILENAME = '/var/log/url_access_checker.log' + + def __init__(self): + super(UrlAccessCheckApp, self).__init__( + description='Url access check application', + version='0.1', + command_manager=CommandManager('urlaccesscheck'), + ) + + +def main(argv=sys.argv[1:]): + myapp = UrlAccessCheckApp() + return myapp.run(argv) + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/network_checker/url_access_checker/commands.py b/network_checker/url_access_checker/commands.py new file mode 100644 index 0000000000..e6397b8b3d --- /dev/null +++ b/network_checker/url_access_checker/commands.py @@ -0,0 +1,40 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +from cliff import command + +import url_access_checker.api as api +import url_access_checker.errors as errors + +LOG = logging.getLogger(__name__) + + +class CheckUrls(command.Command): + """Check if it is possible to retrieve urls.""" + def get_parser(self, prog_name): + parser = super(CheckUrls, self).get_parser(prog_name) + parser.add_argument('urls', type=str, nargs='+', + help='List of urls to check') + return parser + + def take_action(self, parsed_args): + LOG.info('Starting url access check for {0}'.format(parsed_args.urls)) + try: + api.check_urls(parsed_args.urls) + except errors.UrlNotAvailable as e: + sys.stdout.write(str(e)) + raise e diff --git a/network_checker/url_access_checker/errors.py b/network_checker/url_access_checker/errors.py new file mode 100644 index 0000000000..ccc0dc1938 --- /dev/null +++ b/network_checker/url_access_checker/errors.py @@ -0,0 +1,17 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +class UrlNotAvailable(Exception): + pass diff --git a/network_checker/url_access_checker/tests/unit/__init__.py b/network_checker/url_access_checker/tests/unit/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/network_checker/url_access_checker/tests/unit/test_commands.py b/network_checker/url_access_checker/tests/unit/test_commands.py new file mode 100644 index 0000000000..4d73a26b78 --- /dev/null +++ b/network_checker/url_access_checker/tests/unit/test_commands.py @@ -0,0 +1,47 @@ +# Copyright 2015 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import unittest + +import mock + +from url_access_checker import cli + + +class TestUrlCheckerCommands(unittest.TestCase): + + def setUp(self): + self.urls = ['url{0}'.format(i) for i in range(10)] + + @mock.patch('requests.get') + def test_check_urls_success(self, get_mock): + response_mock = mock.Mock() + response_mock.status_code = 200 + get_mock.return_value = response_mock + + exit_code = cli.main(['check'] + self.urls) + self.assertEqual(exit_code, 0) + + @mock.patch('requests.get') + def test_check_urls_fail(self, get_mock): + response_mock = mock.Mock() + response_mock.status_code = 404 + get_mock.return_value = response_mock + + exit_code = cli.main(['check'] + self.urls) + self.assertEqual(exit_code, 1) + + def test_check_urls_fail_on_requests_error(self): + exit_code = cli.main(['check'] + self.urls) + self.assertEqual(exit_code, 1) diff --git a/specs/nailgun.spec b/specs/nailgun.spec index 8a0ba8d7d7..033942bc1f 100644 --- a/specs/nailgun.spec +++ b/specs/nailgun.spec @@ -148,6 +148,7 @@ Requires: python-stevedore Requires: python-daemonize Requires: python-yaml Requires: tcpdump +Requires: python-requests %description -n nailgun-net-check