Add url_access_checker to network checker

This change introduces new entry point for network_checker, which allows
to check if it is possible to access urls give to the command.

Usage: urlchecker check <url1> <url2> <url3> ...

It will be used to check repository connectivity from slave nodes.

python-requests is added to nailgun-net-check package requirements in
both rpm spec and debian `control` file.

Change-Id: Idc04f74ad7364dee452e9151391654f828e2342d
Partial-Bug: #1439686

debian/control

@@ -31,6 +31,6 @@ Description: <insert up to 60 chars description>
 
 Package: nailgun-net-check
 Architecture: all
-Depends: ${misc:Depends}, ${python:Depends}, python-pypcap, vlan, python-scapy, cliff-tablib, python-stevedore, python-daemonize, python-yaml, tcpdump
+Depends: ${misc:Depends}, ${python:Depends}, python-pypcap, vlan, python-scapy, cliff-tablib, python-stevedore, python-daemonize, python-yaml, tcpdump, python-requests
 Description: NailGun client net-check
  .

network_checker/dhcp_checker/cli.py

@@ -13,18 +13,19 @@
 #    under the License.
 
 import logging
-from logging import handlers
 import os
 import sys
 # fixed in cmd2 >=0.6.6
 os.environ['EDITOR'] = '/usr/bin/nano'
 
-from cliff.app import App
 from cliff.commandmanager import CommandManager
 
+from fuel_network_checker import base_app
 
-class DhcpApp(App):
+
+class DhcpApp(base_app.BaseApp):
     DEFAULT_VERBOSE_LEVEL = 0
+    LOG_FILENAME = '/var/log/dhcp_checker.log'
 
     def __init__(self):
         super(DhcpApp, self).__init__(
@@ -35,23 +36,6 @@ class DhcpApp(App):
 
     def configure_logging(self):
         super(DhcpApp, self).configure_logging()
-        logger = logging.getLogger()
-        logger.setLevel(logging.DEBUG)
-        formatter = logging.Formatter(
-            '%(asctime)s %(levelname)s (%(module)s) %(message)s',
-            "%Y-%m-%d %H:%M:%S")
-
-        stream_handler = logging.StreamHandler()
-        stream_handler.setLevel(logging.ERROR)
-        stream_handler.setFormatter(formatter)
-
-        file_handler = handlers.TimedRotatingFileHandler(
-            '/var/log/dhcp_checker.log')
-        file_handler.setLevel(logging.DEBUG)
-        file_handler.setFormatter(formatter)
-
-        logger.addHandler(stream_handler)
-        logger.addHandler(file_handler)
 
         # set scapy logger level only to ERROR
         # due to a lot of spam

network_checker/fuel_network_checker/base_app.py

@@ -0,0 +1,43 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+#
+import logging
+from logging import handlers
+
+from cliff.app import App
+
+
+class BaseApp(App):
+    DEFAULT_VERBOSE_LEVEL = 0
+    LOG_FILENAME = ''  # This needs to be redefined in child class
+
+    def configure_logging(self):
+        super(BaseApp, self).configure_logging()
+        logger = logging.getLogger()
+        logger.setLevel(logging.DEBUG)
+        formatter = logging.Formatter(
+            '%(asctime)s %(levelname)s (%(module)s) %(message)s',
+            "%Y-%m-%d %H:%M:%S")
+
+        stream_handler = logging.StreamHandler()
+        stream_handler.setLevel(logging.ERROR)
+        stream_handler.setFormatter(formatter)
+
+        file_handler = handlers.TimedRotatingFileHandler(
+            self.LOG_FILENAME)
+        file_handler.setLevel(logging.DEBUG)
+        file_handler.setFormatter(formatter)
+
+        logger.addHandler(stream_handler)
+        logger.addHandler(file_handler)

network_checker/requirements.txt

@@ -5,3 +5,4 @@ pypcap==1.1.1
 stevedore
 daemonize
 pyyaml
+requests

network_checker/setup.py

@@ -34,16 +34,19 @@ setuptools.setup(
             'net_probe.py = network_checker.net_check.api:main',
             'fuel-netcheck = network_checker.cli:main',
             'dhcpcheck = dhcp_checker.cli:main',
+            'urlaccesscheck = url_access_checker.cli:main',
         ],
         'dhcp.check': [
             'discover = dhcp_checker.commands:ListDhcpServers',
             'request = dhcp_checker.commands:ListDhcpAssignment',
             'vlans = dhcp_checker.commands:DhcpWithVlansCheck'
-
         ],
         'network_checker': [
             'multicast = network_checker.multicast.api:MulticastChecker',
             'simple = network_checker.tests.simple:SimpleChecker'
-        ]
+        ],
+        'urlaccesscheck': [
+            'check = url_access_checker.commands:CheckUrls'
+        ],
     },
 )

network_checker/url_access_checker/api.py

@@ -0,0 +1,50 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import json
+import socket
+
+import requests
+
+import url_access_checker.errors as errors
+
+
+def check_urls(urls):
+    responses = map(_get_response_tuple, urls)
+    failed_responses = filter(lambda x: x[0], responses)
+
+    if failed_responses:
+        raise errors.UrlNotAvailable(json.dumps(
+            {'failed_urls': map(lambda r: r[1], failed_responses)}))
+
+
+def _get_response_tuple(url):
+    """Return a tuple which contains a result of url test
+
+    Arguments:
+    url -- a string containing url for testing
+
+    Result tuple content:
+        result[0] -- boolean value, True if the url is deemed failed
+        result[1] -- unchange url argument
+    """
+    try:
+        response = requests.get(url)
+        return (response.status_code != 200, url)
+    except (requests.exceptions.ConnectionError,
+            requests.exceptions.Timeout,
+            requests.exceptions.HTTPError,
+            ValueError,
+            socket.timeout):
+        return (True, url)

network_checker/url_access_checker/cli.py

@@ -0,0 +1,39 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sys
+
+from cliff.commandmanager import CommandManager
+
+from fuel_network_checker import base_app
+
+
+class UrlAccessCheckApp(base_app.BaseApp):
+    LOG_FILENAME = '/var/log/url_access_checker.log'
+
+    def __init__(self):
+        super(UrlAccessCheckApp, self).__init__(
+            description='Url access check application',
+            version='0.1',
+            command_manager=CommandManager('urlaccesscheck'),
+        )
+
+
+def main(argv=sys.argv[1:]):
+    myapp = UrlAccessCheckApp()
+    return myapp.run(argv)
+
+
+if __name__ == '__main__':
+    sys.exit(main(sys.argv[1:]))

network_checker/url_access_checker/commands.py

@@ -0,0 +1,40 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
+import sys
+
+from cliff import command
+
+import url_access_checker.api as api
+import url_access_checker.errors as errors
+
+LOG = logging.getLogger(__name__)
+
+
+class CheckUrls(command.Command):
+    """Check if it is possible to retrieve urls."""
+    def get_parser(self, prog_name):
+        parser = super(CheckUrls, self).get_parser(prog_name)
+        parser.add_argument('urls', type=str, nargs='+',
+                            help='List of urls to check')
+        return parser
+
+    def take_action(self, parsed_args):
+        LOG.info('Starting url access check for {0}'.format(parsed_args.urls))
+        try:
+            api.check_urls(parsed_args.urls)
+        except errors.UrlNotAvailable as e:
+            sys.stdout.write(str(e))
+            raise e

network_checker/url_access_checker/errors.py

@@ -0,0 +1,17 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+class UrlNotAvailable(Exception):
+    pass

network_checker/url_access_checker/tests/unit/test_commands.py

@@ -0,0 +1,47 @@
+#    Copyright 2015 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import unittest
+
+import mock
+
+from url_access_checker import cli
+
+
+class TestUrlCheckerCommands(unittest.TestCase):
+
+    def setUp(self):
+        self.urls = ['url{0}'.format(i) for i in range(10)]
+
+    @mock.patch('requests.get')
+    def test_check_urls_success(self, get_mock):
+        response_mock = mock.Mock()
+        response_mock.status_code = 200
+        get_mock.return_value = response_mock
+
+        exit_code = cli.main(['check'] + self.urls)
+        self.assertEqual(exit_code, 0)
+
+    @mock.patch('requests.get')
+    def test_check_urls_fail(self, get_mock):
+        response_mock = mock.Mock()
+        response_mock.status_code = 404
+        get_mock.return_value = response_mock
+
+        exit_code = cli.main(['check'] + self.urls)
+        self.assertEqual(exit_code, 1)
+
+    def test_check_urls_fail_on_requests_error(self):
+        exit_code = cli.main(['check'] + self.urls)
+        self.assertEqual(exit_code, 1)

specs/nailgun.spec

@@ -148,6 +148,7 @@ Requires:  python-stevedore
 Requires:  python-daemonize
 Requires:  python-yaml
 Requires:  tcpdump
+Requires:  python-requests
 
 
 %description -n nailgun-net-check