From c382a3e860ba42743de581d9615b8fd1cec1e41e Mon Sep 17 00:00:00 2001 From: Ziad Sawalha Date: Tue, 6 May 2014 09:37:53 -0500 Subject: [PATCH] Implement New Discovery Schema Change-Id: Ibd34c44211920197cf19df535ef038279fff2714 --- doc/source/conf.py | 1 + doc/source/index.rst | 10 ++- doc/source/openstack_resources.rst | 8 +- doc/source/schema.rst | 33 ++++++++ satori/common/templating.py | 11 ++- satori/discovery.py | 94 ++++++++++++++++----- satori/dns.py | 27 +++--- satori/formats/text.jinja | 72 +++++++++++----- satori/shell.py | 20 ++++- satori/sysinfo/__init__.py | 2 +- satori/tests/test_dns.py | 4 +- satori/tests/test_formats_text.py | 131 ++++++++++++++++++++++++++--- 12 files changed, 334 insertions(+), 79 deletions(-) create mode 100644 doc/source/schema.rst diff --git a/doc/source/conf.py b/doc/source/conf.py index e5be5cc..2e88fa5 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -28,6 +28,7 @@ sys.path.insert(0, ROOT) sys.path.insert(0, BASE_DIR) sys.path = PLUGIN_DIRS + sys.path + # # Automatically write module docs # diff --git a/doc/source/index.rst b/doc/source/index.rst index 9c80b46..069e280 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -10,6 +10,7 @@ Satori is a configuration discovery tool for OpenStack and OpenStack tenant host contributing releases terminology + schema satori @@ -51,8 +52,7 @@ Use Satori Host: 4.4.4.4 (www.foo.com) is hosted on a Nova Instance Instance Information: - URI: https://nova.api.somecloud.com/v2/111222/servers/d9119040-f767-414 - 1-95a4-d4dbf452363a + URI: https://nova.api.somecloud.com/v2/111222/servers/d9119040 Name: sampleserver01.foo.com ID: d9119040-f767-4141-95a4-d4dbf452363a ip-addresses: @@ -61,6 +61,12 @@ Use Satori 4.4.4.4 private: 10.1.1.156 + Listening Services: + 0.0.0.0:6082 varnishd + 127.0.0.1:8080 apache2 + 127.0.0.1:3306 mysqld + Talking to: + 10.1.1.71 on 27017 Links diff --git a/doc/source/openstack_resources.rst b/doc/source/openstack_resources.rst index 9ca2e98..156166b 100644 --- a/doc/source/openstack_resources.rst +++ b/doc/source/openstack_resources.rst @@ -31,7 +31,7 @@ Alternatively, the credentials can be passed on the command line: --os-username yourname \ --os-password yadayadayada \ --os-tenant-name myproject \ - --os-auth-url http://... + --os-auth-url http://... Discovered Host @@ -67,6 +67,12 @@ control plane discovery was possible using the OpenStack credentials. 192.0.2.10 private: 10.1.1.156 + Listening Services: + 0.0.0.0:6082 varnishd + 127.0.0.1:8080 apache2 + 127.0.0.1:3306 mysqld + Talking to: + 10.1.1.71 on 27017 .. _nova: https://github.com/openstack/python-novaclient .. _OpenStack Nova conventions: https://github.com/openstack/python-novaclient/blob/master/README.rst#id1 diff --git a/doc/source/schema.rst b/doc/source/schema.rst new file mode 100644 index 0000000..5091c05 --- /dev/null +++ b/doc/source/schema.rst @@ -0,0 +1,33 @@ +====== +Schema +====== + +The following list of fields describes the data returned from Satori. + + +Target +====== + +Target contains the address suplplied to run the discovery. + + +Found +===== + +All data items discovered are returned under the found key. Keys to resources +discovered are also added under found, but the actual resources are stored +under the resources key. + + +Resources +========= + +All resources (servers, load balancers, DNS domains, etc...) are stored under +the resources key. + +Each resource contains the following keys: + +* **key**: a globally unique identifier for the resource (could be a URI) +* **id**: the id in the system that hosts the resource +* **type**: the resource type using Heat or Heat-like resource types +* **data**: any additional fields for that resource diff --git a/satori/common/templating.py b/satori/common/templating.py index 8c15f93..2929348 100644 --- a/satori/common/templating.py +++ b/satori/common/templating.py @@ -76,12 +76,12 @@ def preserve_linefeeds(value): return value.replace("\n", "\\n").replace("\r", "") -def get_jinja_environment(template, extra_globals=None): +def get_jinja_environment(template, extra_globals=None, **env_vars): """Return a sandboxed jinja environment.""" template_map = {'template': template} env = sandbox.ImmutableSandboxedEnvironment( loader=jinja2.DictLoader(template_map), - bytecode_cache=CompilerCache()) + bytecode_cache=CompilerCache(), **env_vars) env.filters['prepend'] = do_prepend env.filters['preserve'] = preserve_linefeeds env.globals['json'] = json @@ -90,14 +90,17 @@ def get_jinja_environment(template, extra_globals=None): return env -def parse(template, extra_globals=None, **kwargs): +def parse(template, extra_globals=None, env_vars=None, **kwargs): """Parse template. :param template: the template contents as a string :param extra_globals: additional globals to include :param kwargs: extra arguments are passed to the renderer """ - env = get_jinja_environment(template, extra_globals=extra_globals) + if env_vars is None: + env_vars = {} + env = get_jinja_environment(template, extra_globals=extra_globals, + **env_vars) minimum_kwargs = { 'data': {}, diff --git a/satori/discovery.py b/satori/discovery.py index 6a8269c..e60c065 100644 --- a/satori/discovery.py +++ b/satori/discovery.py @@ -23,7 +23,10 @@ Example usage: from __future__ import print_function -import ipaddress as ipaddress_module +import sys +import traceback + +import ipaddress from novaclient.v1_1 import client from pythonwhois import shared import six @@ -32,48 +35,95 @@ from satori import dns from satori import utils -def run(address, config=None, interactive=False): +def run(target, config=None, interactive=False): """Run discovery and return results.""" if config is None: config = {} - results = {} - if utils.is_valid_ip_address(address): - ipaddress = address + found = {} + resources = {} + errors = {} + results = { + 'target': target, + 'created': utils.get_time_string(), + 'found': found, + 'resources': resources, + } + if utils.is_valid_ip_address(target): + ip_address = target else: - ipaddress = dns.resolve_hostname(address) + hostname = dns.parse_target_hostname(target) + found['hostname'] = hostname + ip_address = six.text_type(dns.resolve_hostname(hostname)) #TODO(sam): Use ipaddress.ip_address.is_global # " .is_private # " .is_unspecified # " .is_multicast # To determine address "type" - if not ipaddress_module.ip_address(unicode(ipaddress)).is_loopback: + if not ipaddress.ip_address(ip_address).is_loopback: try: - results['domain'] = dns.domain_info(address) + domain_info = dns.domain_info(hostname) + resource_type = 'OS::DNS::Domain' + identifier = '%s:%s' % (resource_type, hostname) + resources[identifier] = { + 'type': resource_type, + 'key': identifier, + } + found['domain-key'] = identifier + resources[identifier]['data'] = domain_info + if 'registered' in domain_info: + found['registered-domain'] = domain_info['registered'] except shared.WhoisException as exc: results['domain'] = str(exc) + found['ip-address'] = ip_address - results['address'] = ipaddress + host, host_errors = discover_host(ip_address, config, + interactive=interactive) + if host_errors: + errors.update(host_errors) + key = host.get('key') or ip_address + resources[key] = host + found['host-key'] = key + results['updated'] = utils.get_time_string() + return results, errors - results['host'] = host = {'type': 'Undetermined'} + +def discover_host(address, config, interactive=False): + """Discover host by IP address.""" + host = {} + errors = {} if config.get('username'): - server = find_nova_host(ipaddress, config) + server = find_nova_host(address, config) if server: - host['type'] = 'Nova instance' - host['uri'] = [l['href'] for l in server.links + host['type'] = 'OS::Nova::Instance' + data = {} + host['data'] = data + data['uri'] = [l['href'] for l in server.links if l['rel'] == 'self'][0] - host['name'] = server.name - host['id'] = server.id - host['addresses'] = server.addresses + data['name'] = server.name + data['id'] = server.id + data['addresses'] = server.addresses + host['key'] = data['uri'] + if config.get('system_info'): module_name = config['system_info'].replace("-", "_") if '.' not in module_name: module_name = 'satori.sysinfo.%s' % module_name system_info_module = utils.import_object(module_name) - result = system_info_module.get_systeminfo(ipaddress, config, - interactive=interactive) - host['system_info'] = result - return results + try: + result = system_info_module.get_systeminfo( + address, config, interactive=interactive) + host.setdefault('data', {}) + host['data']['system_info'] = result + except Exception as exc: + exc_traceback = sys.exc_info()[2] + errors['system_info'] = { + 'type': "ERROR", + 'message': str(exc), + 'exception': exc, + 'traceback': traceback.format_tb(exc_traceback), + } + return host, errors def find_nova_host(address, config): @@ -86,6 +136,6 @@ def find_nova_host(address, config): service_type="compute") for server in nova.servers.list(): for network_addresses in six.itervalues(server.addresses): - for ipaddress in network_addresses: - if ipaddress['addr'] == address: + for ip_address in network_addresses: + if ip_address['addr'] == address: return server diff --git a/satori/dns.py b/satori/dns.py index fb0e5d9..fae6a72 100644 --- a/satori/dns.py +++ b/satori/dns.py @@ -9,7 +9,7 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. -# + """Satori DNS Discovery.""" import datetime @@ -27,21 +27,24 @@ from satori import utils LOG = logging.getLogger(__name__) -def resolve_hostname(host): - """Get IP address of hostname or URL.""" +def parse_target_hostname(target): + """Get IP address or FQDN of a target which could be a URL or address.""" + if not target: + raise errors.SatoriInvalidNetloc("Target must be supplied.") try: - if not host: - raise AttributeError("Host must be supplied.") - parsed = urlparse.urlparse(host) + parsed = urlparse.urlparse(target) except AttributeError as err: - error = "Hostname `%s` is unparseable. Error: %s" % (host, err) + error = "Target `%s` is unparseable. Error: %s" % (target, err) LOG.exception(error) raise errors.SatoriInvalidNetloc(error) # Domain names and IP are in netloc when parsed with a protocol # they will be in path if parsed without a protocol - hostname = parsed.netloc or parsed.path + return parsed.netloc or parsed.path + +def resolve_hostname(hostname): + """Get IP address of hostname.""" try: address = socket.gethostbyname(hostname) except socket.gaierror: @@ -55,13 +58,13 @@ def get_registered_domain(hostname): return tldextract.extract(hostname).registered_domain -def ip_info(ip): +def ip_info(ip_address): """Get as much information as possible for a given ip address.""" - if not utils.is_valid_ip_address(ip): - error = "`%s` is an invalid IP address." % ip + if not utils.is_valid_ip_address(ip_address): + error = "`%s` is an invalid IP address." % ip_address raise errors.SatoriInvalidIP(error) - result = pythonwhois.get_whois(ip) + result = pythonwhois.get_whois(ip_address) return { 'whois': result['raw'] diff --git a/satori/formats/text.jinja b/satori/formats/text.jinja index 2d089ae..d400a1e 100644 --- a/satori/formats/text.jinja +++ b/satori/formats/text.jinja @@ -1,20 +1,54 @@ -{% if data['address'] != target %}Address: - {{ target }} resolves to IPv4 address {{ data.address }} -{% endif %}{% if data['domain'] %}Domain: {{ data.domain.name }} - Registrar: {{ data.domain.registrar }}{% if data.domain.nameservers %} - Nameservers: {% for nameserver in data.domain.nameservers %}{{nameserver}}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %}{% endif %}{% if data['domain'] and data.domain.days_until_expires %} Expires: {{ data.domain.days_until_expires }} days{% endif %}{% if data['host'] and data.host.type == 'Nova instance' %}Host: - {{ data.address }} ({{ target }}) is hosted on a {{ data.host.type }} - Instance Information: - URI: {{ data.host.uri }} - Name: {{ data.host.name }} - ID: {{ data.host.id }} - ip-addresses:{% for name, addresses in data.host.addresses.items() %} - {{ name }}:{% for address in addresses %} - {{ address.addr }}{% endfor %}{% endfor %} -{% elif data['address'] %}Host: - ip-address: {{ data.address }} -{% else %}Host not found -{% endif %}{% if data['host'] and data.host.system_info %}Packages: -{% for name, package in data.host.system_info.packages.items() %} {{ name }}: {{ package.version }} -{% endfor %} +{% set found = data['found'] | default({}) %} +{% set resources = data['resources'] | default({'n/a': {}}) %} +{% set address = found['ip-address'] %} +{% set hostkey = found['host-key'] | default('n/a') %} +{% set domainkey = found['domain-key'] | default('n/a') %} +{% set server = resources[hostkey] | default(False) %} +{% set domain = resources[domainkey] | default(False) %} + +{% if found['ip-address'] != target %}Address: + {{ target }} resolves to IPv4 address {{ found['ip-address'] }} +{%- endif %} + +{% if domain %}Domain: {{ domain['data'].name }} + Registrar: {{ domain['data'].registrar }} +{% if domain['data'].nameservers %} + Nameservers: {% for nameserver in domain['data'].nameservers %}{{nameserver}}{% if not loop.last %}, {% endif %}{% endfor %} + +{% endif %} +{% if domain['data'].days_until_expires %} + Expires: {{ domain['data'].days_until_expires }} days +{% endif %} +{%- endif %} +{% if server and server.type == 'OS::Nova::Instance' %} +Host: + {{ found['ip-address'] }} ({{ target }}) is hosted on a Nova instance +{% if 'data' in server %} Instance Information: + URI: {{ server['data'].uri | default('n/a') }} + Name: {{ server['data'].name | default('n/a') }} + ID: {{ server['data'].id | default('n/a') }} +{% if 'addresses' in server['data'] %} ip-addresses: +{% for name, addresses in server['data'].addresses.items() %} + {{ name }}: +{% for address in addresses %} + {{ address.addr }} +{% endfor %} +{% endfor %}{% endif %}{% endif %} +{% elif found['ip-address'] %} +Host: + ip-address: {{ found['ip-address'] }} +{% else %}Host not found +{% endif %} +{% if server and 'data' in server and server['data'].system_info %} +{% if 'remote_services' in server['data'].system_info %} + Listening Services: +{% for remote in server['data'].system_info.remote_services | sort %} + {{ remote.ip }}:{{ remote.port }} {{ remote.process }} +{% endfor %}{% endif %} +{% if 'connections' in server['data'].system_info %} + Talking to: +{% for connection in server['data'].system_info.connections | dictsort %} + {{ connection[0] }}{% if connection[1] %} on {% for port in connection[1] %}{{ port }}{% if not loop.last %}, {% endif %}{% endfor %}{% endif %} + +{% endfor %}{% endif %} {% endif %} diff --git a/satori/shell.py b/satori/shell.py index 5b193a4..a9b2d32 100644 --- a/satori/shell.py +++ b/satori/shell.py @@ -17,7 +17,6 @@ Accept a network location, run through the discovery process and report the findings back to the user. - """ from __future__ import print_function @@ -225,9 +224,12 @@ def main(argv=None): get_template_path(config['format'])) try: - results = discovery.run(config['netloc'], config, interactive=True) + results, errors = discovery.run(config['netloc'], config, + interactive=True) print(format_output(config['netloc'], results, template_name=config['format'])) + if errors: + sys.stderr.write(format_errors(errors, config)) except Exception as exc: # pylint: disable=W0703 if config['debug']: LOG.exception(exc) @@ -263,8 +265,20 @@ def format_output(discovered_target, results, template_name="text"): return(json.dumps(results, indent=2)) else: template = get_template(template_name) + env_vars = dict(lstrip_blocks=True, trim_blocks=True) return templating.parse(template, target=discovered_target, - data=results).strip('\n') + data=results, env_vars=env_vars).strip('\n') + + +def format_errors(errors, config): + """Format errors for output to console.""" + if config['debug']: + return str(errors) + else: + formatted = {} + for key, error in errors.items(): + formatted[key] = error['message'] + return str(formatted) if __name__ == "__main__": diff --git a/satori/sysinfo/__init__.py b/satori/sysinfo/__init__.py index 198ce9c..921cf73 100644 --- a/satori/sysinfo/__init__.py +++ b/satori/sysinfo/__init__.py @@ -1 +1 @@ -""".""" +"""Modules for Data Plane Discovery.""" diff --git a/satori/tests/test_dns.py b/satori/tests/test_dns.py index 7203625..a8a5685 100644 --- a/satori/tests/test_dns.py +++ b/satori/tests/test_dns.py @@ -136,13 +136,13 @@ class TestDNS(utils.TestCase): def test_resolve_int_raises_invalid_netloc_error(self): self.assertRaises( errors.SatoriInvalidNetloc, - dns.resolve_hostname, + dns.parse_target_hostname, 100) def test_resolve_none_raises_invalid_netloc_error(self): self.assertRaises( errors.SatoriInvalidNetloc, - dns.resolve_hostname, + dns.parse_target_hostname, None) def test_registered_domain_subdomain_removed(self): diff --git a/satori/tests/test_formats_text.py b/satori/tests/test_formats_text.py index 1a7a533..9d457d5 100644 --- a/satori/tests/test_formats_text.py +++ b/satori/tests/test_formats_text.py @@ -29,20 +29,25 @@ class TestTextTemplate(unittest.TestCase): def test_no_data(self): """Handles response with no host.""" - result = templating.parse(self.template, data={}) + env_vars = dict(lstrip_blocks=True, trim_blocks=True) + result = templating.parse(self.template, data={}, env_vars=env_vars) self.assertEqual(result.strip('\n'), 'Host not found') def test_target_is_ip(self): """Handles response when host is just the supplied address.""" + env_vars = dict(lstrip_blocks=True, trim_blocks=True) result = templating.parse(self.template, target='127.0.0.1', - data={'address': '127.0.0.1'}) + data={'found': {'ip-address': '127.0.0.1'}}, + env_vars=env_vars) self.assertEqual(result.strip('\n'), 'Host:\n ip-address: 127.0.0.1') def test_host_not_server(self): """Handles response when host is not a nova instance.""" + env_vars = dict(lstrip_blocks=True, trim_blocks=True) result = templating.parse(self.template, target='localhost', - data={'address': '127.0.0.1'}) + data={'found': {'ip-address': '127.0.0.1'}}, + env_vars=env_vars) self.assertEqual(result.strip('\n'), 'Address:\n localhost resolves to IPv4 address ' '127.0.0.1\nHost:\n ip-address: 127.0.0.1') @@ -50,20 +55,44 @@ class TestTextTemplate(unittest.TestCase): def test_host_is_nova_instance(self): """Handles response when host is a nova instance.""" data = { - 'address': '10.1.1.45', - 'host': { - 'type': 'Nova instance', - 'uri': 'https://servers/path', - 'id': '1000B', - 'name': 'x', - 'addresses': { - 'public': [{'type': 'ipv4', 'addr': '10.1.1.45'}] + 'found': { + 'ip-address': '10.1.1.45', + 'hostname': 'x', + 'host-key': 'https://servers/path' + }, + 'target': 'instance.nova.local', + 'resources': { + 'https://servers/path': { + 'type': 'OS::Nova::Instance', + 'data': { + 'uri': 'https://servers/path', + 'id': '1000B', + 'name': 'x', + 'addresses': { + 'public': [{'type': 'ipv4', 'addr': '10.1.1.45'}] + }, + 'system_info': { + 'connections': { + '192.168.2.100': [], + '192.168.2.101': [433], + '192.168.2.102': [8080, 8081] + }, + 'remote_services': [ + { + 'ip': '0.0.0.0', + 'process': 'nginx', + 'port': 80 + } + ] + } + } } } } + env_vars = dict(lstrip_blocks=True, trim_blocks=True) result = templating.parse(self.template, target='instance.nova.local', - data=data) + data=data, env_vars=env_vars) expected = """\ Address: instance.nova.local resolves to IPv4 address 10.1.1.45 @@ -75,7 +104,83 @@ Host: ID: 1000B ip-addresses: public: - 10.1.1.45""" + 10.1.1.45 + Listening Services: + 0.0.0.0:80 nginx + Talking to: + 192.168.2.100 + 192.168.2.101 on 433 + 192.168.2.102 on 8080, 8081""" + self.assertEqual(result.strip('\n'), expected) + + def test_host_has_no_data(self): + """Handles response when host is a nova instance.""" + data = { + 'found': { + 'ip-address': '10.1.1.45', + 'hostname': 'x', + 'host-key': 'https://servers/path' + }, + 'target': 'instance.nova.local', + 'resources': { + 'https://servers/path': { + 'type': 'OS::Nova::Instance' + } + } + } + env_vars = dict(lstrip_blocks=True, trim_blocks=True) + result = templating.parse(self.template, + target='instance.nova.local', + data=data, env_vars=env_vars) + expected = """\ +Address: + instance.nova.local resolves to IPv4 address 10.1.1.45 +Host: + 10.1.1.45 (instance.nova.local) is hosted on a Nova instance""" + self.assertEqual(result.strip('\n'), expected) + + def test_host_data_missing_items(self): + """Handles response when host is a nova instance.""" + data = { + 'found': { + 'ip-address': '10.1.1.45', + 'hostname': 'x', + 'host-key': 'https://servers/path' + }, + 'target': 'instance.nova.local', + 'resources': { + 'https://servers/path': { + 'type': 'OS::Nova::Instance', + 'data': { + 'id': '1000B', + 'system_info': { + 'remote_services': [ + { + 'ip': '0.0.0.0', + 'process': 'nginx', + 'port': 80 + } + ] + } + } + } + } + } + env_vars = dict(lstrip_blocks=True, trim_blocks=True) + result = templating.parse(self.template, + target='instance.nova.local', + data=data, env_vars=env_vars) + expected = """\ +Address: + instance.nova.local resolves to IPv4 address 10.1.1.45 +Host: + 10.1.1.45 (instance.nova.local) is hosted on a Nova instance + Instance Information: + URI: n/a + Name: n/a + ID: 1000B + Listening Services: + 0.0.0.0:80 nginx""" self.assertEqual(result.strip('\n'), expected)