Moved resource information collection into discovery process; implemented some keystone database inspections

discover_test.py

@@ -4,7 +4,7 @@ from itertools import groupby
 from ostack_validator.common import Issue, MarkedIssue, Inspection
 from ostack_validator.model import OpenstackComponent
 from ostack_validator.discovery import OpenstackDiscovery
-from ostack_validator.inspections import KeystoneAuthtokenSettingsInspection
+from ostack_validator.inspections import KeystoneAuthtokenSettingsInspection, KeystoneEndpointsInspection
 
 def print_components(openstack):
   for host in openstack.hosts:
@@ -46,7 +46,7 @@ def print_issues(issues):
         print('  [%s] %s (line %d column %d)' % (issue.type, issue.message, issue.mark.line+1, issue.mark.column+1))
     else:
       for issue in issues:
-        print(issue)
+        print('[%s] %s' % (issue.type, issue.message))
 
 def main():
   logging.basicConfig(level=logging.WARNING)
@@ -60,12 +60,12 @@ def main():
 
   print_components(openstack)
 
-  all_inspections = [KeystoneAuthtokenSettingsInspection]
+  all_inspections = [KeystoneAuthtokenSettingsInspection, KeystoneEndpointsInspection]
   for inspection in all_inspections:
     x = inspection()
     x.inspect(openstack)
 
-  print_issues(openstack.issues)
+  print_issues(openstack.all_issues)
 
 if __name__ == '__main__':
   main()

ostack_validator/discovery.py

@@ -6,7 +6,7 @@ import logging
 
 import spur
 
-from ostack_validator.common import Issue, Mark, MarkedIssue, index
+from ostack_validator.common import Issue, Mark, MarkedIssue, index, path_relative_to
 from ostack_validator.model import Openstack, Host, OpenstackComponent, KeystoneComponent, NovaComputeComponent, GlanceApiComponent
 
 
@@ -16,8 +16,8 @@ class NodeClient(object):
     super(NodeClient, self).__init__()
     self.shell = spur.SshShell(hostname=node_address, port=ssh_port, username=username, private_key_file=private_key_file, missing_host_key=spur.ssh.MissingHostKey.accept)
 
-  def run(self, command):
-    return self.shell.run(command)
+  def run(self, command, *args, **kwargs):
+    return self.shell.run(command, *args, **kwargs)
 
   def open(self, path, mode='r'):
     return self.shell.open(path, mode)
@@ -68,46 +68,24 @@ class OpenstackDiscovery(object):
 
   def _discover_node(self, client):
     hostname = client.run(['hostname']).output.strip()
-    metadata = {}
 
-    host = Host(name=hostname, metadata=metadata, client=client)
+    host = Host(name=hostname)
+    host.id = self._collect_host_id(client)
+    host.network_addresses = self._collect_host_network_addresses(client)
 
-    processes = [line.split() for line in client.run(['ps', '-Ao', 'cmd', '--no-headers']).output.split("\n")]
+    keystone = self._collect_keystone_data(client)
+    if keystone:
+      host.add_component(keystone)
 
-    keystone_process = self._find_python_process(processes, 'keystone-all')
-    if keystone_process:
-      p = index(keystone_process, lambda s: s == '--config-file')
-      if p != -1 and p+1 < len(keystone_process):
-        config_file = keystone_process[p+1]
-      else:
-        config_file = '/etc/keystone/keystone.conf'
-
-      host.add_component(KeystoneComponent(config_file))
-
-    glance_api_process = self._find_python_process(processes, 'glance-api')
-    if glance_api_process:
-      p = index(glance_api_process, lambda s: s == '--config-file')
-      if p != -1 and p+1 < len(glance_api_process):
-        config_file = glance_api_process[p+1]
-      else:
-        config_file = '/etc/glance/glance-api.conf'
-
-      host.add_component(GlanceApiComponent(config_file))
-
-    nova_compute_process = self._find_python_process(processes, 'nova-compute')
-    if nova_compute_process:
-      p = index(nova_compute_process, lambda s: s == '--config-file')
-      if p != -1 and p+1 < len(nova_compute_process):
-        config_file = nova_compute_process[p+1]
-      else:
-        config_file = '/etc/nova/nova.conf'
-
-      host.add_component(NovaComputeComponent(config_file))
+    nova_compute = self._collect_nova_data(client)
+    if nova_compute:
+      host.add_component(nova_compute)
 
     return host
 
 
-  def _find_python_process(self, processes, name):
+  def _find_python_process(self, client, name):
+    processes = self._get_processes(client)
     for line in processes:
       if len(line) > 0 and (line[0] == name or line[0].endswith('/'+name)):
         return line
@@ -116,3 +94,126 @@ class OpenstackDiscovery(object):
 
     return None
 
+  def _find_python_package_version(self, client, package):
+    result = client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % package])
+    
+    s = result.output.strip()
+    parts = []
+    for p in s.split('.'):
+      if not p[0].isdigit(): break
+
+      parts.append(p)
+
+    version = '.'.join(parts)
+
+    return version
+
+  def _get_processes(self, client):
+    return [line.split() for line in client.run(['ps', '-Ao', 'cmd', '--no-headers']).output.split("\n")]
+
+  def _collect_host_id(self, client):
+    ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
+    result = client.run(['bash', '-c', 'ip link | grep "link/ether "'])
+    macs = []
+    for match in ether_re.finditer(result.output):
+      macs.append(match.group(1).replace(':', ''))
+    return ''.join(macs)
+
+  def _collect_host_network_addresses(self, client):
+    ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
+    addresses = []
+    result = client.run(['bash', '-c', 'ip address list | grep "inet "'])
+    for match in ipaddr_re.finditer(result.output):
+      addresses.append(match.group(1))
+    return addresses
+
+  def _get_keystone_db_data(self, client, command, env={}):
+    result = client.run(['keystone', command], update_env=env)
+    if result.return_code != 0:
+      return []
+
+    lines = result.output.strip().split("\n")
+
+    columns = []
+    last_pos = 0
+    l = lines[0]
+    while True:
+      pos = l.find('+', last_pos+1)
+      if pos == -1:
+        break
+
+      columns.append({'start': last_pos+1, 'end': pos-1})
+
+      last_pos = pos
+
+    l = lines[1]
+    for c in columns:
+      c['name'] = l[c['start']:c['end']].strip()
+
+    data = []
+    for l in lines[3:-1]:
+      d = dict()
+      for c in columns:
+        d[c['name']] = l[c['start']:c['end']].strip()
+
+      data.append(d)
+
+    return data
+
+  def _collect_keystone_data(self, client):
+    keystone_process = self._find_python_process(client, 'keystone-all')
+    if not keystone_process:
+      return None
+
+    p = index(keystone_process, lambda s: s == '--config-file')
+    if p != -1 and p+1 < len(keystone_process):
+      config_path = keystone_process[p+1]
+    else:
+      config_path = '/etc/keystone/keystone.conf'
+
+    keystone = KeystoneComponent(config_path)
+    keystone.version = self._find_python_package_version(client, 'keystone')
+    with client.open(config_path) as f:
+      keystone.config_contents = f.read()
+
+    token = keystone.config['DEFAULT']['admin_token']
+    host = keystone.config['DEFAULT']['bind_host']
+    if host == '0.0.0.0':
+      host = '127.0.0.1'
+    port = int(keystone.config['DEFAULT']['admin_port'])
+
+    keystone_env = {
+      'OS_SERVICE_TOKEN': token,
+      'OS_SERVICE_ENDPOINT': 'http://%s:%d/v2.0' % (host, port)
+    }
+
+    keystone.db = dict()
+    keystone.db['tenants'] = self._get_keystone_db_data(client, 'tenant-list', env=keystone_env)
+    keystone.db['users'] = self._get_keystone_db_data(client, 'user-list', env=keystone_env)
+    keystone.db['services'] = self._get_keystone_db_data(client, 'service-list', env=keystone_env)
+    keystone.db['endpoints'] = self._get_keystone_db_data(client, 'endpoint-list', env=keystone_env)
+
+    return keystone
+
+  def _collect_nova_data(self, client):
+    keystone_process = self._find_python_process(client, 'nova-compute')
+    if not keystone_process:
+      return None
+
+    p = index(keystone_process, lambda s: s == '--config-file')
+    if p != -1 and p+1 < len(keystone_process):
+      config_path = keystone_process[p+1]
+    else:
+      config_path = '/etc/nova/nova.conf'
+
+    nova_compute = NovaComputeComponent(config_path)
+    nova_compute.version = self._find_python_package_version(client, 'nova')
+    with client.open(config_path) as f:
+      nova_compute.config_contents = f.read()
+
+    nova_compute.paste_config_path = path_relative_to(nova_compute.config['DEFAULT']['api_paste_config'], os.path.dirname(config_path))
+    with client.open(nova_compute.paste_config_path) as f:
+      nova_compute.paste_config_contents = f.read()
+
+    return nova_compute
+

ostack_validator/inspections/__init__.py

@@ -1,2 +1,3 @@
 from ostack_validator.inspections.keystone_authtoken import KeystoneAuthtokenSettingsInspection
+from ostack_validator.inspections.keystone_endpoints import KeystoneEndpointsInspection
 

ostack_validator/inspections/keystone_authtoken.py

@@ -34,13 +34,50 @@ class KeystoneAuthtokenSettingsInspection(Inspection):
       if not authtoken_section: continue
 
       authtoken_settings = nova.paste_config[authtoken_section]
-      if not 'auth_host' in authtoken_settings:
-        openstack.report_issue(Issue(Issue.ERROR, 'Service "%s" on host "%s" miss "auth_host" setting in keystone authtoken config' % (nova.name, nova.host.name)))
-      elif not authtoken_settings['auth_host'] in keystone_addresses:
-        openstack.report_issue(Issue(Issue.ERROR, 'Service "%s" on host "%s" has incorrect "auth_host" setting in keystone authtoken config' % (nova.name, nova.host.name)))
 
-      if not 'auth_port' in authtoken_settings:
-        openstack.report_issue(Issue(Issue.ERROR, 'Service "%s" on host "%s" miss "auth_port" setting in keystone authtoken config' % (nova.name, nova.host.name)))
-      elif authtoken_settings['auth_port'] != keystone.config['DEFAULT']['admin_port']:
-        openstack.report_issue(Issue(Issue.ERROR, 'Service "%s" on host "%s" has incorrect "auth_port" setting in keystone authtoken config' % (nova.name, nova.host.name)))
+
+      def get_value(name):
+        return authtoken_settings[name] or nova.config['keystone_authtoken', name]
+
+      auth_host = get_value('auth_host')
+      auth_port = get_value('auth_port')
+      auth_protocol = get_value('auth_protocol')
+      admin_user = get_value('admin_user')
+      admin_password = get_value('admin_password')
+      admin_tenant_name = get_value('admin_tenant_name')
+      admin_token = get_value('admin_token')
+
+      msg_prefix = 'Service "%s" on host "%s"' % (nova.name, nova.host.name)
+
+      if not auth_host:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_host" setting in keystone authtoken config'))
+      elif not auth_host in keystone_addresses:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_host" setting in keystone authtoken config'))
+
+      if not auth_port:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_port" setting in keystone authtoken config'))
+      elif auth_port != keystone.config['DEFAULT']['admin_port']:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_port" setting in keystone authtoken config'))
+
+      if not auth_protocol:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "auth_protocol" setting in keystone authtoken config'))
+      elif not auth_protocol in ['http', 'https']:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has incorrect "auth_protocol" setting in keystone authtoken config'))
+
+      if not admin_user:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "admin_user" setting in keystone authtoken config'))
+      else:
+        user = find(keystone.db['users'], lambda u: u['name'] == admin_user)
+        if not user:
+          nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has "admin_user" that is missing in Keystone catalog'))
+
+      if not admin_tenant_name:
+        nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' miss "admin_tenant_name" setting in keystone authtoken config'))
+      else:
+        tenant = find(keystone.db['tenants'], lambda t: t['name'] == admin_tenant_name)
+        if not tenant:
+          nova.report_issue(Issue(Issue.ERROR, msg_prefix + ' has "admin_tenant_name" that is missing in Keystone catalog'))
+
+      if admin_token:
+        nova.report_issue(Issue(Issue.WARNING, msg_prefix + ' uses insecure admin_token for authentication'))
 

ostack_validator/inspections/keystone_endpoints.py

@@ -0,0 +1,41 @@
+from urlparse import urlparse
+
+from ostack_validator.common import Inspection, Issue, find
+
+class KeystoneEndpointsInspection(Inspection):
+  name = 'Keystone endpoints'
+  description = 'Validate that each keystone endpoint leads to proper service'
+
+  def inspect(self, openstack):
+    keystone = find(openstack.components, lambda c: c.name == 'keystone')
+    if not keystone:
+      return
+
+    for service in keystone.db['services']:
+      if service['type'] == 'compute':
+        endpoint = find(keystone.db['endpoints'], lambda e: e['service_id'] == service['id'])
+        if not endpoint:
+          keystone.report_issue(Issue(Issue.WARNING, 'Keystone catalog contains service "%s" that has no defined endpoints' % service['name']))
+          continue
+
+        for url_attr in ['adminurl', 'publicurl', 'internalurl']:
+          url = urlparse(endpoint[url_attr])
+
+          # TODO: resolve endpoint url host address
+          host = find(openstack.hosts, lambda h: url.hostname in h.network_addresses)
+          if not host:
+            keystone.report_issue(Issue(Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to unknown host' % (service['name'], service['id'], url_attr)))
+            continue
+
+          nova_compute = None
+          for c in host.components:
+            if c.name != 'nova-compute': continue
+
+            if c.config['DEFAULT', 'osapi_compute_listen'] in ['0.0.0.0', url.hostname] and c.config['DEFAULT', 'osapi_compute_listen_port'] == url.port:
+              nova_compute = c
+              break
+
+          if not nova_compute:
+            keystone.report_issue(Issue(Issue.ERROR, 'Keystone catalog has endpoint for service "%s" (id %s) that has "%s" set pointing to no service' % (service['name'], service['id'], url_attr)))
+
+

ostack_validator/model.py

@@ -3,7 +3,7 @@ import re
 import logging
 from itertools import groupby
 
-from ostack_validator.common import Mark, Issue, MarkedIssue, path_relative_to
+from ostack_validator.common import Mark, Issue, MarkedIssue
 from ostack_validator.schema import ConfigSchemaRegistry, TypeValidatorRegistry
 from ostack_validator.config_model import Configuration
 import ostack_validator.schemas
@@ -14,32 +14,44 @@ class IssueReporter(object):
     super(IssueReporter, self).__init__()
     self.issues = []
 
-  def report(self, issue):
+  def report_issue(self, issue):
     self.issues.append(issue)
 
-class Openstack(object):
+  @property
+  def all_issues(self):
+    return list(self.issues)
+
+class Openstack(IssueReporter):
   def __init__(self):
     super(Openstack, self).__init__()
     self.hosts = []
-    self.issue_reporter = IssueReporter()
 
   def add_host(self, host):
     self.hosts.append(host)
     host.parent = self
 
-  def report_issue(self, issue):
-    self.issue_reporter.report(issue)
+  @property
+  def all_issues(self):
+    result = super(Openstack, self).all_issues
+
+    for host in self.hosts:
+      result.extend(host.all_issues)
+
+    return result
 
   @property
-  def issues(self):
-    return self.issue_reporter.issues
+  def components(self):
+    components = []
+    for host in self.hosts:
+      components.extend(host.components)
 
-class Host(object):
-  def __init__(self, name, metadata, client):
+    return components
+
+
+class Host(IssueReporter):
+  def __init__(self, name):
     super(Host, self).__init__()
     self.name = name
-    self.metadata = metadata
-    self.client = client
     self.components = []
 
   def add_component(self, component):
@@ -51,34 +63,31 @@ class Host(object):
     return self.parent
 
   @property
-  def id(self):
-    ether_re = re.compile('link/ether (([0-9a-f]{2}:){5}([0-9a-f]{2})) ')
-    result = self.client.run(['bash', '-c', 'ip link | grep "link/ether "'])
-    macs = []
-    for match in ether_re.finditer(result.output):
-      macs.append(match.group(1).replace(':', ''))
-    return ''.join(macs)
-    
+  def all_issues(self):
+    result = super(Host, self).all_issues
+
+    for component in self.components:
+      result.extend(component.all_issues)
+
+    return result
+
+
+class Service(IssueReporter):
+  def __init__(self):
+    super(Service, self).__init__()
+    self.issues = []
+
+  def report_issue(self, issue):
+    self.issues.append(issue)
 
   @property
-  def network_addresses(self):
-    ipaddr_re = re.compile('inet (\d+\.\d+\.\d+\.\d+)/\d+')
-    addresses = []
-    result = self.client.run(['bash', '-c', 'ip address list | grep "inet "'])
-    for match in ipaddr_re.finditer(result.output):
-      addresses.append(match.group(1))
-    return addresses
+  def host(self):
+    return self.parent
 
-  def __getstate__(self):
-    return {
-      'name': self.name,
-      'metadata': self.metadata,
-      'client': None,
-      'components': self.components,
-      'parent': self.parent
-    }
+  @property
+  def openstack(self):
+    return self.host.openstack
 
-class Service(object): pass
 
 class OpenstackComponent(Service):
   logger = logging.getLogger('ostack_validator.model.openstack_component')
@@ -89,14 +98,6 @@ class OpenstackComponent(Service):
     self.config_path = config_path
     self.config_dir = os.path.dirname(config_path)
 
-  @property
-  def host(self):
-    return self.parent
-
-  @property
-  def openstack(self):
-    return self.host.openstack
-
   @property
   def config(self):
     if not hasattr(self, '_config'):
@@ -105,31 +106,11 @@ class OpenstackComponent(Service):
         self.logger.debug('No schema for component "%s" main config version %s. Skipping it' % (self.component, self.version))
         self._config = None
       else:
-        with self.host.client.open(self.config_path) as f:
-          config_contents = f.read()
-
-        self._config = self._parse_config_file(Mark('%s:%s' % (self.host.name, self.config_path)), config_contents, schema, self.openstack)
+        self._config = self._parse_config_file(Mark(self.config_path), self.config_contents, schema, self)
 
     return self._config
 
 
-  @property
-  def version(self):
-    if not hasattr(self, '_version'):
-      result = self.host.client.run(['python', '-c', 'import pkg_resources; version = pkg_resources.get_provider(pkg_resources.Requirement.parse("%s")).version; print(version)' % self.component])
-      
-      s = result.output.strip()
-      parts = []
-      for p in s.split('.'):
-        if not p[0].isdigit(): break
-
-        parts.append(p)
-
-      self._version = '.'.join(parts)
-
-    return self._version
-    
-
   def _parse_config_file(self, base_mark, config_contents, schema=None, issue_reporter=None):
     if issue_reporter:
       def report_issue(issue):
@@ -220,14 +201,10 @@ class NovaComputeComponent(OpenstackComponent):
   @property
   def paste_config(self):
     if not hasattr(self, '_paste_config'): 
-      paste_config_path = path_relative_to(self.config['DEFAULT']['api_paste_config'], self.config_dir)
-      with self.host.client.open(paste_config_path) as f:
-        paste_config_contents = f.read()
-
       self._paste_config = self._parse_config_file(
-        Mark('%s:%s' % (self.host.name, paste_config_path)),
-        paste_config_contents,
-        issue_reporter=self.openstack
+        Mark(self.paste_config_path),
+        self.paste_config_contents,
+        issue_reporter=self
       )
 
     return self._paste_config