Add /info and /{tenant}/info route to zuul-web

There are a few pieces of information that are useful to know in the web
layer.

websocket_url is a config setting that, if set, is needed by the
console streaming. We currently pass this in appended to the streaming
url as a url parameter (which since it's a URL is a bit extra odd)

The endpoint is normally relative to the webapp,
but may need to be overridden in cases like publishing the html and
javascript to a disconnected location such as the draft output into the
log server in openstack or publishing built html/javascript to swift.

Add WebInfo and TenantWebInfo objects and corresponding /info and
/{tenant}/info routes. As an alternative, we could collapse WebInfo
and TenantWebInfo to just WebInfo and leave the tenant field set to None
for the /info route.

Some of the API functions are optionally provided by
plugins. The github plugin provides webhook URLs and the SQLReporter
plugin is needed for the builds endpoints. Add a Capabilities object
that can report on the existance of such things and pass it to plugin
route registration so that capabilities can be registered.

Add support for configuring stats_url

The old zuul status page had sparklines and other graphs on it, which
are not present in the current one because the graphite server wasn't
parameterized.

Add a config setting allowing a URL to a graphite server to be set and
expose that in the /info endpoint. Since statsd itself can emit to multiple
different backends, add a setting for the type of server, defaulting to
graphite.

Change-Id: I606a3b2cdf03cb73aa3ffd69d9d64c171b23b97a

doc/source/admin/components.rst

@@ -660,6 +660,16 @@ sections of ``zuul.conf`` are used by the web server:
       Base URL on which the websocket service is exposed, if different
       than the base URL of the web app.
 
+   .. attr:: stats_url
+
+      Base URL from which statistics emitted via statsd can be queried.
+
+   .. attr:: stats_type
+      :default: graphite
+
+      Type of server hosting the statistics information. Currently only
+      'graphite' is supported by the dashboard.
+
    .. attr:: static_cache_expiry
       :default: 3600
 

tests/unit/test_web.py

@@ -28,11 +28,24 @@ import zuul.web
 from tests.base import ZuulTestCase, FIXTURE_DIR
 
 
-class TestWeb(ZuulTestCase):
+class FakeConfig(object):
+
+    def __init__(self, config):
+        self.config = config or {}
+
+    def has_option(self, section, option):
+        return option in self.config.get(section, {})
+
+    def get(self, section, option):
+        return self.config.get(section, {}).get(option)
+
+
+class BaseTestWeb(ZuulTestCase):
     tenant_config_file = 'config/single-tenant/main.yaml'
+    config_ini_data = {}
 
     def setUp(self):
-        super(TestWeb, self).setUp()
+        super(BaseTestWeb, self).setUp()
         self.executor_server.hold_jobs_in_build = True
         A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
         A.addApproval('Code-Review', 2)
@@ -42,10 +55,13 @@ class TestWeb(ZuulTestCase):
         self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
         self.waitUntilSettled()
 
+        self.zuul_ini_config = FakeConfig(self.config_ini_data)
         # Start the web server
         self.web = zuul.web.ZuulWeb(
             listen_address='127.0.0.1', listen_port=0,
-            gear_server='127.0.0.1', gear_port=self.gearman_server.port)
+            gear_server='127.0.0.1', gear_port=self.gearman_server.port,
+            info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config)
+        )
         loop = asyncio.new_event_loop()
         loop.set_debug(True)
         ws_thread = threading.Thread(target=self.web.run, args=(loop,))
@@ -72,7 +88,10 @@ class TestWeb(ZuulTestCase):
         self.executor_server.hold_jobs_in_build = False
         self.executor_server.release()
         self.waitUntilSettled()
-        super(TestWeb, self).tearDown()
+        super(BaseTestWeb, self).tearDown()
+
+
+class TestWeb(BaseTestWeb):
 
     def test_web_status(self):
         "Test that we can retrieve JSON status info"
@@ -215,3 +234,78 @@ class TestWeb(ZuulTestCase):
         e = self.assertRaises(
             urllib.error.HTTPError, urllib.request.urlopen, req)
         self.assertEqual(404, e.code)
+
+
+class TestInfo(BaseTestWeb):
+
+    def setUp(self):
+        super(TestInfo, self).setUp()
+        web_config = self.config_ini_data.get('web', {})
+        self.websocket_url = web_config.get('websocket_url')
+        self.stats_url = web_config.get('stats_url')
+        statsd_config = self.config_ini_data.get('statsd', {})
+        self.stats_prefix = statsd_config.get('prefix')
+
+    def test_info(self):
+        req = urllib.request.Request(
+            "http://localhost:%s/info" % self.port)
+        f = urllib.request.urlopen(req)
+        info = json.loads(f.read().decode('utf8'))
+        self.assertEqual(
+            info, {
+                "info": {
+                    "endpoint": "http://localhost:%s" % self.port,
+                    "capabilities": {
+                        "job_history": False
+                    },
+                    "stats": {
+                        "url": self.stats_url,
+                        "prefix": self.stats_prefix,
+                        "type": "graphite",
+                    },
+                    "websocket_url": self.websocket_url,
+                }
+            })
+
+    def test_tenant_info(self):
+        req = urllib.request.Request(
+            "http://localhost:%s/tenant-one/info" % self.port)
+        f = urllib.request.urlopen(req)
+        info = json.loads(f.read().decode('utf8'))
+        self.assertEqual(
+            info, {
+                "info": {
+                    "endpoint": "http://localhost:%s" % self.port,
+                    "tenant": "tenant-one",
+                    "capabilities": {
+                        "job_history": False
+                    },
+                    "stats": {
+                        "url": self.stats_url,
+                        "prefix": self.stats_prefix,
+                        "type": "graphite",
+                    },
+                    "websocket_url": self.websocket_url,
+                }
+            })
+
+
+class TestWebSocketInfo(TestInfo):
+
+    config_ini_data = {
+        'web': {
+            'websocket_url': 'wss://ws.example.com'
+        }
+    }
+
+
+class TestGraphiteUrl(TestInfo):
+
+    config_ini_data = {
+        'statsd': {
+            'prefix': 'example'
+        },
+        'web': {
+            'stats_url': 'https://graphite.example.com',
+        }
+    }

zuul/cmd/web.py

@@ -20,6 +20,7 @@ import sys
 import threading
 
 import zuul.cmd
+import zuul.model
 import zuul.web
 
 from zuul.lib.config import get_default
@@ -33,8 +34,11 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
         self.web.stop()
 
     def _run(self):
+        info = zuul.model.WebInfo.fromConfig(self.config)
+
         params = dict()
 
+        params['info'] = info
         params['listen_address'] = get_default(self.config,
                                                'web', 'listen_address',
                                                '127.0.0.1')

zuul/connection/__init__.py

@@ -75,11 +75,14 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
         still in use.  Anything in our cache that isn't in the supplied
         list should be safe to remove from the cache."""
 
-    def getWebHandlers(self, zuul_web):
+    def getWebHandlers(self, zuul_web, info):
         """Return a list of web handlers to register with zuul-web.
 
         :param zuul.web.ZuulWeb zuul_web:
             Zuul Web instance.
+        :param zuul.model.WebInfo info:
+            The WebInfo object for the Zuul Web instance. Can be used by
+            plugins to toggle API capabilities.
         :returns: List of `zuul.web.handler.BaseWebHandler` instances.
         """
         return []

zuul/driver/github/githubconnection.py

@@ -1141,7 +1141,7 @@ class GithubConnection(BaseConnection):
 
         return statuses
 
-    def getWebHandlers(self, zuul_web):
+    def getWebHandlers(self, zuul_web, info):
         return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')]
 
     def validateWebConfig(self, config, connections):

zuul/driver/sql/sqlconnection.py

@@ -125,7 +125,8 @@ class SQLConnection(BaseConnection):
 
         return zuul_buildset_table, zuul_build_table
 
-    def getWebHandlers(self, zuul_web):
+    def getWebHandlers(self, zuul_web, info):
+        info.capabilities.job_history = True
         return [
             SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
             StaticHandler(zuul_web, '/{tenant}/builds.html'),

zuul/model.py

@@ -24,6 +24,7 @@ import urllib.parse
 import textwrap
 
 from zuul import change_matcher
+from zuul.lib.config import get_default
 
 MERGER_MERGE = 1          # "git merge"
 MERGER_MERGE_RESOLVE = 2  # "git merge -s resolve"
@@ -3182,3 +3183,80 @@ class TimeDataBase(object):
         td = self._getTD(build)
         td.add(elapsed, result)
         td.save()
+
+
+class Capabilities(object):
+    """The set of capabilities this Zuul installation has.
+
+    Some plugins add elements to the external API. In order to
+    facilitate consumers knowing if functionality is available
+    or not, keep track of distinct capability flags.
+    """
+    def __init__(self, job_history=False):
+        self.job_history = job_history
+
+    def __repr__(self):
+        return '<Capabilities 0x%x %s>' % (id(self), self._renderFlags())
+
+    def _renderFlags(self):
+        d = self.toDict()
+        return " ".join(['{k}={v}'.format(k=k, v=v) for (k, v) in d.items()])
+
+    def copy(self):
+        return Capabilities(**self.toDict())
+
+    def toDict(self):
+        d = dict()
+        d['job_history'] = self.job_history
+        return d
+
+
+class WebInfo(object):
+    """Information about the system needed by zuul-web /info."""
+
+    def __init__(self, websocket_url=None, endpoint=None,
+                 capabilities=None, stats_url=None,
+                 stats_prefix=None, stats_type=None):
+        self.capabilities = capabilities or Capabilities()
+        self.websocket_url = websocket_url
+        self.stats_url = stats_url
+        self.stats_prefix = stats_prefix
+        self.stats_type = stats_type
+        self.endpoint = endpoint
+        self.tenant = None
+
+    def __repr__(self):
+        return '<WebInfo 0x%x capabilities=%s>' % (
+            id(self), str(self.capabilities))
+
+    def copy(self):
+        return WebInfo(
+            websocket_url=self.websocket_url,
+            endpoint=self.endpoint,
+            stats_url=self.stats_url,
+            stats_prefix=self.stats_prefix,
+            stats_type=self.stats_type,
+            capabilities=self.capabilities.copy())
+
+    @staticmethod
+    def fromConfig(config):
+        return WebInfo(
+            websocket_url=get_default(config, 'web', 'websocket_url', None),
+            stats_url=get_default(config, 'web', 'stats_url', None),
+            stats_prefix=get_default(config, 'statsd', 'prefix'),
+            stats_type=get_default(config, 'web', 'stats_type', 'graphite'),
+        )
+
+    def toDict(self):
+        d = dict()
+        d['websocket_url'] = self.websocket_url
+        stats = dict()
+        stats['url'] = self.stats_url
+        stats['prefix'] = self.stats_prefix
+        stats['type'] = self.stats_type
+        d['stats'] = stats
+        d['endpoint'] = self.endpoint
+        d['capabilities'] = self.capabilities.toDict()
+        if self.tenant:
+            d['tenant'] = self.tenant
+        return d

zuul/web/__init__.py

@@ -26,6 +26,7 @@ import uvloop
 import aiohttp
 from aiohttp import web
 
+import zuul.model
 import zuul.rpcclient
 from zuul.web.handler import StaticHandler
 
@@ -235,13 +236,16 @@ class ZuulWeb(object):
                  gear_server, gear_port,
                  ssl_key=None, ssl_cert=None, ssl_ca=None,
                  static_cache_expiry=3600,
-                 connections=None):
+                 connections=None,
+                 info=None):
+        self.start_time = time.time()
         self.listen_address = listen_address
         self.listen_port = listen_port
         self.event_loop = None
         self.term = None
         self.server = None
         self.static_cache_expiry = static_cache_expiry
+        self.info = info
         # instanciate handlers
         self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
                                             ssl_key, ssl_cert, ssl_ca)
@@ -250,12 +254,37 @@ class ZuulWeb(object):
         self._plugin_routes = []  # type: List[zuul.web.handler.BaseWebHandler]
         connections = connections or []
         for connection in connections:
-            self._plugin_routes.extend(connection.getWebHandlers(self))
+            self._plugin_routes.extend(
+                connection.getWebHandlers(self, self.info))
 
     async def _handleWebsocket(self, request):
         return await self.log_streaming_handler.processRequest(
             request)
 
+    async def _handleRootInfo(self, request):
+        info = self.info.copy()
+        info.endpoint = str(request.url.parent)
+        return self._handleInfo(info)
+
+    def _handleTenantInfo(self, request):
+        info = self.info.copy()
+        info.tenant = request.match_info["tenant"]
+        # yarl.URL.parent on a root url returns the root url, so this is
+        # both safe and accurate for white-labeled tenants like OpenStack,
+        # zuul-web running on / and zuul-web running on a sub-url like
+        # softwarefactory-project.io
+        info.endpoint = str(request.url.parent.parent.parent)
+        return self._handleInfo(info)
+
+    def _handleInfo(self, info):
+        resp = web.json_response({'info': info.toDict()}, status=200)
+        resp.headers['Access-Control-Allow-Origin'] = '*'
+        if self.static_cache_expiry:
+            resp.headers['Cache-Control'] = "public, max-age=%d" % \
+                self.static_cache_expiry
+        resp.last_modified = self.start_time
+        return resp
+
     async def _handleTenantsRequest(self, request):
         return await self.gearman_handler.processRequest(request,
                                                          'tenant_list')
@@ -286,6 +315,8 @@ class ZuulWeb(object):
             is run within a separate (non-main) thread.
         """
         routes = [
+            ('GET', '/info', self._handleRootInfo),
+            ('GET', '/{tenant}/info', self._handleTenantInfo),
             ('GET', '/tenants', self._handleTenantsRequest),
             ('GET', '/{tenant}/status', self._handleStatusRequest),
             ('GET', '/{tenant}/jobs', self._handleJobsRequest),