Add /info and /{tenant}/info route to zuul-web
There are a few pieces of information that are useful to know in the web layer. websocket_url is a config setting that, if set, is needed by the console streaming. We currently pass this in appended to the streaming url as a url parameter (which since it's a URL is a bit extra odd) The endpoint is normally relative to the webapp, but may need to be overridden in cases like publishing the html and javascript to a disconnected location such as the draft output into the log server in openstack or publishing built html/javascript to swift. Add WebInfo and TenantWebInfo objects and corresponding /info and /{tenant}/info routes. As an alternative, we could collapse WebInfo and TenantWebInfo to just WebInfo and leave the tenant field set to None for the /info route. Some of the API functions are optionally provided by plugins. The github plugin provides webhook URLs and the SQLReporter plugin is needed for the builds endpoints. Add a Capabilities object that can report on the existance of such things and pass it to plugin route registration so that capabilities can be registered. Add support for configuring stats_url The old zuul status page had sparklines and other graphs on it, which are not present in the current one because the graphite server wasn't parameterized. Add a config setting allowing a URL to a graphite server to be set and expose that in the /info endpoint. Since statsd itself can emit to multiple different backends, add a setting for the type of server, defaulting to graphite. Change-Id: I606a3b2cdf03cb73aa3ffd69d9d64c171b23b97a
This commit is contained in:
parent
9db66082d3
commit
518dcf8bdb
|
@ -660,6 +660,16 @@ sections of ``zuul.conf`` are used by the web server:
|
|||
Base URL on which the websocket service is exposed, if different
|
||||
than the base URL of the web app.
|
||||
|
||||
.. attr:: stats_url
|
||||
|
||||
Base URL from which statistics emitted via statsd can be queried.
|
||||
|
||||
.. attr:: stats_type
|
||||
:default: graphite
|
||||
|
||||
Type of server hosting the statistics information. Currently only
|
||||
'graphite' is supported by the dashboard.
|
||||
|
||||
.. attr:: static_cache_expiry
|
||||
:default: 3600
|
||||
|
||||
|
|
|
@ -28,11 +28,24 @@ import zuul.web
|
|||
from tests.base import ZuulTestCase, FIXTURE_DIR
|
||||
|
||||
|
||||
class TestWeb(ZuulTestCase):
|
||||
class FakeConfig(object):
|
||||
|
||||
def __init__(self, config):
|
||||
self.config = config or {}
|
||||
|
||||
def has_option(self, section, option):
|
||||
return option in self.config.get(section, {})
|
||||
|
||||
def get(self, section, option):
|
||||
return self.config.get(section, {}).get(option)
|
||||
|
||||
|
||||
class BaseTestWeb(ZuulTestCase):
|
||||
tenant_config_file = 'config/single-tenant/main.yaml'
|
||||
config_ini_data = {}
|
||||
|
||||
def setUp(self):
|
||||
super(TestWeb, self).setUp()
|
||||
super(BaseTestWeb, self).setUp()
|
||||
self.executor_server.hold_jobs_in_build = True
|
||||
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
|
||||
A.addApproval('Code-Review', 2)
|
||||
|
@ -42,10 +55,13 @@ class TestWeb(ZuulTestCase):
|
|||
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
|
||||
self.waitUntilSettled()
|
||||
|
||||
self.zuul_ini_config = FakeConfig(self.config_ini_data)
|
||||
# Start the web server
|
||||
self.web = zuul.web.ZuulWeb(
|
||||
listen_address='127.0.0.1', listen_port=0,
|
||||
gear_server='127.0.0.1', gear_port=self.gearman_server.port)
|
||||
gear_server='127.0.0.1', gear_port=self.gearman_server.port,
|
||||
info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config)
|
||||
)
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.set_debug(True)
|
||||
ws_thread = threading.Thread(target=self.web.run, args=(loop,))
|
||||
|
@ -72,7 +88,10 @@ class TestWeb(ZuulTestCase):
|
|||
self.executor_server.hold_jobs_in_build = False
|
||||
self.executor_server.release()
|
||||
self.waitUntilSettled()
|
||||
super(TestWeb, self).tearDown()
|
||||
super(BaseTestWeb, self).tearDown()
|
||||
|
||||
|
||||
class TestWeb(BaseTestWeb):
|
||||
|
||||
def test_web_status(self):
|
||||
"Test that we can retrieve JSON status info"
|
||||
|
@ -215,3 +234,78 @@ class TestWeb(ZuulTestCase):
|
|||
e = self.assertRaises(
|
||||
urllib.error.HTTPError, urllib.request.urlopen, req)
|
||||
self.assertEqual(404, e.code)
|
||||
|
||||
|
||||
class TestInfo(BaseTestWeb):
|
||||
|
||||
def setUp(self):
|
||||
super(TestInfo, self).setUp()
|
||||
web_config = self.config_ini_data.get('web', {})
|
||||
self.websocket_url = web_config.get('websocket_url')
|
||||
self.stats_url = web_config.get('stats_url')
|
||||
statsd_config = self.config_ini_data.get('statsd', {})
|
||||
self.stats_prefix = statsd_config.get('prefix')
|
||||
|
||||
def test_info(self):
|
||||
req = urllib.request.Request(
|
||||
"http://localhost:%s/info" % self.port)
|
||||
f = urllib.request.urlopen(req)
|
||||
info = json.loads(f.read().decode('utf8'))
|
||||
self.assertEqual(
|
||||
info, {
|
||||
"info": {
|
||||
"endpoint": "http://localhost:%s" % self.port,
|
||||
"capabilities": {
|
||||
"job_history": False
|
||||
},
|
||||
"stats": {
|
||||
"url": self.stats_url,
|
||||
"prefix": self.stats_prefix,
|
||||
"type": "graphite",
|
||||
},
|
||||
"websocket_url": self.websocket_url,
|
||||
}
|
||||
})
|
||||
|
||||
def test_tenant_info(self):
|
||||
req = urllib.request.Request(
|
||||
"http://localhost:%s/tenant-one/info" % self.port)
|
||||
f = urllib.request.urlopen(req)
|
||||
info = json.loads(f.read().decode('utf8'))
|
||||
self.assertEqual(
|
||||
info, {
|
||||
"info": {
|
||||
"endpoint": "http://localhost:%s" % self.port,
|
||||
"tenant": "tenant-one",
|
||||
"capabilities": {
|
||||
"job_history": False
|
||||
},
|
||||
"stats": {
|
||||
"url": self.stats_url,
|
||||
"prefix": self.stats_prefix,
|
||||
"type": "graphite",
|
||||
},
|
||||
"websocket_url": self.websocket_url,
|
||||
}
|
||||
})
|
||||
|
||||
|
||||
class TestWebSocketInfo(TestInfo):
|
||||
|
||||
config_ini_data = {
|
||||
'web': {
|
||||
'websocket_url': 'wss://ws.example.com'
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
class TestGraphiteUrl(TestInfo):
|
||||
|
||||
config_ini_data = {
|
||||
'statsd': {
|
||||
'prefix': 'example'
|
||||
},
|
||||
'web': {
|
||||
'stats_url': 'https://graphite.example.com',
|
||||
}
|
||||
}
|
||||
|
|
|
@ -20,6 +20,7 @@ import sys
|
|||
import threading
|
||||
|
||||
import zuul.cmd
|
||||
import zuul.model
|
||||
import zuul.web
|
||||
|
||||
from zuul.lib.config import get_default
|
||||
|
@ -33,8 +34,11 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
|
|||
self.web.stop()
|
||||
|
||||
def _run(self):
|
||||
info = zuul.model.WebInfo.fromConfig(self.config)
|
||||
|
||||
params = dict()
|
||||
|
||||
params['info'] = info
|
||||
params['listen_address'] = get_default(self.config,
|
||||
'web', 'listen_address',
|
||||
'127.0.0.1')
|
||||
|
|
|
@ -75,11 +75,14 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
|
|||
still in use. Anything in our cache that isn't in the supplied
|
||||
list should be safe to remove from the cache."""
|
||||
|
||||
def getWebHandlers(self, zuul_web):
|
||||
def getWebHandlers(self, zuul_web, info):
|
||||
"""Return a list of web handlers to register with zuul-web.
|
||||
|
||||
:param zuul.web.ZuulWeb zuul_web:
|
||||
Zuul Web instance.
|
||||
:param zuul.model.WebInfo info:
|
||||
The WebInfo object for the Zuul Web instance. Can be used by
|
||||
plugins to toggle API capabilities.
|
||||
:returns: List of `zuul.web.handler.BaseWebHandler` instances.
|
||||
"""
|
||||
return []
|
||||
|
|
|
@ -1141,7 +1141,7 @@ class GithubConnection(BaseConnection):
|
|||
|
||||
return statuses
|
||||
|
||||
def getWebHandlers(self, zuul_web):
|
||||
def getWebHandlers(self, zuul_web, info):
|
||||
return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')]
|
||||
|
||||
def validateWebConfig(self, config, connections):
|
||||
|
|
|
@ -125,7 +125,8 @@ class SQLConnection(BaseConnection):
|
|||
|
||||
return zuul_buildset_table, zuul_build_table
|
||||
|
||||
def getWebHandlers(self, zuul_web):
|
||||
def getWebHandlers(self, zuul_web, info):
|
||||
info.capabilities.job_history = True
|
||||
return [
|
||||
SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
|
||||
StaticHandler(zuul_web, '/{tenant}/builds.html'),
|
||||
|
|
|
@ -24,6 +24,7 @@ import urllib.parse
|
|||
import textwrap
|
||||
|
||||
from zuul import change_matcher
|
||||
from zuul.lib.config import get_default
|
||||
|
||||
MERGER_MERGE = 1 # "git merge"
|
||||
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
|
||||
|
@ -3182,3 +3183,80 @@ class TimeDataBase(object):
|
|||
td = self._getTD(build)
|
||||
td.add(elapsed, result)
|
||||
td.save()
|
||||
|
||||
|
||||
class Capabilities(object):
|
||||
"""The set of capabilities this Zuul installation has.
|
||||
|
||||
Some plugins add elements to the external API. In order to
|
||||
facilitate consumers knowing if functionality is available
|
||||
or not, keep track of distinct capability flags.
|
||||
"""
|
||||
def __init__(self, job_history=False):
|
||||
self.job_history = job_history
|
||||
|
||||
def __repr__(self):
|
||||
return '<Capabilities 0x%x %s>' % (id(self), self._renderFlags())
|
||||
|
||||
def _renderFlags(self):
|
||||
d = self.toDict()
|
||||
return " ".join(['{k}={v}'.format(k=k, v=v) for (k, v) in d.items()])
|
||||
|
||||
def copy(self):
|
||||
return Capabilities(**self.toDict())
|
||||
|
||||
def toDict(self):
|
||||
d = dict()
|
||||
d['job_history'] = self.job_history
|
||||
return d
|
||||
|
||||
|
||||
class WebInfo(object):
|
||||
"""Information about the system needed by zuul-web /info."""
|
||||
|
||||
def __init__(self, websocket_url=None, endpoint=None,
|
||||
capabilities=None, stats_url=None,
|
||||
stats_prefix=None, stats_type=None):
|
||||
self.capabilities = capabilities or Capabilities()
|
||||
self.websocket_url = websocket_url
|
||||
self.stats_url = stats_url
|
||||
self.stats_prefix = stats_prefix
|
||||
self.stats_type = stats_type
|
||||
self.endpoint = endpoint
|
||||
self.tenant = None
|
||||
|
||||
def __repr__(self):
|
||||
return '<WebInfo 0x%x capabilities=%s>' % (
|
||||
id(self), str(self.capabilities))
|
||||
|
||||
def copy(self):
|
||||
return WebInfo(
|
||||
websocket_url=self.websocket_url,
|
||||
endpoint=self.endpoint,
|
||||
stats_url=self.stats_url,
|
||||
stats_prefix=self.stats_prefix,
|
||||
stats_type=self.stats_type,
|
||||
capabilities=self.capabilities.copy())
|
||||
|
||||
@staticmethod
|
||||
def fromConfig(config):
|
||||
return WebInfo(
|
||||
websocket_url=get_default(config, 'web', 'websocket_url', None),
|
||||
stats_url=get_default(config, 'web', 'stats_url', None),
|
||||
stats_prefix=get_default(config, 'statsd', 'prefix'),
|
||||
stats_type=get_default(config, 'web', 'stats_type', 'graphite'),
|
||||
)
|
||||
|
||||
def toDict(self):
|
||||
d = dict()
|
||||
d['websocket_url'] = self.websocket_url
|
||||
stats = dict()
|
||||
stats['url'] = self.stats_url
|
||||
stats['prefix'] = self.stats_prefix
|
||||
stats['type'] = self.stats_type
|
||||
d['stats'] = stats
|
||||
d['endpoint'] = self.endpoint
|
||||
d['capabilities'] = self.capabilities.toDict()
|
||||
if self.tenant:
|
||||
d['tenant'] = self.tenant
|
||||
return d
|
||||
|
|
|
@ -26,6 +26,7 @@ import uvloop
|
|||
import aiohttp
|
||||
from aiohttp import web
|
||||
|
||||
import zuul.model
|
||||
import zuul.rpcclient
|
||||
from zuul.web.handler import StaticHandler
|
||||
|
||||
|
@ -235,13 +236,16 @@ class ZuulWeb(object):
|
|||
gear_server, gear_port,
|
||||
ssl_key=None, ssl_cert=None, ssl_ca=None,
|
||||
static_cache_expiry=3600,
|
||||
connections=None):
|
||||
connections=None,
|
||||
info=None):
|
||||
self.start_time = time.time()
|
||||
self.listen_address = listen_address
|
||||
self.listen_port = listen_port
|
||||
self.event_loop = None
|
||||
self.term = None
|
||||
self.server = None
|
||||
self.static_cache_expiry = static_cache_expiry
|
||||
self.info = info
|
||||
# instanciate handlers
|
||||
self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
|
||||
ssl_key, ssl_cert, ssl_ca)
|
||||
|
@ -250,12 +254,37 @@ class ZuulWeb(object):
|
|||
self._plugin_routes = [] # type: List[zuul.web.handler.BaseWebHandler]
|
||||
connections = connections or []
|
||||
for connection in connections:
|
||||
self._plugin_routes.extend(connection.getWebHandlers(self))
|
||||
self._plugin_routes.extend(
|
||||
connection.getWebHandlers(self, self.info))
|
||||
|
||||
async def _handleWebsocket(self, request):
|
||||
return await self.log_streaming_handler.processRequest(
|
||||
request)
|
||||
|
||||
async def _handleRootInfo(self, request):
|
||||
info = self.info.copy()
|
||||
info.endpoint = str(request.url.parent)
|
||||
return self._handleInfo(info)
|
||||
|
||||
def _handleTenantInfo(self, request):
|
||||
info = self.info.copy()
|
||||
info.tenant = request.match_info["tenant"]
|
||||
# yarl.URL.parent on a root url returns the root url, so this is
|
||||
# both safe and accurate for white-labeled tenants like OpenStack,
|
||||
# zuul-web running on / and zuul-web running on a sub-url like
|
||||
# softwarefactory-project.io
|
||||
info.endpoint = str(request.url.parent.parent.parent)
|
||||
return self._handleInfo(info)
|
||||
|
||||
def _handleInfo(self, info):
|
||||
resp = web.json_response({'info': info.toDict()}, status=200)
|
||||
resp.headers['Access-Control-Allow-Origin'] = '*'
|
||||
if self.static_cache_expiry:
|
||||
resp.headers['Cache-Control'] = "public, max-age=%d" % \
|
||||
self.static_cache_expiry
|
||||
resp.last_modified = self.start_time
|
||||
return resp
|
||||
|
||||
async def _handleTenantsRequest(self, request):
|
||||
return await self.gearman_handler.processRequest(request,
|
||||
'tenant_list')
|
||||
|
@ -286,6 +315,8 @@ class ZuulWeb(object):
|
|||
is run within a separate (non-main) thread.
|
||||
"""
|
||||
routes = [
|
||||
('GET', '/info', self._handleRootInfo),
|
||||
('GET', '/{tenant}/info', self._handleTenantInfo),
|
||||
('GET', '/tenants', self._handleTenantsRequest),
|
||||
('GET', '/{tenant}/status', self._handleStatusRequest),
|
||||
('GET', '/{tenant}/jobs', self._handleJobsRequest),
|
||||
|
|
Loading…
Reference in New Issue