Browse Source

Add /info and /{tenant}/info route to zuul-web

There are a few pieces of information that are useful to know in the web
layer.

websocket_url is a config setting that, if set, is needed by the
console streaming. We currently pass this in appended to the streaming
url as a url parameter (which since it's a URL is a bit extra odd)

The endpoint is normally relative to the webapp,
but may need to be overridden in cases like publishing the html and
javascript to a disconnected location such as the draft output into the
log server in openstack or publishing built html/javascript to swift.

Add WebInfo and TenantWebInfo objects and corresponding /info and
/{tenant}/info routes. As an alternative, we could collapse WebInfo
and TenantWebInfo to just WebInfo and leave the tenant field set to None
for the /info route.

Some of the API functions are optionally provided by
plugins. The github plugin provides webhook URLs and the SQLReporter
plugin is needed for the builds endpoints. Add a Capabilities object
that can report on the existance of such things and pass it to plugin
route registration so that capabilities can be registered.

Add support for configuring stats_url

The old zuul status page had sparklines and other graphs on it, which
are not present in the current one because the graphite server wasn't
parameterized.

Add a config setting allowing a URL to a graphite server to be set and
expose that in the /info endpoint. Since statsd itself can emit to multiple
different backends, add a setting for the type of server, defaulting to
graphite.

Change-Id: I606a3b2cdf03cb73aa3ffd69d9d64c171b23b97a
changes/11/537011/9
Monty Taylor 4 years ago
parent
commit
518dcf8bdb
No known key found for this signature in database GPG Key ID: 7BAE94BC7141A594
  1. 10
      doc/source/admin/components.rst
  2. 102
      tests/unit/test_web.py
  3. 4
      zuul/cmd/web.py
  4. 5
      zuul/connection/__init__.py
  5. 2
      zuul/driver/github/githubconnection.py
  6. 3
      zuul/driver/sql/sqlconnection.py
  7. 78
      zuul/model.py
  8. 35
      zuul/web/__init__.py

10
doc/source/admin/components.rst

@ -660,6 +660,16 @@ sections of ``zuul.conf`` are used by the web server:
Base URL on which the websocket service is exposed, if different
than the base URL of the web app.
.. attr:: stats_url
Base URL from which statistics emitted via statsd can be queried.
.. attr:: stats_type
:default: graphite
Type of server hosting the statistics information. Currently only
'graphite' is supported by the dashboard.
.. attr:: static_cache_expiry
:default: 3600

102
tests/unit/test_web.py

@ -28,11 +28,24 @@ import zuul.web
from tests.base import ZuulTestCase, FIXTURE_DIR
class TestWeb(ZuulTestCase):
class FakeConfig(object):
def __init__(self, config):
self.config = config or {}
def has_option(self, section, option):
return option in self.config.get(section, {})
def get(self, section, option):
return self.config.get(section, {}).get(option)
class BaseTestWeb(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'
config_ini_data = {}
def setUp(self):
super(TestWeb, self).setUp()
super(BaseTestWeb, self).setUp()
self.executor_server.hold_jobs_in_build = True
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
A.addApproval('Code-Review', 2)
@ -42,10 +55,13 @@ class TestWeb(ZuulTestCase):
self.fake_gerrit.addEvent(B.addApproval('Approved', 1))
self.waitUntilSettled()
self.zuul_ini_config = FakeConfig(self.config_ini_data)
# Start the web server
self.web = zuul.web.ZuulWeb(
listen_address='127.0.0.1', listen_port=0,
gear_server='127.0.0.1', gear_port=self.gearman_server.port)
gear_server='127.0.0.1', gear_port=self.gearman_server.port,
info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config)
)
loop = asyncio.new_event_loop()
loop.set_debug(True)
ws_thread = threading.Thread(target=self.web.run, args=(loop,))
@ -72,7 +88,10 @@ class TestWeb(ZuulTestCase):
self.executor_server.hold_jobs_in_build = False
self.executor_server.release()
self.waitUntilSettled()
super(TestWeb, self).tearDown()
super(BaseTestWeb, self).tearDown()
class TestWeb(BaseTestWeb):
def test_web_status(self):
"Test that we can retrieve JSON status info"
@ -215,3 +234,78 @@ class TestWeb(ZuulTestCase):
e = self.assertRaises(
urllib.error.HTTPError, urllib.request.urlopen, req)
self.assertEqual(404, e.code)
class TestInfo(BaseTestWeb):
def setUp(self):
super(TestInfo, self).setUp()
web_config = self.config_ini_data.get('web', {})
self.websocket_url = web_config.get('websocket_url')
self.stats_url = web_config.get('stats_url')
statsd_config = self.config_ini_data.get('statsd', {})
self.stats_prefix = statsd_config.get('prefix')
def test_info(self):
req = urllib.request.Request(
"http://localhost:%s/info" % self.port)
f = urllib.request.urlopen(req)
info = json.loads(f.read().decode('utf8'))
self.assertEqual(
info, {
"info": {
"endpoint": "http://localhost:%s" % self.port,
"capabilities": {
"job_history": False
},
"stats": {
"url": self.stats_url,
"prefix": self.stats_prefix,
"type": "graphite",
},
"websocket_url": self.websocket_url,
}
})
def test_tenant_info(self):
req = urllib.request.Request(
"http://localhost:%s/tenant-one/info" % self.port)
f = urllib.request.urlopen(req)
info = json.loads(f.read().decode('utf8'))
self.assertEqual(
info, {
"info": {
"endpoint": "http://localhost:%s" % self.port,
"tenant": "tenant-one",
"capabilities": {
"job_history": False
},
"stats": {
"url": self.stats_url,
"prefix": self.stats_prefix,
"type": "graphite",
},
"websocket_url": self.websocket_url,
}
})
class TestWebSocketInfo(TestInfo):
config_ini_data = {
'web': {
'websocket_url': 'wss://ws.example.com'
}
}
class TestGraphiteUrl(TestInfo):
config_ini_data = {
'statsd': {
'prefix': 'example'
},
'web': {
'stats_url': 'https://graphite.example.com',
}
}

4
zuul/cmd/web.py

@ -20,6 +20,7 @@ import sys
import threading
import zuul.cmd
import zuul.model
import zuul.web
from zuul.lib.config import get_default
@ -33,8 +34,11 @@ class WebServer(zuul.cmd.ZuulDaemonApp):
self.web.stop()
def _run(self):
info = zuul.model.WebInfo.fromConfig(self.config)
params = dict()
params['info'] = info
params['listen_address'] = get_default(self.config,
'web', 'listen_address',
'127.0.0.1')

5
zuul/connection/__init__.py

@ -75,11 +75,14 @@ class BaseConnection(object, metaclass=abc.ABCMeta):
still in use. Anything in our cache that isn't in the supplied
list should be safe to remove from the cache."""
def getWebHandlers(self, zuul_web):
def getWebHandlers(self, zuul_web, info):
"""Return a list of web handlers to register with zuul-web.
:param zuul.web.ZuulWeb zuul_web:
Zuul Web instance.
:param zuul.model.WebInfo info:
The WebInfo object for the Zuul Web instance. Can be used by
plugins to toggle API capabilities.
:returns: List of `zuul.web.handler.BaseWebHandler` instances.
"""
return []

2
zuul/driver/github/githubconnection.py

@ -1141,7 +1141,7 @@ class GithubConnection(BaseConnection):
return statuses
def getWebHandlers(self, zuul_web):
def getWebHandlers(self, zuul_web, info):
return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')]
def validateWebConfig(self, config, connections):

3
zuul/driver/sql/sqlconnection.py

@ -125,7 +125,8 @@ class SQLConnection(BaseConnection):
return zuul_buildset_table, zuul_build_table
def getWebHandlers(self, zuul_web):
def getWebHandlers(self, zuul_web, info):
info.capabilities.job_history = True
return [
SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
StaticHandler(zuul_web, '/{tenant}/builds.html'),

78
zuul/model.py

@ -24,6 +24,7 @@ import urllib.parse
import textwrap
from zuul import change_matcher
from zuul.lib.config import get_default
MERGER_MERGE = 1 # "git merge"
MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve"
@ -3182,3 +3183,80 @@ class TimeDataBase(object):
td = self._getTD(build)
td.add(elapsed, result)
td.save()
class Capabilities(object):
"""The set of capabilities this Zuul installation has.
Some plugins add elements to the external API. In order to
facilitate consumers knowing if functionality is available
or not, keep track of distinct capability flags.
"""
def __init__(self, job_history=False):
self.job_history = job_history
def __repr__(self):
return '<Capabilities 0x%x %s>' % (id(self), self._renderFlags())
def _renderFlags(self):
d = self.toDict()
return " ".join(['{k}={v}'.format(k=k, v=v) for (k, v) in d.items()])
def copy(self):
return Capabilities(**self.toDict())
def toDict(self):
d = dict()
d['job_history'] = self.job_history
return d
class WebInfo(object):
"""Information about the system needed by zuul-web /info."""
def __init__(self, websocket_url=None, endpoint=None,
capabilities=None, stats_url=None,
stats_prefix=None, stats_type=None):
self.capabilities = capabilities or Capabilities()
self.websocket_url = websocket_url
self.stats_url = stats_url
self.stats_prefix = stats_prefix
self.stats_type = stats_type
self.endpoint = endpoint
self.tenant = None
def __repr__(self):
return '<WebInfo 0x%x capabilities=%s>' % (
id(self), str(self.capabilities))
def copy(self):
return WebInfo(
websocket_url=self.websocket_url,
endpoint=self.endpoint,
stats_url=self.stats_url,
stats_prefix=self.stats_prefix,
stats_type=self.stats_type,
capabilities=self.capabilities.copy())
@staticmethod
def fromConfig(config):
return WebInfo(
websocket_url=get_default(config, 'web', 'websocket_url', None),
stats_url=get_default(config, 'web', 'stats_url', None),
stats_prefix=get_default(config, 'statsd', 'prefix'),
stats_type=get_default(config, 'web', 'stats_type', 'graphite'),
)
def toDict(self):
d = dict()
d['websocket_url'] = self.websocket_url
stats = dict()
stats['url'] = self.stats_url
stats['prefix'] = self.stats_prefix
stats['type'] = self.stats_type
d['stats'] = stats
d['endpoint'] = self.endpoint
d['capabilities'] = self.capabilities.toDict()
if self.tenant:
d['tenant'] = self.tenant
return d

35
zuul/web/__init__.py

@ -26,6 +26,7 @@ import uvloop
import aiohttp
from aiohttp import web
import zuul.model
import zuul.rpcclient
from zuul.web.handler import StaticHandler
@ -235,13 +236,16 @@ class ZuulWeb(object):
gear_server, gear_port,
ssl_key=None, ssl_cert=None, ssl_ca=None,
static_cache_expiry=3600,
connections=None):
connections=None,
info=None):
self.start_time = time.time()
self.listen_address = listen_address
self.listen_port = listen_port
self.event_loop = None
self.term = None
self.server = None
self.static_cache_expiry = static_cache_expiry
self.info = info
# instanciate handlers
self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port,
ssl_key, ssl_cert, ssl_ca)
@ -250,12 +254,37 @@ class ZuulWeb(object):
self._plugin_routes = [] # type: List[zuul.web.handler.BaseWebHandler]
connections = connections or []
for connection in connections:
self._plugin_routes.extend(connection.getWebHandlers(self))
self._plugin_routes.extend(
connection.getWebHandlers(self, self.info))
async def _handleWebsocket(self, request):
return await self.log_streaming_handler.processRequest(
request)
async def _handleRootInfo(self, request):
info = self.info.copy()
info.endpoint = str(request.url.parent)
return self._handleInfo(info)
def _handleTenantInfo(self, request):
info = self.info.copy()
info.tenant = request.match_info["tenant"]
# yarl.URL.parent on a root url returns the root url, so this is
# both safe and accurate for white-labeled tenants like OpenStack,
# zuul-web running on / and zuul-web running on a sub-url like
# softwarefactory-project.io
info.endpoint = str(request.url.parent.parent.parent)
return self._handleInfo(info)
def _handleInfo(self, info):
resp = web.json_response({'info': info.toDict()}, status=200)
resp.headers['Access-Control-Allow-Origin'] = '*'
if self.static_cache_expiry:
resp.headers['Cache-Control'] = "public, max-age=%d" % \
self.static_cache_expiry
resp.last_modified = self.start_time
return resp
async def _handleTenantsRequest(self, request):
return await self.gearman_handler.processRequest(request,
'tenant_list')
@ -286,6 +315,8 @@ class ZuulWeb(object):
is run within a separate (non-main) thread.
"""
routes = [
('GET', '/info', self._handleRootInfo),
('GET', '/{tenant}/info', self._handleTenantInfo),
('GET', '/tenants', self._handleTenantsRequest),
('GET', '/{tenant}/status', self._handleStatusRequest),
('GET', '/{tenant}/jobs', self._handleJobsRequest),

Loading…
Cancel
Save