diff --git a/doc/source/admin/components.rst b/doc/source/admin/components.rst index ba14752750..84ebc10617 100644 --- a/doc/source/admin/components.rst +++ b/doc/source/admin/components.rst @@ -660,6 +660,16 @@ sections of ``zuul.conf`` are used by the web server: Base URL on which the websocket service is exposed, if different than the base URL of the web app. + .. attr:: stats_url + + Base URL from which statistics emitted via statsd can be queried. + + .. attr:: stats_type + :default: graphite + + Type of server hosting the statistics information. Currently only + 'graphite' is supported by the dashboard. + .. attr:: static_cache_expiry :default: 3600 diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index e150a47840..602209f00e 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -28,11 +28,24 @@ import zuul.web from tests.base import ZuulTestCase, FIXTURE_DIR -class TestWeb(ZuulTestCase): +class FakeConfig(object): + + def __init__(self, config): + self.config = config or {} + + def has_option(self, section, option): + return option in self.config.get(section, {}) + + def get(self, section, option): + return self.config.get(section, {}).get(option) + + +class BaseTestWeb(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' + config_ini_data = {} def setUp(self): - super(TestWeb, self).setUp() + super(BaseTestWeb, self).setUp() self.executor_server.hold_jobs_in_build = True A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A') A.addApproval('Code-Review', 2) @@ -42,10 +55,13 @@ class TestWeb(ZuulTestCase): self.fake_gerrit.addEvent(B.addApproval('Approved', 1)) self.waitUntilSettled() + self.zuul_ini_config = FakeConfig(self.config_ini_data) # Start the web server self.web = zuul.web.ZuulWeb( listen_address='127.0.0.1', listen_port=0, - gear_server='127.0.0.1', gear_port=self.gearman_server.port) + gear_server='127.0.0.1', gear_port=self.gearman_server.port, + info=zuul.model.WebInfo.fromConfig(self.zuul_ini_config) + ) loop = asyncio.new_event_loop() loop.set_debug(True) ws_thread = threading.Thread(target=self.web.run, args=(loop,)) @@ -72,7 +88,10 @@ class TestWeb(ZuulTestCase): self.executor_server.hold_jobs_in_build = False self.executor_server.release() self.waitUntilSettled() - super(TestWeb, self).tearDown() + super(BaseTestWeb, self).tearDown() + + +class TestWeb(BaseTestWeb): def test_web_status(self): "Test that we can retrieve JSON status info" @@ -215,3 +234,78 @@ class TestWeb(ZuulTestCase): e = self.assertRaises( urllib.error.HTTPError, urllib.request.urlopen, req) self.assertEqual(404, e.code) + + +class TestInfo(BaseTestWeb): + + def setUp(self): + super(TestInfo, self).setUp() + web_config = self.config_ini_data.get('web', {}) + self.websocket_url = web_config.get('websocket_url') + self.stats_url = web_config.get('stats_url') + statsd_config = self.config_ini_data.get('statsd', {}) + self.stats_prefix = statsd_config.get('prefix') + + def test_info(self): + req = urllib.request.Request( + "http://localhost:%s/info" % self.port) + f = urllib.request.urlopen(req) + info = json.loads(f.read().decode('utf8')) + self.assertEqual( + info, { + "info": { + "endpoint": "http://localhost:%s" % self.port, + "capabilities": { + "job_history": False + }, + "stats": { + "url": self.stats_url, + "prefix": self.stats_prefix, + "type": "graphite", + }, + "websocket_url": self.websocket_url, + } + }) + + def test_tenant_info(self): + req = urllib.request.Request( + "http://localhost:%s/tenant-one/info" % self.port) + f = urllib.request.urlopen(req) + info = json.loads(f.read().decode('utf8')) + self.assertEqual( + info, { + "info": { + "endpoint": "http://localhost:%s" % self.port, + "tenant": "tenant-one", + "capabilities": { + "job_history": False + }, + "stats": { + "url": self.stats_url, + "prefix": self.stats_prefix, + "type": "graphite", + }, + "websocket_url": self.websocket_url, + } + }) + + +class TestWebSocketInfo(TestInfo): + + config_ini_data = { + 'web': { + 'websocket_url': 'wss://ws.example.com' + } + } + + +class TestGraphiteUrl(TestInfo): + + config_ini_data = { + 'statsd': { + 'prefix': 'example' + }, + 'web': { + 'stats_url': 'https://graphite.example.com', + } + } diff --git a/zuul/cmd/web.py b/zuul/cmd/web.py index abdb1cb405..8b0e3eefe6 100755 --- a/zuul/cmd/web.py +++ b/zuul/cmd/web.py @@ -20,6 +20,7 @@ import sys import threading import zuul.cmd +import zuul.model import zuul.web from zuul.lib.config import get_default @@ -33,8 +34,11 @@ class WebServer(zuul.cmd.ZuulDaemonApp): self.web.stop() def _run(self): + info = zuul.model.WebInfo.fromConfig(self.config) + params = dict() + params['info'] = info params['listen_address'] = get_default(self.config, 'web', 'listen_address', '127.0.0.1') diff --git a/zuul/connection/__init__.py b/zuul/connection/__init__.py index 86f14d6c2a..1c62f4dab0 100644 --- a/zuul/connection/__init__.py +++ b/zuul/connection/__init__.py @@ -75,11 +75,14 @@ class BaseConnection(object, metaclass=abc.ABCMeta): still in use. Anything in our cache that isn't in the supplied list should be safe to remove from the cache.""" - def getWebHandlers(self, zuul_web): + def getWebHandlers(self, zuul_web, info): """Return a list of web handlers to register with zuul-web. :param zuul.web.ZuulWeb zuul_web: Zuul Web instance. + :param zuul.model.WebInfo info: + The WebInfo object for the Zuul Web instance. Can be used by + plugins to toggle API capabilities. :returns: List of `zuul.web.handler.BaseWebHandler` instances. """ return [] diff --git a/zuul/driver/github/githubconnection.py b/zuul/driver/github/githubconnection.py index 6dfcdd3b99..772ba9b97a 100644 --- a/zuul/driver/github/githubconnection.py +++ b/zuul/driver/github/githubconnection.py @@ -1141,7 +1141,7 @@ class GithubConnection(BaseConnection): return statuses - def getWebHandlers(self, zuul_web): + def getWebHandlers(self, zuul_web, info): return [GithubWebhookHandler(self, zuul_web, 'POST', 'payload')] def validateWebConfig(self, config, connections): diff --git a/zuul/driver/sql/sqlconnection.py b/zuul/driver/sql/sqlconnection.py index d16a6230f4..e9313014a1 100644 --- a/zuul/driver/sql/sqlconnection.py +++ b/zuul/driver/sql/sqlconnection.py @@ -125,7 +125,8 @@ class SQLConnection(BaseConnection): return zuul_buildset_table, zuul_build_table - def getWebHandlers(self, zuul_web): + def getWebHandlers(self, zuul_web, info): + info.capabilities.job_history = True return [ SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'), StaticHandler(zuul_web, '/{tenant}/builds.html'), diff --git a/zuul/model.py b/zuul/model.py index bd4f4d11ee..44e8d06f05 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -24,6 +24,7 @@ import urllib.parse import textwrap from zuul import change_matcher +from zuul.lib.config import get_default MERGER_MERGE = 1 # "git merge" MERGER_MERGE_RESOLVE = 2 # "git merge -s resolve" @@ -3182,3 +3183,80 @@ class TimeDataBase(object): td = self._getTD(build) td.add(elapsed, result) td.save() + + +class Capabilities(object): + """The set of capabilities this Zuul installation has. + + Some plugins add elements to the external API. In order to + facilitate consumers knowing if functionality is available + or not, keep track of distinct capability flags. + """ + def __init__(self, job_history=False): + self.job_history = job_history + + def __repr__(self): + return '' % (id(self), self._renderFlags()) + + def _renderFlags(self): + d = self.toDict() + return " ".join(['{k}={v}'.format(k=k, v=v) for (k, v) in d.items()]) + + def copy(self): + return Capabilities(**self.toDict()) + + def toDict(self): + d = dict() + d['job_history'] = self.job_history + return d + + +class WebInfo(object): + """Information about the system needed by zuul-web /info.""" + + def __init__(self, websocket_url=None, endpoint=None, + capabilities=None, stats_url=None, + stats_prefix=None, stats_type=None): + self.capabilities = capabilities or Capabilities() + self.websocket_url = websocket_url + self.stats_url = stats_url + self.stats_prefix = stats_prefix + self.stats_type = stats_type + self.endpoint = endpoint + self.tenant = None + + def __repr__(self): + return '' % ( + id(self), str(self.capabilities)) + + def copy(self): + return WebInfo( + websocket_url=self.websocket_url, + endpoint=self.endpoint, + stats_url=self.stats_url, + stats_prefix=self.stats_prefix, + stats_type=self.stats_type, + capabilities=self.capabilities.copy()) + + @staticmethod + def fromConfig(config): + return WebInfo( + websocket_url=get_default(config, 'web', 'websocket_url', None), + stats_url=get_default(config, 'web', 'stats_url', None), + stats_prefix=get_default(config, 'statsd', 'prefix'), + stats_type=get_default(config, 'web', 'stats_type', 'graphite'), + ) + + def toDict(self): + d = dict() + d['websocket_url'] = self.websocket_url + stats = dict() + stats['url'] = self.stats_url + stats['prefix'] = self.stats_prefix + stats['type'] = self.stats_type + d['stats'] = stats + d['endpoint'] = self.endpoint + d['capabilities'] = self.capabilities.toDict() + if self.tenant: + d['tenant'] = self.tenant + return d diff --git a/zuul/web/__init__.py b/zuul/web/__init__.py index 41b1b81f79..7a1af306c9 100755 --- a/zuul/web/__init__.py +++ b/zuul/web/__init__.py @@ -26,6 +26,7 @@ import uvloop import aiohttp from aiohttp import web +import zuul.model import zuul.rpcclient from zuul.web.handler import StaticHandler @@ -235,13 +236,16 @@ class ZuulWeb(object): gear_server, gear_port, ssl_key=None, ssl_cert=None, ssl_ca=None, static_cache_expiry=3600, - connections=None): + connections=None, + info=None): + self.start_time = time.time() self.listen_address = listen_address self.listen_port = listen_port self.event_loop = None self.term = None self.server = None self.static_cache_expiry = static_cache_expiry + self.info = info # instanciate handlers self.rpc = zuul.rpcclient.RPCClient(gear_server, gear_port, ssl_key, ssl_cert, ssl_ca) @@ -250,12 +254,37 @@ class ZuulWeb(object): self._plugin_routes = [] # type: List[zuul.web.handler.BaseWebHandler] connections = connections or [] for connection in connections: - self._plugin_routes.extend(connection.getWebHandlers(self)) + self._plugin_routes.extend( + connection.getWebHandlers(self, self.info)) async def _handleWebsocket(self, request): return await self.log_streaming_handler.processRequest( request) + async def _handleRootInfo(self, request): + info = self.info.copy() + info.endpoint = str(request.url.parent) + return self._handleInfo(info) + + def _handleTenantInfo(self, request): + info = self.info.copy() + info.tenant = request.match_info["tenant"] + # yarl.URL.parent on a root url returns the root url, so this is + # both safe and accurate for white-labeled tenants like OpenStack, + # zuul-web running on / and zuul-web running on a sub-url like + # softwarefactory-project.io + info.endpoint = str(request.url.parent.parent.parent) + return self._handleInfo(info) + + def _handleInfo(self, info): + resp = web.json_response({'info': info.toDict()}, status=200) + resp.headers['Access-Control-Allow-Origin'] = '*' + if self.static_cache_expiry: + resp.headers['Cache-Control'] = "public, max-age=%d" % \ + self.static_cache_expiry + resp.last_modified = self.start_time + return resp + async def _handleTenantsRequest(self, request): return await self.gearman_handler.processRequest(request, 'tenant_list') @@ -286,6 +315,8 @@ class ZuulWeb(object): is run within a separate (non-main) thread. """ routes = [ + ('GET', '/info', self._handleRootInfo), + ('GET', '/{tenant}/info', self._handleTenantInfo), ('GET', '/tenants', self._handleTenantsRequest), ('GET', '/{tenant}/status', self._handleStatusRequest), ('GET', '/{tenant}/jobs', self._handleJobsRequest),