Browse Source

Reorganize REST API and dashboard urls

The existing structure with the API and HTML interleaved makes it hard
to have apache do html offloading for whitelabeled deploys (like
openstack's) where the api and web are served from the same host.
Similarly, the tenant selector in the url for the html content being bare
and not prefixed by anything makes it harder to pull routing information
javascript-side.

Introduce an 'api' prefix to the REST API calls so that we can apply
rewrite rules differently for things starting with /api than things that
don't. Add the word 'tenant' before each tenant subpath.

Also add a '/t/' to the url for the html, so that we have anchors for
routing regexes but the urls don't get too long and unweildy.

Finally, also add /key as a prefix to the key route for similar reasons.

Change-Id: I0cbbf6f1958e91b5910738da9b4fb4c563d48dd4
changes/67/556967/8
Monty Taylor 3 years ago
parent
commit
9b57c4a68e
No known key found for this signature in database GPG Key ID: 7BAE94BC7141A594
17 changed files with 165 additions and 52 deletions
  1. +66
    -3
      doc/source/admin/installation.rst
  2. +3
    -2
      tests/base.py
  3. +1
    -1
      tests/unit/test_streaming.py
  4. +9
    -9
      tests/unit/test_web.py
  5. +22
    -4
      tests/unit/test_web_urls.py
  6. +1
    -1
      tools/encrypt_secret.py
  7. +1
    -1
      web/dashboard.js
  8. +4
    -4
      web/status.js
  9. +6
    -4
      web/stream.js
  10. +1
    -1
      web/templates/builds.ejs
  11. +1
    -1
      web/templates/jobs.ejs
  12. +1
    -1
      web/templates/status.ejs
  13. +3
    -3
      web/templates/tenants.ejs
  14. +23
    -2
      web/util.js
  15. +3
    -4
      zuul/driver/sql/sqlconnection.py
  16. +12
    -10
      zuul/web/__init__.py
  17. +8
    -1
      zuul/web/handler.py

+ 66
- 3
doc/source/admin/installation.rst View File

@ -86,8 +86,11 @@ White Label
Static Offload
Shift the duties of serving static files, such as HTML, Javascript, CSS or
images either to the Reverse Proxy server or to a completely separate
location such as a Swift Object Store or a CDN-enabled static web server.
images to the Reverse Proxy server.
Static External
Serve the static files from a completely separate location that does not
support programmatic rewrite rules such as a Swift Object Store.
Sub-URL
Serve a Zuul dashboard from a location below the root URL as part of
@ -107,4 +110,64 @@ Using Apache as the Reverse Proxy requires the ``mod_proxy``,
``mod_proxy_http`` and ``mod_proxy_wstunnel`` modules to be installed and
enabled. Static Offload and White Label additionally require ``mod_rewrite``.
.. TODO(mordred): Fill in specifics for all three methods
All of the cases require a rewrite rule for the websocket streaming, so the
simplest reverse-proxy case is::
RewriteEngine on
RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P]
RewriteRule ^/(.*)$ http://localhost:9000/$1 [P]
Static Offload
--------------
To have the Reverse Proxy serve the static html/javscript assets instead of
proxying them to the REST layer, register the location where you unpacked
the web application as the document root and add a simple rewrite rule::
DocumentRoot /var/lib/html
<Directory /var/lib/html>
Require all granted
</Directory>
RewriteEngine on
RewriteRule ^/t/.*/(.*)$ /$1 [L]
RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P]
RewriteRule ^/api/(.*)$ http://localhost:9000/api/$1 [P]
White Labeled Tenant
--------------------
Running a white-labeled tenant is similar to the offload case, but adds a
rule to ensure connection webhooks don't try to get put into the tenant scope.
.. note::
It's possible to do white-labelling without static offload, but it is more
complex with no benefit.
Assuming the zuul tenant name is "example", the rewrite rules are::
DocumentRoot /var/lib/html
<Directory /var/lib/html>
Require all granted
</Directory>
RewriteEngine on
RewriteRule ^/api/connection/(.*)$ http://localhost:9000/api/connection/$1 [P]
RewriteRule ^/api/console-stream ws://localhost:9000/api/tenant/example/console-stream [P]
RewriteRule ^/api/(.*)$ http://localhost:9000/api/tenant/example/$1 [P]
Static External
---------------
.. note::
Hosting zuul dashboard on an external static location that does not support
dynamic url rewrite rules only works for white-labeled deployments.
In order to serve the zuul dashboard code from an external static location,
``ZUUL_API_URL`` must be set at javascript build time by passing the
``--define`` flag to the ``npm build:dist`` command.
.. code-block:: bash
npm build:dist -- --define "ZUUL_API_URL='http://zuul-web.example.com'"

+ 3
- 2
tests/base.py View File

@ -1031,7 +1031,7 @@ class FakeGithubConnection(githubconnection.GithubConnection):
if use_zuulweb:
return requests.post(
'http://127.0.0.1:%s/connection/%s/payload'
'http://127.0.0.1:%s/api/connection/%s/payload'
% (self.zuul_web_port, self.connection_name),
json=data, headers=headers)
else:
@ -1863,7 +1863,8 @@ class ZuulWebFixture(fixtures.Fixture):
# Start the web server
self.web = zuul.web.ZuulWeb(
listen_address='127.0.0.1', listen_port=0,
gear_server='127.0.0.1', gear_port=self.gearman_server_port)
gear_server='127.0.0.1', gear_port=self.gearman_server_port,
info=zuul.model.WebInfo())
loop = asyncio.new_event_loop()
loop.set_debug(True)
ws_thread = threading.Thread(target=self.web.run, args=(loop,))


+ 1
- 1
tests/unit/test_streaming.py View File

@ -170,7 +170,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase):
def runWSClient(self, build_uuid, event):
async def client(loop, build_uuid, event):
uri = 'http://[::1]:9000/tenant-one/console-stream'
uri = 'http://[::1]:9000/api/tenant/tenant-one/console-stream'
try:
session = aiohttp.ClientSession(loop=loop)
async with session.ws_connect(uri) as ws:


+ 9
- 9
tests/unit/test_web.py View File

@ -114,7 +114,7 @@ class TestWeb(BaseTestWeb):
self.executor_server.release('project-merge')
self.waitUntilSettled()
resp = self.get_url("tenant-one/status")
resp = self.get_url("api/tenant/tenant-one/status")
self.assertIn('Content-Length', resp.headers)
self.assertIn('Content-Type', resp.headers)
self.assertEqual(
@ -211,7 +211,7 @@ class TestWeb(BaseTestWeb):
self.executor_server.release('project-merge')
self.waitUntilSettled()
resp = self.get_url("tenants")
resp = self.get_url("api/tenants")
self.assertIn('Content-Length', resp.headers)
self.assertIn('Content-Type', resp.headers)
self.assertEqual(
@ -230,7 +230,7 @@ class TestWeb(BaseTestWeb):
self.executor_server.release()
self.waitUntilSettled()
data = self.get_url("tenants").json()
data = self.get_url("api/tenants").json()
self.assertEqual('tenant-one', data[0]['name'])
self.assertEqual(3, data[0]['projects'])
self.assertEqual(0, data[0]['queue'])
@ -242,12 +242,12 @@ class TestWeb(BaseTestWeb):
def test_web_find_change(self):
# can we filter by change id
data = self.get_url("tenant-one/status/change/1,1").json()
data = self.get_url("api/tenant/tenant-one/status/change/1,1").json()
self.assertEqual(1, len(data), data)
self.assertEqual("org/project", data[0]['project'])
data = self.get_url("tenant-one/status/change/2,1").json()
data = self.get_url("api/tenant/tenant-one/status/change/2,1").json()
self.assertEqual(1, len(data), data)
self.assertEqual("org/project1", data[0]['project'], data)
@ -256,11 +256,11 @@ class TestWeb(BaseTestWeb):
with open(os.path.join(FIXTURE_DIR, 'public.pem'), 'rb') as f:
public_pem = f.read()
resp = self.get_url("tenant-one/org/project.pub")
resp = self.get_url("api/tenant/tenant-one/key/org/project.pub")
self.assertEqual(resp.content, public_pem)
def test_web_404_on_unknown_tenant(self):
resp = self.get_url("non-tenant/status")
resp = self.get_url("api/tenant/non-tenant/status")
self.assertEqual(404, resp.status_code)
@ -275,7 +275,7 @@ class TestInfo(BaseTestWeb):
self.stats_prefix = statsd_config.get('prefix')
def test_info(self):
info = self.get_url("info").json()
info = self.get_url("api/info").json()
self.assertEqual(
info, {
"info": {
@ -293,7 +293,7 @@ class TestInfo(BaseTestWeb):
})
def test_tenant_info(self):
info = self.get_url("tenant-one/info").json()
info = self.get_url("api/tenant/tenant-one/info").json()
self.assertEqual(
info, {
"info": {


+ 22
- 4
tests/unit/test_web_urls.py View File

@ -14,6 +14,7 @@
# License for the specific language governing permissions and limitations
# under the License.
import json
import urllib
from bs4 import BeautifulSoup
@ -66,7 +67,7 @@ class TestDirect(TestWebURLs, ZuulTestCase):
self.port = self.web.port
def test_status_page(self):
self._crawl('/tenant-one/status.html')
self._crawl('/t/tenant-one/status.html')
class TestWhiteLabel(TestWebURLs, ZuulTestCase):
@ -75,8 +76,7 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase):
def setUp(self):
super(TestWhiteLabel, self).setUp()
rules = [
('^/(.*)$', 'http://localhost:{}/tenant-one/\\1'.format(
self.web.port)),
('^/(.*)$', 'http://localhost:{}/\\1'.format(self.web.port)),
]
self.proxy = self.useFixture(WebProxyFixture(rules))
self.port = self.proxy.port
@ -85,6 +85,24 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase):
self._crawl('/status.html')
class TestWhiteLabelAPI(TestWebURLs, ZuulTestCase):
# Test a zuul-web behind a whitelabel proxy (i.e., what
# zuul.openstack.org does).
def setUp(self):
super(TestWhiteLabelAPI, self).setUp()
rules = [
('^/api/(.*)$',
'http://localhost:{}/api/tenant/tenant-one/\\1'.format(
self.web.port)),
]
self.proxy = self.useFixture(WebProxyFixture(rules))
self.port = self.proxy.port
def test_info(self):
info = json.loads(self._get(self.port, '/api/info').decode('utf-8'))
self.assertEqual('tenant-one', info['info']['tenant'])
class TestSuburl(TestWebURLs, ZuulTestCase):
# Test a zuul-web mounted on a suburl (i.e., what software factory
# does).
@ -98,4 +116,4 @@ class TestSuburl(TestWebURLs, ZuulTestCase):
self.port = self.proxy.port
def test_status_page(self):
self._crawl('/zuul3/tenant-one/status.html')
self._crawl('/zuul3/t/tenant-one/status.html')

+ 1
- 1
tools/encrypt_secret.py View File

@ -70,7 +70,7 @@ def main():
"unencrypted connection. Your secret may get "
"compromised.\n")
req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project))
req = Request("%s/key/%s.pub" % (args.url.rstrip('/'), args.project))
pubkey = urlopen(req)
if args.infile:


+ 1
- 1
web/dashboard.js View File

@ -29,7 +29,7 @@ angular.module('zuulTenants', []).controller(
'mainController', function ($scope, $http, $location) {
$scope.tenants = undefined
$scope.tenants_fetch = function () {
$http.get(getSourceUrl('tenants', $location))
$http.get(getSourceUrl('api/tenants', $location))
.then(function success (result) {
$scope.tenants = result.data
})


+ 4
- 4
web/status.js View File

@ -36,7 +36,7 @@ import { getSourceUrl } from './util'
/**
* @return The $.zuul instance
*/
function zuulStart ($) {
function zuulStart ($, $location) {
// Start the zuul app (expects default dom)
let $container, $indicator
@ -60,7 +60,7 @@ function zuulStart ($) {
params['source_data'] = DemoStatusTree
}
} else {
params['source'] = getSourceUrl('status')
params['source'] = getSourceUrl('status', $location)
}
let zuul = $.zuul(params)
@ -120,7 +120,7 @@ if (module.hot) {
}
angular.module('zuulStatus', []).controller(
'mainController', function ($scope, $http) {
zuulStart(jQuery)
'mainController', function ($scope, $http, $location) {
zuulStart(jQuery, $location)
}
)

+ 6
- 4
web/stream.js View File

@ -24,6 +24,8 @@
import angular from 'angular'
import './styles/stream.css'
import { getSourceUrl } from './util'
function escapeLog (text) {
const pattern = /[<>&"']/g
@ -32,7 +34,7 @@ function escapeLog (text) {
})
}
function zuulStartStream () {
function zuulStartStream ($location) {
let pageUpdateInMS = 250
let receiveBuffer = ''
@ -71,7 +73,7 @@ function zuulStartStream () {
} else {
protocol = 'ws://'
}
let path = url['pathname'].replace(/stream.html.*$/g, '') + 'console-stream'
let path = getSourceUrl('console-stream', $location)
params['websocket_url'] = protocol + url['host'] + path
}
let ws = new WebSocket(params['websocket_url'])
@ -93,7 +95,7 @@ function zuulStartStream () {
}
angular.module('zuulStream', []).controller(
'mainController', function ($scope, $http) {
window.onload = zuulStartStream()
'mainController', function ($scope, $http, $location) {
window.onload = zuulStartStream($location)
}
)

+ 1
- 1
web/templates/builds.ejs View File

@ -23,7 +23,7 @@ under the License.
<nav class="navbar navbar-default">
<div class="container-fluid">
<div class="navbar-header">
<a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
<a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
</div>
<ul class="nav navbar-nav">
<li><a href="status.html" target="_self">Status</a></li>


+ 1
- 1
web/templates/jobs.ejs View File

@ -23,7 +23,7 @@ under the License.
<nav class="navbar navbar-default">
<div class="container-fluid">
<div class="navbar-header">
<a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
<a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
</div>
<ul class="nav navbar-nav">
<li><a href="status.html" target="_self">Status</a></li>


+ 1
- 1
web/templates/status.ejs View File

@ -23,7 +23,7 @@ under the License.
<nav class="navbar navbar-default">
<div class="container-fluid">
<div class="navbar-header">
<a class="navbar-brand" href="../" target="_self">Zuul Dashboard</a>
<a class="navbar-brand" href="../../" target="_self">Zuul Dashboard</a>
</div>
<ul class="nav navbar-nav">
<li class="active"><a href="status.html" target="_self">Status</a></li>


+ 3
- 3
web/templates/tenants.ejs View File

@ -44,9 +44,9 @@ under the License.
<tbody>
<tr ng-repeat="tenant in tenants">
<td>{{ tenant.name }}</td>
<td><a href="{{ tenant.name }}/status.html">status</a></td>
<td><a href="{{ tenant.name }}/jobs.html">jobs</a></td>
<td><a href="{{ tenant.name }}/builds.html">builds</a></td>
<td><a href="t/{{ tenant.name }}/status.html">status</a></td>
<td><a href="t/{{ tenant.name }}/jobs.html">jobs</a></td>
<td><a href="t/{{ tenant.name }}/builds.html">builds</a></td>
<td>{{ tenant.projects }}</td>
<td>{{ tenant.queue }}</td>
</tr>


+ 23
- 2
web/util.js View File

@ -19,12 +19,33 @@
// @licend The above is the entire license notice
// for the JavaScript code in this page.
// TODO(mordred) This is a temporary hack until we're on @angular/router
function extractTenant (url) {
if (url.includes('/t/')) {
// This is a multi-tenant deploy, find the tenant
const tenantStart = url.lastIndexOf('/t/') + 3
const tenantEnd = url.indexOf('/', tenantStart)
return url.slice(tenantStart, tenantEnd)
} else {
return null
}
}
// TODO(mordred) This should be encapsulated in an Angular Service singleton
// that fetches the other things from the info endpoint.
export function getSourceUrl (filename, $location) {
if (typeof ZUUL_API_URL !== 'undefined') {
return ZUUL_API_URL + '/' + filename
return `${ZUUL_API_URL}/api/${filename}`
} else {
return filename
let tenant = extractTenant($location.url())
if (tenant) {
// Multi-tenant deploy. This is at t/a-tenant/x.html. api path is at
// api/tenant/a-tenant/x, so should be at ../../api/tenant/a-tenant/x
return `../../api/tenant/${tenant}/${filename}`
} else {
// Whilelabel deploy. This is at x.html. api path is at
// api/x, so should be at api/x
return `api/${filename}`
}
}
}

+ 3
- 4
zuul/driver/sql/sqlconnection.py View File

@ -26,7 +26,7 @@ import voluptuous
from zuul.connection import BaseConnection
from zuul.lib.config import get_default
from zuul.web.handler import BaseWebHandler, StaticHandler
from zuul.web.handler import BaseTenantWebHandler
BUILDSET_TABLE = 'zuul_buildset'
BUILD_TABLE = 'zuul_build'
@ -129,8 +129,7 @@ class SQLConnection(BaseConnection):
def getWebHandlers(self, zuul_web, info):
info.capabilities.job_history = True
return [
SqlWebHandler(self, zuul_web, 'GET', '/{tenant}/builds'),
StaticHandler(zuul_web, '/{tenant}/builds.html'),
SqlWebHandler(self, zuul_web, 'GET', 'builds'),
]
def validateWebConfig(self, config, connections):
@ -155,7 +154,7 @@ class SQLConnection(BaseConnection):
return True
class SqlWebHandler(BaseWebHandler):
class SqlWebHandler(BaseTenantWebHandler):
log = logging.getLogger("zuul.web.SqlHandler")
filters = ("project", "pipeline", "change", "branch", "patchset", "ref",
"result", "uuid", "job_name", "voting", "node_name", "newrev")


+ 12
- 10
zuul/web/__init__.py View File

@ -320,19 +320,21 @@ class ZuulWeb(object):
is run within a separate (non-main) thread.
"""
routes = [
('GET', '/info', self._handleRootInfo),
('GET', '/{tenant}/info', self._handleTenantInfo),
('GET', '/tenants', self._handleTenantsRequest),
('GET', '/{tenant}/status', self._handleStatusRequest),
('GET', '/{tenant}/jobs', self._handleJobsRequest),
('GET', '/{tenant}/status/change/{change}',
('GET', '/api/info', self._handleRootInfo),
('GET', '/api/tenants', self._handleTenantsRequest),
('GET', '/api/tenant/{tenant}/info', self._handleTenantInfo),
('GET', '/api/tenant/{tenant}/status', self._handleStatusRequest),
('GET', '/api/tenant/{tenant}/jobs', self._handleJobsRequest),
('GET', '/api/tenant/{tenant}/status/change/{change}',
self._handleStatusChangeRequest),
('GET', '/{tenant}/console-stream', self._handleWebsocket),
('GET', '/{tenant}/{project:.*}.pub', self._handleKeyRequest),
('GET', '/api/tenant/{tenant}/console-stream',
self._handleWebsocket),
('GET', '/api/tenant/{tenant}/key/{project:.*}.pub',
self._handleKeyRequest),
]
static_routes = [
StaticHandler(self, '/{tenant}/', 'status.html'),
StaticHandler(self, '/t/{tenant}/', 'status.html'),
StaticHandler(self, '/', 'tenants.html'),
]
@ -340,7 +342,7 @@ class ZuulWeb(object):
routes.append((route.method, route.path, route.handleRequest))
# Add fallthrough routes at the end for the static html/js files
routes.append(('GET', '/{tenant}/{path:.*}', self._handleStatic))
routes.append(('GET', '/t/{tenant}/{path:.*}', self._handleStatic))
routes.append(('GET', '/{path:.*}', self._handleStatic))
self.log.debug("ZuulWeb starting")


+ 8
- 1
zuul/web/handler.py View File

@ -31,6 +31,13 @@ class BaseWebHandler(object, metaclass=abc.ABCMeta):
"""Process a web request."""
class BaseTenantWebHandler(BaseWebHandler):
def __init__(self, connection, zuul_web, method, path):
super(BaseTenantWebHandler, self).__init__(
connection, zuul_web, method, '/api/tenant/{tenant}/' + path)
class BaseDriverWebHandler(BaseWebHandler):
def __init__(self, connection, zuul_web, method, path):
@ -38,7 +45,7 @@ class BaseDriverWebHandler(BaseWebHandler):
connection=connection, zuul_web=zuul_web, method=method, path=path)
if path.startswith('/'):
path = path[1:]
self.path = '/connection/{connection}/{path}'.format(
self.path = '/api/connection/{connection}/{path}'.format(
connection=self.connection.connection_name,
path=path)


Loading…
Cancel
Save