From 9b57c4a68e3e5c8d5c2e14cc6df1705b8fe84c71 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Tue, 27 Mar 2018 11:24:45 -0500 Subject: [PATCH] Reorganize REST API and dashboard urls The existing structure with the API and HTML interleaved makes it hard to have apache do html offloading for whitelabeled deploys (like openstack's) where the api and web are served from the same host. Similarly, the tenant selector in the url for the html content being bare and not prefixed by anything makes it harder to pull routing information javascript-side. Introduce an 'api' prefix to the REST API calls so that we can apply rewrite rules differently for things starting with /api than things that don't. Add the word 'tenant' before each tenant subpath. Also add a '/t/' to the url for the html, so that we have anchors for routing regexes but the urls don't get too long and unweildy. Finally, also add /key as a prefix to the key route for similar reasons. Change-Id: I0cbbf6f1958e91b5910738da9b4fb4c563d48dd4 --- doc/source/admin/installation.rst | 69 +++++++++++++++++++++++++++++-- tests/base.py | 5 ++- tests/unit/test_streaming.py | 2 +- tests/unit/test_web.py | 18 ++++---- tests/unit/test_web_urls.py | 26 ++++++++++-- tools/encrypt_secret.py | 2 +- web/dashboard.js | 2 +- web/status.js | 8 ++-- web/stream.js | 10 +++-- web/templates/builds.ejs | 2 +- web/templates/jobs.ejs | 2 +- web/templates/status.ejs | 2 +- web/templates/tenants.ejs | 6 +-- web/util.js | 25 ++++++++++- zuul/driver/sql/sqlconnection.py | 7 ++-- zuul/web/__init__.py | 22 +++++----- zuul/web/handler.py | 9 +++- 17 files changed, 165 insertions(+), 52 deletions(-) diff --git a/doc/source/admin/installation.rst b/doc/source/admin/installation.rst index 6ba6def8e2..3be964a2f1 100644 --- a/doc/source/admin/installation.rst +++ b/doc/source/admin/installation.rst @@ -86,8 +86,11 @@ White Label Static Offload Shift the duties of serving static files, such as HTML, Javascript, CSS or - images either to the Reverse Proxy server or to a completely separate - location such as a Swift Object Store or a CDN-enabled static web server. + images to the Reverse Proxy server. + +Static External + Serve the static files from a completely separate location that does not + support programmatic rewrite rules such as a Swift Object Store. Sub-URL Serve a Zuul dashboard from a location below the root URL as part of @@ -107,4 +110,64 @@ Using Apache as the Reverse Proxy requires the ``mod_proxy``, ``mod_proxy_http`` and ``mod_proxy_wstunnel`` modules to be installed and enabled. Static Offload and White Label additionally require ``mod_rewrite``. -.. TODO(mordred): Fill in specifics for all three methods +All of the cases require a rewrite rule for the websocket streaming, so the +simplest reverse-proxy case is:: + + RewriteEngine on + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/(.*)$ http://localhost:9000/$1 [P] + + +Static Offload +-------------- + +To have the Reverse Proxy serve the static html/javscript assets instead of +proxying them to the REST layer, register the location where you unpacked +the web application as the document root and add a simple rewrite rule:: + + DocumentRoot /var/lib/html + + Require all granted + + RewriteEngine on + RewriteRule ^/t/.*/(.*)$ /$1 [L] + RewriteRule ^/api/tenant/(.*)/console-stream ws://localhost:9000/api/tenant/$1/console-stream [P] + RewriteRule ^/api/(.*)$ http://localhost:9000/api/$1 [P] + +White Labeled Tenant +-------------------- + +Running a white-labeled tenant is similar to the offload case, but adds a +rule to ensure connection webhooks don't try to get put into the tenant scope. + +.. note:: + + It's possible to do white-labelling without static offload, but it is more + complex with no benefit. + +Assuming the zuul tenant name is "example", the rewrite rules are:: + + DocumentRoot /var/lib/html + + Require all granted + + RewriteEngine on + RewriteRule ^/api/connection/(.*)$ http://localhost:9000/api/connection/$1 [P] + RewriteRule ^/api/console-stream ws://localhost:9000/api/tenant/example/console-stream [P] + RewriteRule ^/api/(.*)$ http://localhost:9000/api/tenant/example/$1 [P] + +Static External +--------------- + +.. note:: + + Hosting zuul dashboard on an external static location that does not support + dynamic url rewrite rules only works for white-labeled deployments. + +In order to serve the zuul dashboard code from an external static location, +``ZUUL_API_URL`` must be set at javascript build time by passing the +``--define`` flag to the ``npm build:dist`` command. + +.. code-block:: bash + + npm build:dist -- --define "ZUUL_API_URL='http://zuul-web.example.com'" diff --git a/tests/base.py b/tests/base.py index 123d98a149..fb0d15076b 100755 --- a/tests/base.py +++ b/tests/base.py @@ -1031,7 +1031,7 @@ class FakeGithubConnection(githubconnection.GithubConnection): if use_zuulweb: return requests.post( - 'http://127.0.0.1:%s/connection/%s/payload' + 'http://127.0.0.1:%s/api/connection/%s/payload' % (self.zuul_web_port, self.connection_name), json=data, headers=headers) else: @@ -1863,7 +1863,8 @@ class ZuulWebFixture(fixtures.Fixture): # Start the web server self.web = zuul.web.ZuulWeb( listen_address='127.0.0.1', listen_port=0, - gear_server='127.0.0.1', gear_port=self.gearman_server_port) + gear_server='127.0.0.1', gear_port=self.gearman_server_port, + info=zuul.model.WebInfo()) loop = asyncio.new_event_loop() loop.set_debug(True) ws_thread = threading.Thread(target=self.web.run, args=(loop,)) diff --git a/tests/unit/test_streaming.py b/tests/unit/test_streaming.py index e17fc52443..bba77e6761 100644 --- a/tests/unit/test_streaming.py +++ b/tests/unit/test_streaming.py @@ -170,7 +170,7 @@ class TestStreaming(tests.base.AnsibleZuulTestCase): def runWSClient(self, build_uuid, event): async def client(loop, build_uuid, event): - uri = 'http://[::1]:9000/tenant-one/console-stream' + uri = 'http://[::1]:9000/api/tenant/tenant-one/console-stream' try: session = aiohttp.ClientSession(loop=loop) async with session.ws_connect(uri) as ws: diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index 13924c4e1a..b6d0035634 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -114,7 +114,7 @@ class TestWeb(BaseTestWeb): self.executor_server.release('project-merge') self.waitUntilSettled() - resp = self.get_url("tenant-one/status") + resp = self.get_url("api/tenant/tenant-one/status") self.assertIn('Content-Length', resp.headers) self.assertIn('Content-Type', resp.headers) self.assertEqual( @@ -211,7 +211,7 @@ class TestWeb(BaseTestWeb): self.executor_server.release('project-merge') self.waitUntilSettled() - resp = self.get_url("tenants") + resp = self.get_url("api/tenants") self.assertIn('Content-Length', resp.headers) self.assertIn('Content-Type', resp.headers) self.assertEqual( @@ -230,7 +230,7 @@ class TestWeb(BaseTestWeb): self.executor_server.release() self.waitUntilSettled() - data = self.get_url("tenants").json() + data = self.get_url("api/tenants").json() self.assertEqual('tenant-one', data[0]['name']) self.assertEqual(3, data[0]['projects']) self.assertEqual(0, data[0]['queue']) @@ -242,12 +242,12 @@ class TestWeb(BaseTestWeb): def test_web_find_change(self): # can we filter by change id - data = self.get_url("tenant-one/status/change/1,1").json() + data = self.get_url("api/tenant/tenant-one/status/change/1,1").json() self.assertEqual(1, len(data), data) self.assertEqual("org/project", data[0]['project']) - data = self.get_url("tenant-one/status/change/2,1").json() + data = self.get_url("api/tenant/tenant-one/status/change/2,1").json() self.assertEqual(1, len(data), data) self.assertEqual("org/project1", data[0]['project'], data) @@ -256,11 +256,11 @@ class TestWeb(BaseTestWeb): with open(os.path.join(FIXTURE_DIR, 'public.pem'), 'rb') as f: public_pem = f.read() - resp = self.get_url("tenant-one/org/project.pub") + resp = self.get_url("api/tenant/tenant-one/key/org/project.pub") self.assertEqual(resp.content, public_pem) def test_web_404_on_unknown_tenant(self): - resp = self.get_url("non-tenant/status") + resp = self.get_url("api/tenant/non-tenant/status") self.assertEqual(404, resp.status_code) @@ -275,7 +275,7 @@ class TestInfo(BaseTestWeb): self.stats_prefix = statsd_config.get('prefix') def test_info(self): - info = self.get_url("info").json() + info = self.get_url("api/info").json() self.assertEqual( info, { "info": { @@ -293,7 +293,7 @@ class TestInfo(BaseTestWeb): }) def test_tenant_info(self): - info = self.get_url("tenant-one/info").json() + info = self.get_url("api/tenant/tenant-one/info").json() self.assertEqual( info, { "info": { diff --git a/tests/unit/test_web_urls.py b/tests/unit/test_web_urls.py index 3e7f69e206..8e372fb79e 100644 --- a/tests/unit/test_web_urls.py +++ b/tests/unit/test_web_urls.py @@ -14,6 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. +import json import urllib from bs4 import BeautifulSoup @@ -66,7 +67,7 @@ class TestDirect(TestWebURLs, ZuulTestCase): self.port = self.web.port def test_status_page(self): - self._crawl('/tenant-one/status.html') + self._crawl('/t/tenant-one/status.html') class TestWhiteLabel(TestWebURLs, ZuulTestCase): @@ -75,8 +76,7 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase): def setUp(self): super(TestWhiteLabel, self).setUp() rules = [ - ('^/(.*)$', 'http://localhost:{}/tenant-one/\\1'.format( - self.web.port)), + ('^/(.*)$', 'http://localhost:{}/\\1'.format(self.web.port)), ] self.proxy = self.useFixture(WebProxyFixture(rules)) self.port = self.proxy.port @@ -85,6 +85,24 @@ class TestWhiteLabel(TestWebURLs, ZuulTestCase): self._crawl('/status.html') +class TestWhiteLabelAPI(TestWebURLs, ZuulTestCase): + # Test a zuul-web behind a whitelabel proxy (i.e., what + # zuul.openstack.org does). + def setUp(self): + super(TestWhiteLabelAPI, self).setUp() + rules = [ + ('^/api/(.*)$', + 'http://localhost:{}/api/tenant/tenant-one/\\1'.format( + self.web.port)), + ] + self.proxy = self.useFixture(WebProxyFixture(rules)) + self.port = self.proxy.port + + def test_info(self): + info = json.loads(self._get(self.port, '/api/info').decode('utf-8')) + self.assertEqual('tenant-one', info['info']['tenant']) + + class TestSuburl(TestWebURLs, ZuulTestCase): # Test a zuul-web mounted on a suburl (i.e., what software factory # does). @@ -98,4 +116,4 @@ class TestSuburl(TestWebURLs, ZuulTestCase): self.port = self.proxy.port def test_status_page(self): - self._crawl('/zuul3/tenant-one/status.html') + self._crawl('/zuul3/t/tenant-one/status.html') diff --git a/tools/encrypt_secret.py b/tools/encrypt_secret.py index 45ad68ca6e..f086402d9d 100755 --- a/tools/encrypt_secret.py +++ b/tools/encrypt_secret.py @@ -70,7 +70,7 @@ def main(): "unencrypted connection. Your secret may get " "compromised.\n") - req = Request("%s/%s.pub" % (args.url.rstrip('/'), args.project)) + req = Request("%s/key/%s.pub" % (args.url.rstrip('/'), args.project)) pubkey = urlopen(req) if args.infile: diff --git a/web/dashboard.js b/web/dashboard.js index 77884e5b50..409d2bdf26 100644 --- a/web/dashboard.js +++ b/web/dashboard.js @@ -29,7 +29,7 @@ angular.module('zuulTenants', []).controller( 'mainController', function ($scope, $http, $location) { $scope.tenants = undefined $scope.tenants_fetch = function () { - $http.get(getSourceUrl('tenants', $location)) + $http.get(getSourceUrl('api/tenants', $location)) .then(function success (result) { $scope.tenants = result.data }) diff --git a/web/status.js b/web/status.js index 81658e624e..56a698b303 100644 --- a/web/status.js +++ b/web/status.js @@ -36,7 +36,7 @@ import { getSourceUrl } from './util' /** * @return The $.zuul instance */ -function zuulStart ($) { +function zuulStart ($, $location) { // Start the zuul app (expects default dom) let $container, $indicator @@ -60,7 +60,7 @@ function zuulStart ($) { params['source_data'] = DemoStatusTree } } else { - params['source'] = getSourceUrl('status') + params['source'] = getSourceUrl('status', $location) } let zuul = $.zuul(params) @@ -120,7 +120,7 @@ if (module.hot) { } angular.module('zuulStatus', []).controller( - 'mainController', function ($scope, $http) { - zuulStart(jQuery) + 'mainController', function ($scope, $http, $location) { + zuulStart(jQuery, $location) } ) diff --git a/web/stream.js b/web/stream.js index 824948da11..aade1594b6 100644 --- a/web/stream.js +++ b/web/stream.js @@ -24,6 +24,8 @@ import angular from 'angular' import './styles/stream.css' +import { getSourceUrl } from './util' + function escapeLog (text) { const pattern = /[<>&"']/g @@ -32,7 +34,7 @@ function escapeLog (text) { }) } -function zuulStartStream () { +function zuulStartStream ($location) { let pageUpdateInMS = 250 let receiveBuffer = '' @@ -71,7 +73,7 @@ function zuulStartStream () { } else { protocol = 'ws://' } - let path = url['pathname'].replace(/stream.html.*$/g, '') + 'console-stream' + let path = getSourceUrl('console-stream', $location) params['websocket_url'] = protocol + url['host'] + path } let ws = new WebSocket(params['websocket_url']) @@ -93,7 +95,7 @@ function zuulStartStream () { } angular.module('zuulStream', []).controller( - 'mainController', function ($scope, $http) { - window.onload = zuulStartStream() + 'mainController', function ($scope, $http, $location) { + window.onload = zuulStartStream($location) } ) diff --git a/web/templates/builds.ejs b/web/templates/builds.ejs index 25461ac781..a093ccc3e2 100644 --- a/web/templates/builds.ejs +++ b/web/templates/builds.ejs @@ -23,7 +23,7 @@ under the License.