From 2fb12bb33938d2e0317a5c947286423310c3385d Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 2 Mar 2018 19:37:28 -0500 Subject: [PATCH] Update zuul-web install for javascript stack With the update to the use of yarn/webpack for the zuul-web javascript stack, there is now a pre-built tarball of the web assets. Update the code to use it. An alternative to this approach would be to install nodejs and yarn on the machine and just do a pip install . in the zuul source dir which would also work ... but since there is an option that doesn't require the need to run the build tools, why not use it? Because spec/acceptance/fixtures/default.pp is hard-coded to the zuulv3 install path, we can remove the now unnecessary checks from it. Move the status backups to /var/lib/zuul/backup, instead of www/backup as the www directory no longer exists. Also remove it as the documentroot. Depends-On: https://review.openstack.org/547790 Change-Id: Ib88b48ddea26ef5d8d0d152774112844b180f77e --- manifests/init.pp | 270 ++++++++++++++++++---------------- manifests/web.pp | 133 ++++++++--------- spec/acceptance/basic_spec.rb | 27 +--- templates/zuul.vhost.erb | 36 +---- templates/zuulv3.conf.erb | 1 + 5 files changed, 216 insertions(+), 251 deletions(-) diff --git a/manifests/init.pp b/manifests/init.pp index 01a3c9b..8881fe4 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -114,8 +114,6 @@ class zuul ( $packages = [ 'libffi-dev', 'libssl-dev', - 'python-paste', - 'python-webob', ] package { $packages: @@ -135,37 +133,39 @@ class zuul ( require => Class['pip'], } - # needed by python-keystoneclient, has system bindings - # Zuul and Nodepool both need it, so make it conditional - if ! defined(Package['python-lxml']) { - package { 'python-lxml': - ensure => present, + if ! $zuulv3 { + # needed by python-keystoneclient, has system bindings + # Zuul and Nodepool both need it, so make it conditional + if ! defined(Package['python-lxml']) { + package { 'python-lxml': + ensure => present, + } } - } - # A lot of things need yaml, be conservative requiring this package to avoid - # conflicts with other modules. - if ! defined(Package['python-yaml']) { - package { 'python-yaml': - ensure => present, + # A lot of things need yaml, be conservative requiring this package to avoid + # conflicts with other modules. + if ! defined(Package['python-yaml']) { + package { 'python-yaml': + ensure => present, + } } - } - if ! defined(Package['python-paramiko']) { - package { 'python-paramiko': - ensure => present, + if ! defined(Package['python-paramiko']) { + package { 'python-paramiko': + ensure => present, + } } - } - if ! defined(Package['python-daemon']) { - package { 'python-daemon': - ensure => present, + if ! defined(Package['python-daemon']) { + package { 'python-daemon': + ensure => present, + } } - } - if ! defined(Package['yui-compressor']) { - package { 'yui-compressor': - ensure => present, + if ! defined(Package['yui-compressor']) { + package { 'yui-compressor': + ensure => present, + } } } @@ -199,17 +199,19 @@ class zuul ( Package['build-essential'], Package['libffi-dev'], Package['libssl-dev'], - Package['python-daemon'], - Package['python-lxml'], - Package['python-paramiko'], - Package['python-paste'], - Package['python-webob'], - Package['python-yaml'], Package['yappi'], - Package['yui-compressor'], ], } + if ! $zuulv3 { + Exec['install_zuul'] -> Package['python-daemon'] + Exec['install_zuul'] -> Package['python-daemon'] + Exec['install_zuul'] -> Package['python-lxml'] + Exec['install_zuul'] -> Package['python-paramiko'] + Exec['install_zuul'] -> Package['python-yaml'] + Exec['install_zuul'] -> Package['yui-compressor'] + } + file { '/etc/zuul': ensure => directory, group => 'zuul', @@ -365,117 +367,135 @@ class zuul ( content => $zuul_ssh_private_key, } - file { '/var/lib/zuul/www': - ensure => directory, - require => File['/var/lib/zuul'], - } + if $zuulv3 { + package { 'libjs-jquery': + ensure => absent, + } + $v2_web_dirs = [ + '/opt/twitter-bootstrap', + '/opt/jquery-visibility', + '/var/lib/zuul/www', + '/opt/graphitejs', + ] - file { '/var/lib/zuul/www/lib': - ensure => directory, - require => File['/var/lib/zuul/www'], - } + file { $v2_web_dirs: + ensure => absent, + } - package { 'libjs-jquery': - ensure => present, - } + } else { - file { '/var/lib/zuul/www/jquery.min.js': - ensure => absent - } + file { '/var/lib/zuul/www': + ensure => directory, + require => File['/var/lib/zuul'], + } - file { '/var/lib/zuul/www/lib/jquery.min.js': - ensure => link, - target => '/usr/share/javascript/jquery/jquery.min.js', - require => [File['/var/lib/zuul/www/lib'], - Package['libjs-jquery']], - } + file { '/var/lib/zuul/www/lib': + ensure => directory, + require => File['/var/lib/zuul/www'], + } - vcsrepo { '/opt/twitter-bootstrap': - ensure => latest, - provider => git, - revision => 'v3.1.1', - source => 'https://github.com/twbs/bootstrap.git', - } + package { 'libjs-jquery': + ensure => present, + } - file { '/var/lib/zuul/www/bootstrap': - ensure => absent - } + file { '/var/lib/zuul/www/jquery.min.js': + ensure => absent + } - file { '/var/lib/zuul/www/lib/bootstrap': - ensure => link, - target => '/opt/twitter-bootstrap/dist', - require => [File['/var/lib/zuul/www/lib'], - Package['libjs-jquery'], - Vcsrepo['/opt/twitter-bootstrap']], - } + file { '/var/lib/zuul/www/lib/jquery.min.js': + ensure => link, + target => '/usr/share/javascript/jquery/jquery.min.js', + require => [File['/var/lib/zuul/www/lib'], + Package['libjs-jquery']], + } - vcsrepo { '/opt/jquery-visibility': - ensure => latest, - provider => git, - revision => 'master', - source => 'https://github.com/mathiasbynens/jquery-visibility.git', - } + vcsrepo { '/opt/twitter-bootstrap': + ensure => latest, + provider => git, + revision => 'v3.1.1', + source => 'https://github.com/twbs/bootstrap.git', + } - file { '/var/lib/zuul/www/jquery-visibility.min.js': - ensure => absent - } + file { '/var/lib/zuul/www/bootstrap': + ensure => absent + } - exec { 'install-jquery-visibility': - command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', - path => 'bin:/usr/bin', - refreshonly => true, - subscribe => Vcsrepo['/opt/jquery-visibility'], - require => [File['/var/lib/zuul/www/lib'], - Package['yui-compressor'], - Vcsrepo['/opt/jquery-visibility']], - } + file { '/var/lib/zuul/www/lib/bootstrap': + ensure => link, + target => '/opt/twitter-bootstrap/dist', + require => [File['/var/lib/zuul/www/lib'], + Package['libjs-jquery'], + Vcsrepo['/opt/twitter-bootstrap']], + } - vcsrepo { '/opt/graphitejs': - ensure => latest, - provider => git, - revision => 'master', - source => 'https://github.com/prestontimmons/graphitejs.git', - } + vcsrepo { '/opt/jquery-visibility': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://github.com/mathiasbynens/jquery-visibility.git', + } - file { '/var/lib/zuul/www/jquery.graphite.js': - ensure => absent - } + file { '/var/lib/zuul/www/jquery-visibility.min.js': + ensure => absent + } - file { '/var/lib/zuul/www/lib/jquery.graphite.js': - ensure => link, - target => '/opt/graphitejs/jquery.graphite.js', - require => [File['/var/lib/zuul/www/lib'], - Vcsrepo['/opt/graphitejs']], - } + exec { 'install-jquery-visibility': + command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', + path => 'bin:/usr/bin', + refreshonly => true, + subscribe => Vcsrepo['/opt/jquery-visibility'], + require => [File['/var/lib/zuul/www/lib'], + Package['yui-compressor'], + Vcsrepo['/opt/jquery-visibility']], + } - file { '/var/lib/zuul/www/index.html': - ensure => link, - target => '/opt/zuul/etc/status/public_html/index.html', - require => File['/var/lib/zuul/www'], - } + vcsrepo { '/opt/graphitejs': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://github.com/prestontimmons/graphitejs.git', + } - file { '/var/lib/zuul/www/styles': - ensure => link, - target => '/opt/zuul/etc/status/public_html/styles', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/jquery.graphite.js': + ensure => absent + } - file { '/var/lib/zuul/www/zuul.app.js': - ensure => link, - target => '/opt/zuul/etc/status/public_html/zuul.app.js', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/lib/jquery.graphite.js': + ensure => link, + target => '/opt/graphitejs/jquery.graphite.js', + require => [File['/var/lib/zuul/www/lib'], + Vcsrepo['/opt/graphitejs']], + } - file { '/var/lib/zuul/www/jquery.zuul.js': - ensure => link, - target => '/opt/zuul/etc/status/public_html/jquery.zuul.js', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/index.html': + ensure => link, + target => '/opt/zuul/etc/status/public_html/index.html', + require => File['/var/lib/zuul/www'], + } - file { '/var/lib/zuul/www/images': - ensure => link, - target => '/opt/zuul/etc/status/public_html/images', - require => File['/var/lib/zuul/www'], + file { '/var/lib/zuul/www/styles': + ensure => link, + target => '/opt/zuul/etc/status/public_html/styles', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/zuul.app.js': + ensure => link, + target => '/opt/zuul/etc/status/public_html/zuul.app.js', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/jquery.zuul.js': + ensure => link, + target => '/opt/zuul/etc/status/public_html/jquery.zuul.js', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/images': + ensure => link, + target => '/opt/zuul/etc/status/public_html/images', + require => File['/var/lib/zuul/www'], + } } if $zuulv3 { diff --git a/manifests/web.pp b/manifests/web.pp index edfb550..db83dd2 100644 --- a/manifests/web.pp +++ b/manifests/web.pp @@ -77,9 +77,9 @@ class zuul::web ( } } - file { '/var/lib/zuul/www/backup': + file { '/var/lib/zuul/backup': ensure => directory, - require => File['/var/lib/zuul/www'], + require => File['/var/lib/zuul'], } if $enable_status_backups { @@ -97,10 +97,10 @@ class zuul::web ( } cron { 'zuul_scheduler_status_backup': user => 'root', - command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/www/backup/status_$(date +\\%s).json 2>/dev/null", + command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/backup/status_$(date +\\%s).json 2>/dev/null", require => [Package['curl'], User['zuul'], - File['/var/lib/zuul/www/backup']], + File['/var/lib/zuul/backup']], } # Rotate backups and keep no more than 120 files -- or 2 hours worth of # backup if Zuul has 100% uptime. @@ -111,84 +111,85 @@ class zuul::web ( cron { 'zuul_scheduler_status_prune': user => 'root', minute => '0', - command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/www/backup/* |sed -e "1,120d" |xargs rm -f', + command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/backup/* |sed -e "1,120d" |xargs rm -f', require => Cron['zuul_scheduler_status_backup'], } } file { '/var/lib/zuul/www/static': + ensure => absent, + } + + $zuul_web_root = '/opt/zuul-web' + $zuul_web_content_root = '/opt/zuul-web/content' + $zuul_web_src_root = '/opt/zuul-web/source' + $zuul_web_filename = 'zuul-content-latest.tar.gz' + $zuul_web_url = "http://tarballs.openstack.org/zuul/${zuul_web_filename}" + + file { $zuul_web_root: ensure => directory, - require => File['/var/lib/zuul/www'], + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => User['zuul'], } - file { '/var/lib/zuul/www/static/js': + file { $zuul_web_content_root: ensure => directory, - require => File['/var/lib/zuul/www/static'], + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => [ + File[$zuul_web_root], + User['zuul'], + ] } - file { '/var/lib/zuul/www/static/js/jquery.min.js': - ensure => link, - target => '/usr/share/javascript/jquery/jquery.min.js', - require => [File['/var/lib/zuul/www/static/js'], - Package['libjs-jquery']], + file { $zuul_web_src_root: + ensure => directory, + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => [ + File[$zuul_web_root], + User['zuul'], + ] } - file { '/var/lib/zuul/www/static/bootstrap': - ensure => link, - target => '/opt/twitter-bootstrap/dist', - require => [File['/var/lib/zuul/www/static'], - Package['libjs-jquery'], - Vcsrepo['/opt/twitter-bootstrap']], - } - - exec { 'install-jquery-visibility-zuul-web': - command => 'yui-compressor -o /var/lib/zuul/www/static/js/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', - path => 'bin:/usr/bin', - refreshonly => true, - subscribe => Vcsrepo['/opt/jquery-visibility'], - require => [File['/var/lib/zuul/www/static/js'], - Package['yui-compressor'], - Vcsrepo['/opt/jquery-visibility']], - } - - file { '/var/lib/zuul/www/static/js/jquery.graphite.js': - ensure => link, - target => '/opt/graphitejs/jquery.graphite.js', - require => [File['/var/lib/zuul/www/static/js'], - Vcsrepo['/opt/graphitejs']], - } - - # Download angular - # NOTE: This is using a hardcoded URL because soon this will shift to being - # based on a more javascript-native toolchain. - exec { 'get-angular-zuul-web': - command => 'curl https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js -o /var/lib/zuul/www/static/js/angular.min.js', + # Download the latest zuul-web + exec { 'get-zuul-web': + command => "curl ${zuul_web_url} -z ./${zuul_web_filename} -o ${zuul_web_filename}", path => '/bin:/usr/bin', - require => [Package[curl], - File['/var/lib/zuul/www/static/js']], - onlyif => "curl -I https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js | grep '200 OK'", - creates => '/var/lib/zuul/www/static/js/angular.min.js', + cwd => $zuul_web_root, + require => [ + File[$zuul_web_root], + File[$zuul_web_content_root], + File[$zuul_web_src_root], + ], + onlyif => "curl -I ${zuul_web_url} -z ./${zuul_web_filename} | grep '200 OK'", } - # For now, symlink in the static parts of zuul-web which are not - # tenant-scoped since they share a URL space with the external - # dependencies. - file { '/var/lib/zuul/www/static/javascripts': - ensure => link, - target => '/opt/zuul/zuul/web/static/javascripts', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], + # Unpack storyboard-zuul_web + exec { 'unpack-zuul-web': + command => "rm -rf ${zuul_web_src_root}/* && tar -C ${zuul_web_src_root} -xzf ./${zuul_web_filename}", + path => '/bin:/usr/bin', + refreshonly => true, + cwd => $zuul_web_root, + require => Exec['get-zuul-web'], + subscribe => Exec['get-zuul-web'], } - file { '/var/lib/zuul/www/static/images': - ensure => link, - target => '/opt/zuul/zuul/web/static/images', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], - } - file { '/var/lib/zuul/www/static/styles': - ensure => link, - target => '/opt/zuul/zuul/web/static/styles', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], + + # Sync zuul-web to the directory we serve it from. This is so that we don't + # have files go missing - but also so that we can clean up old verisons of + # files. The assets built by webpack have hashes in the filenames to help + # with caching. + exec { 'sync-zuul-web': + command => "rsync -rl --delete-delay . ${zuul_web_content_root}/", + path => '/bin:/usr/bin', + refreshonly => true, + cwd => $zuul_web_src_root, + require => Exec['unpack-zuul-web'], + subscribe => Exec['unpack-zuul-web'], } + } diff --git a/spec/acceptance/basic_spec.rb b/spec/acceptance/basic_spec.rb index a83c5ac..6813f80 100644 --- a/spec/acceptance/basic_spec.rb +++ b/spec/acceptance/basic_spec.rb @@ -1,3 +1,4 @@ +# coding: utf-8 require 'puppet-openstack_infra_spec_helper/spec_helper_acceptance' # https://blog.lorentzca.me/add-custom-matcher-of-serverspec/ @@ -63,13 +64,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family]) packages = [ package('git'), package('build-essential'), - package('python-lxml'), - package('python-yaml'), - package('python-paramiko'), - package('python-daemon'), - package('yui-compressor'), - package('python-paste'), - package('python-webob') ] packages << package('apache2') if ['ubuntu', 'debian'].include?(os[:family]) packages << package('httpd') if ['centos', 'redhat'].include?(os[:family]) @@ -136,25 +130,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family]) end end - describe 'public_html symlinks' do - symlinkies = { - file('/var/lib/zuul/www/images') => '/opt/zuul/etc/status/public_html/images', - file('/var/lib/zuul/www/index.html') => '/opt/zuul/etc/status/public_html/index.html', - file('/var/lib/zuul/www/jquery.zuul.js') => '/opt/zuul/etc/status/public_html/jquery.zuul.js', - file('/var/lib/zuul/www/styles') => '/opt/zuul/etc/status/public_html/styles', - file('/var/lib/zuul/www/zuul.app.js') => '/opt/zuul/etc/status/public_html/zuul.app.js', - file('/var/lib/zuul/www/lib/jquery.graphite.js') => '/opt/graphitejs/jquery.graphite.js', - file('/var/lib/zuul/www/lib/bootstrap') => '/opt/twitter-bootstrap/dist', - } - - symlinkies.each do |link, destination| - describe link do - it { should be_symlink } - it { should be_linked_to destination } - end - end - end - describe file('/var/lib/zuul/ssh/id_rsa') do it { should be_file } it { should contain('-----BEGIN RSA PRIVATE KEY-----') } diff --git a/templates/zuul.vhost.erb b/templates/zuul.vhost.erb index 924d513..fee230a 100644 --- a/templates/zuul.vhost.erb +++ b/templates/zuul.vhost.erb @@ -1,17 +1,7 @@ ServerName <%= @vhost_name %> ServerAdmin <%= @serveradmin %> - DocumentRoot /var/lib/zuul/www - - = 2.4> - Require all granted - - - Order deny,allow - Allow from all - - = 2.4> Require all granted @@ -42,19 +32,8 @@ <% end %> <% if @zuulv3 %> - RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P] - RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P] - RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P] - RewriteRule ^/status <%= @zuul_web_full_url %>/status [P] - RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P] - RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P] - RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P] - RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P] - RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P] - RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] - RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] - RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P] + RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] <% end %> @@ -138,19 +117,8 @@ <% end %> <% if @zuulv3 %> - RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P] - RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P] - RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P] - RewriteRule ^/status <%= @zuul_web_full_url %>/status [P] - RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P] - RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P] - RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P] - RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P] - RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P] - RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] - RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] - RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P] + RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] <% end %> diff --git a/templates/zuulv3.conf.erb b/templates/zuulv3.conf.erb index 187e7e6..180c8c3 100644 --- a/templates/zuulv3.conf.erb +++ b/templates/zuulv3.conf.erb @@ -74,6 +74,7 @@ disk_limit_per_job=<%= @disk_limit_per_job %> [web] log_config=/etc/zuul/web-logging.conf +static_path=/opt/zuul-web/content <% if @web_listen_address != "" -%> listen_address=<%= @web_listen_address %> <% end -%>