opendev/puppet-zuul
Code Issues Proposed changes

Update zuul-web install for javascript stack

Browse Source 
With the update to the use of yarn/webpack for the zuul-web javascript
stack, there is now a pre-built tarball of the web assets. Update the
code to use it.

An alternative to this approach would be to install nodejs and yarn on
the machine and just do a pip install . in the zuul source dir which
would also work ... but since there is an option that doesn't require
the need to run the build tools, why not use it?

Because spec/acceptance/fixtures/default.pp is hard-coded to the
zuulv3 install path, we can remove the now unnecessary checks from it.

Move the status backups to /var/lib/zuul/backup, instead of www/backup
as the www directory no longer exists.  Also remove it as the
documentroot.

Depends-On: https://review.openstack.org/547790
Change-Id: Ib88b48ddea26ef5d8d0d152774112844b180f77e
This commit is contained in:
Monty Taylor 2018-03-02 19:37:28 -05:00
parent 8f68109b2e
commit 2fb12bb339
No known key found for this signature in database
GPG Key ID: 7BAE94BC7141A594
5 changed files with 216 additions and 251 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

270
manifests/init.pp
View File

@ -114,8 +114,6 @@ class zuul (
$packages = [ $packages = [
'libffi-dev', 'libffi-dev',
'libssl-dev', 'libssl-dev',
'python-paste',
'python-webob',
] ]
package { $packages: package { $packages:
@ -135,37 +133,39 @@ class zuul (
require => Class['pip'], require => Class['pip'],
} }
# needed by python-keystoneclient, has system bindings if ! $zuulv3 {
# Zuul and Nodepool both need it, so make it conditional # needed by python-keystoneclient, has system bindings
if ! defined(Package['python-lxml']) { # Zuul and Nodepool both need it, so make it conditional
package { 'python-lxml': if ! defined(Package['python-lxml']) {
ensure => present, package { 'python-lxml':
ensure => present,
}
} }
}
# A lot of things need yaml, be conservative requiring this package to avoid # A lot of things need yaml, be conservative requiring this package to avoid
# conflicts with other modules. # conflicts with other modules.
if ! defined(Package['python-yaml']) { if ! defined(Package['python-yaml']) {
package { 'python-yaml': package { 'python-yaml':
ensure => present, ensure => present,
}
} }
}
if ! defined(Package['python-paramiko']) { if ! defined(Package['python-paramiko']) {
package { 'python-paramiko': package { 'python-paramiko':
ensure => present, ensure => present,
}
} }
}
if ! defined(Package['python-daemon']) { if ! defined(Package['python-daemon']) {
package { 'python-daemon': package { 'python-daemon':
ensure => present, ensure => present,
}
} }
}
if ! defined(Package['yui-compressor']) { if ! defined(Package['yui-compressor']) {
package { 'yui-compressor': package { 'yui-compressor':
ensure => present, ensure => present,
}
} }
} }
@ -199,17 +199,19 @@ class zuul (
Package['build-essential'], Package['build-essential'],
Package['libffi-dev'], Package['libffi-dev'],
Package['libssl-dev'], Package['libssl-dev'],
Package['python-daemon'],
Package['python-lxml'],
Package['python-paramiko'],
Package['python-paste'],
Package['python-webob'],
Package['python-yaml'],
Package['yappi'], Package['yappi'],
Package['yui-compressor'],
], ],
} }
if ! $zuulv3 {
Exec['install_zuul'] -> Package['python-daemon']
Exec['install_zuul'] -> Package['python-daemon']
Exec['install_zuul'] -> Package['python-lxml']
Exec['install_zuul'] -> Package['python-paramiko']
Exec['install_zuul'] -> Package['python-yaml']
Exec['install_zuul'] -> Package['yui-compressor']
}
file { '/etc/zuul': file { '/etc/zuul':
ensure => directory, ensure => directory,
group => 'zuul', group => 'zuul',
@ -365,117 +367,135 @@ class zuul (
content => $zuul_ssh_private_key, content => $zuul_ssh_private_key,
} }
file { '/var/lib/zuul/www': if $zuulv3 {
ensure => directory, package { 'libjs-jquery':
require => File['/var/lib/zuul'], ensure => absent,
} }
$v2_web_dirs = [
'/opt/twitter-bootstrap',
'/opt/jquery-visibility',
'/var/lib/zuul/www',
'/opt/graphitejs',
]
file { '/var/lib/zuul/www/lib': file { $v2_web_dirs:
ensure => directory, ensure => absent,
require => File['/var/lib/zuul/www'], }
}
package { 'libjs-jquery': } else {
ensure => present,
}
file { '/var/lib/zuul/www/jquery.min.js': file { '/var/lib/zuul/www':
ensure => absent ensure => directory,
} require => File['/var/lib/zuul'],
}
file { '/var/lib/zuul/www/lib/jquery.min.js': file { '/var/lib/zuul/www/lib':
ensure => link, ensure => directory,
target => '/usr/share/javascript/jquery/jquery.min.js', require => File['/var/lib/zuul/www'],
require => [File['/var/lib/zuul/www/lib'], }
Package['libjs-jquery']],
}
vcsrepo { '/opt/twitter-bootstrap': package { 'libjs-jquery':
ensure => latest, ensure => present,
provider => git, }
revision => 'v3.1.1',
source => 'https://github.com/twbs/bootstrap.git',
}
file { '/var/lib/zuul/www/bootstrap': file { '/var/lib/zuul/www/jquery.min.js':
ensure => absent ensure => absent
} }
file { '/var/lib/zuul/www/lib/bootstrap': file { '/var/lib/zuul/www/lib/jquery.min.js':
ensure => link, ensure => link,
target => '/opt/twitter-bootstrap/dist', target => '/usr/share/javascript/jquery/jquery.min.js',
require => [File['/var/lib/zuul/www/lib'], require => [File['/var/lib/zuul/www/lib'],
Package['libjs-jquery'], Package['libjs-jquery']],
Vcsrepo['/opt/twitter-bootstrap']], }
}
vcsrepo { '/opt/jquery-visibility': vcsrepo { '/opt/twitter-bootstrap':
ensure => latest, ensure => latest,
provider => git, provider => git,
revision => 'master', revision => 'v3.1.1',
source => 'https://github.com/mathiasbynens/jquery-visibility.git', source => 'https://github.com/twbs/bootstrap.git',
} }
file { '/var/lib/zuul/www/jquery-visibility.min.js': file { '/var/lib/zuul/www/bootstrap':
ensure => absent ensure => absent
} }
exec { 'install-jquery-visibility': file { '/var/lib/zuul/www/lib/bootstrap':
command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', ensure => link,
path => 'bin:/usr/bin', target => '/opt/twitter-bootstrap/dist',
refreshonly => true, require => [File['/var/lib/zuul/www/lib'],
subscribe => Vcsrepo['/opt/jquery-visibility'], Package['libjs-jquery'],
require => [File['/var/lib/zuul/www/lib'], Vcsrepo['/opt/twitter-bootstrap']],
Package['yui-compressor'], }
Vcsrepo['/opt/jquery-visibility']],
}
vcsrepo { '/opt/graphitejs': vcsrepo { '/opt/jquery-visibility':
ensure => latest, ensure => latest,
provider => git, provider => git,
revision => 'master', revision => 'master',
source => 'https://github.com/prestontimmons/graphitejs.git', source => 'https://github.com/mathiasbynens/jquery-visibility.git',
} }
file { '/var/lib/zuul/www/jquery.graphite.js': file { '/var/lib/zuul/www/jquery-visibility.min.js':
ensure => absent ensure => absent
} }
file { '/var/lib/zuul/www/lib/jquery.graphite.js': exec { 'install-jquery-visibility':
ensure => link, command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js',
target => '/opt/graphitejs/jquery.graphite.js', path => 'bin:/usr/bin',
require => [File['/var/lib/zuul/www/lib'], refreshonly => true,
Vcsrepo['/opt/graphitejs']], subscribe => Vcsrepo['/opt/jquery-visibility'],
} require => [File['/var/lib/zuul/www/lib'],
Package['yui-compressor'],
Vcsrepo['/opt/jquery-visibility']],
}
file { '/var/lib/zuul/www/index.html': vcsrepo { '/opt/graphitejs':
ensure => link, ensure => latest,
target => '/opt/zuul/etc/status/public_html/index.html', provider => git,
require => File['/var/lib/zuul/www'], revision => 'master',
} source => 'https://github.com/prestontimmons/graphitejs.git',
}
file { '/var/lib/zuul/www/styles': file { '/var/lib/zuul/www/jquery.graphite.js':
ensure => link, ensure => absent
target => '/opt/zuul/etc/status/public_html/styles', }
require => File['/var/lib/zuul/www'],
}
file { '/var/lib/zuul/www/zuul.app.js': file { '/var/lib/zuul/www/lib/jquery.graphite.js':
ensure => link, ensure => link,
target => '/opt/zuul/etc/status/public_html/zuul.app.js', target => '/opt/graphitejs/jquery.graphite.js',
require => File['/var/lib/zuul/www'], require => [File['/var/lib/zuul/www/lib'],
} Vcsrepo['/opt/graphitejs']],
}
file { '/var/lib/zuul/www/jquery.zuul.js': file { '/var/lib/zuul/www/index.html':
ensure => link, ensure => link,
target => '/opt/zuul/etc/status/public_html/jquery.zuul.js', target => '/opt/zuul/etc/status/public_html/index.html',
require => File['/var/lib/zuul/www'], require => File['/var/lib/zuul/www'],
} }
file { '/var/lib/zuul/www/images': file { '/var/lib/zuul/www/styles':
ensure => link, ensure => link,
target => '/opt/zuul/etc/status/public_html/images', target => '/opt/zuul/etc/status/public_html/styles',
require => File['/var/lib/zuul/www'], require => File['/var/lib/zuul/www'],
}
file { '/var/lib/zuul/www/zuul.app.js':
ensure => link,
target => '/opt/zuul/etc/status/public_html/zuul.app.js',
require => File['/var/lib/zuul/www'],
}
file { '/var/lib/zuul/www/jquery.zuul.js':
ensure => link,
target => '/opt/zuul/etc/status/public_html/jquery.zuul.js',
require => File['/var/lib/zuul/www'],
}
file { '/var/lib/zuul/www/images':
ensure => link,
target => '/opt/zuul/etc/status/public_html/images',
require => File['/var/lib/zuul/www'],
}
} }
if $zuulv3 { if $zuulv3 {

133
manifests/web.pp
View File

@ -77,9 +77,9 @@ class zuul::web (
} }
} }
file { '/var/lib/zuul/www/backup': file { '/var/lib/zuul/backup':
ensure => directory, ensure => directory,
require => File['/var/lib/zuul/www'], require => File['/var/lib/zuul'],
} }
if $enable_status_backups { if $enable_status_backups {
@ -97,10 +97,10 @@ class zuul::web (
} }
cron { 'zuul_scheduler_status_backup': cron { 'zuul_scheduler_status_backup':
user => 'root', user => 'root',
command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/www/backup/status_$(date +\\%s).json 2>/dev/null", command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/backup/status_$(date +\\%s).json 2>/dev/null",
require => [Package['curl'], require => [Package['curl'],
User['zuul'], User['zuul'],
File['/var/lib/zuul/www/backup']], File['/var/lib/zuul/backup']],
} }
# Rotate backups and keep no more than 120 files -- or 2 hours worth of # Rotate backups and keep no more than 120 files -- or 2 hours worth of
# backup if Zuul has 100% uptime. # backup if Zuul has 100% uptime.
@ -111,84 +111,85 @@ class zuul::web (
cron { 'zuul_scheduler_status_prune': cron { 'zuul_scheduler_status_prune':
user => 'root', user => 'root',
minute => '0', minute => '0',
command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/www/backup/* |sed -e "1,120d" |xargs rm -f', command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/backup/* |sed -e "1,120d" |xargs rm -f',
require => Cron['zuul_scheduler_status_backup'], require => Cron['zuul_scheduler_status_backup'],
} }
} }
file { '/var/lib/zuul/www/static': file { '/var/lib/zuul/www/static':
ensure => absent,
}
$zuul_web_root = '/opt/zuul-web'
$zuul_web_content_root = '/opt/zuul-web/content'
$zuul_web_src_root = '/opt/zuul-web/source'
$zuul_web_filename = 'zuul-content-latest.tar.gz'
$zuul_web_url = "http://tarballs.openstack.org/zuul/${zuul_web_filename}"
file { $zuul_web_root:
ensure => directory, ensure => directory,
require => File['/var/lib/zuul/www'], group => 'zuul',
mode => '0755',
owner => 'zuul',
require => User['zuul'],
} }
file { '/var/lib/zuul/www/static/js': file { $zuul_web_content_root:
ensure => directory, ensure => directory,
require => File['/var/lib/zuul/www/static'], group => 'zuul',
mode => '0755',
owner => 'zuul',
require => [
File[$zuul_web_root],
User['zuul'],
]
} }
file { '/var/lib/zuul/www/static/js/jquery.min.js': file { $zuul_web_src_root:
ensure => link, ensure => directory,
target => '/usr/share/javascript/jquery/jquery.min.js', group => 'zuul',
require => [File['/var/lib/zuul/www/static/js'], mode => '0755',
Package['libjs-jquery']], owner => 'zuul',
require => [
File[$zuul_web_root],
User['zuul'],
]
} }
file { '/var/lib/zuul/www/static/bootstrap': # Download the latest zuul-web
ensure => link, exec { 'get-zuul-web':
target => '/opt/twitter-bootstrap/dist', command => "curl ${zuul_web_url} -z ./${zuul_web_filename} -o ${zuul_web_filename}",
require => [File['/var/lib/zuul/www/static'],
Package['libjs-jquery'],
Vcsrepo['/opt/twitter-bootstrap']],
}
exec { 'install-jquery-visibility-zuul-web':
command => 'yui-compressor -o /var/lib/zuul/www/static/js/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js',
path => 'bin:/usr/bin',
refreshonly => true,
subscribe => Vcsrepo['/opt/jquery-visibility'],
require => [File['/var/lib/zuul/www/static/js'],
Package['yui-compressor'],
Vcsrepo['/opt/jquery-visibility']],
}
file { '/var/lib/zuul/www/static/js/jquery.graphite.js':
ensure => link,
target => '/opt/graphitejs/jquery.graphite.js',
require => [File['/var/lib/zuul/www/static/js'],
Vcsrepo['/opt/graphitejs']],
}
# Download angular
# NOTE: This is using a hardcoded URL because soon this will shift to being
# based on a more javascript-native toolchain.
exec { 'get-angular-zuul-web':
command => 'curl https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js -o /var/lib/zuul/www/static/js/angular.min.js',
path => '/bin:/usr/bin', path => '/bin:/usr/bin',
require => [Package[curl], cwd => $zuul_web_root,
File['/var/lib/zuul/www/static/js']], require => [
onlyif => "curl -I https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js | grep '200 OK'", File[$zuul_web_root],
creates => '/var/lib/zuul/www/static/js/angular.min.js', File[$zuul_web_content_root],
File[$zuul_web_src_root],
],
onlyif => "curl -I ${zuul_web_url} -z ./${zuul_web_filename} | grep '200 OK'",
} }
# For now, symlink in the static parts of zuul-web which are not # Unpack storyboard-zuul_web
# tenant-scoped since they share a URL space with the external exec { 'unpack-zuul-web':
# dependencies. command => "rm -rf ${zuul_web_src_root}/* && tar -C ${zuul_web_src_root} -xzf ./${zuul_web_filename}",
file { '/var/lib/zuul/www/static/javascripts': path => '/bin:/usr/bin',
ensure => link, refreshonly => true,
target => '/opt/zuul/zuul/web/static/javascripts', cwd => $zuul_web_root,
require => [File['/var/lib/zuul/www/static'], require => Exec['get-zuul-web'],
Vcsrepo['/opt/zuul']], subscribe => Exec['get-zuul-web'],
} }
file { '/var/lib/zuul/www/static/images':
ensure => link, # Sync zuul-web to the directory we serve it from. This is so that we don't
target => '/opt/zuul/zuul/web/static/images', # have files go missing - but also so that we can clean up old verisons of
require => [File['/var/lib/zuul/www/static'], # files. The assets built by webpack have hashes in the filenames to help
Vcsrepo['/opt/zuul']], # with caching.
} exec { 'sync-zuul-web':
file { '/var/lib/zuul/www/static/styles': command => "rsync -rl --delete-delay . ${zuul_web_content_root}/",
ensure => link, path => '/bin:/usr/bin',
target => '/opt/zuul/zuul/web/static/styles', refreshonly => true,
require => [File['/var/lib/zuul/www/static'], cwd => $zuul_web_src_root,
Vcsrepo['/opt/zuul']], require => Exec['unpack-zuul-web'],
subscribe => Exec['unpack-zuul-web'],
} }
} }

27
spec/acceptance/basic_spec.rb
View File

@ -1,3 +1,4 @@
# coding: utf-8
require 'puppet-openstack_infra_spec_helper/spec_helper_acceptance' require 'puppet-openstack_infra_spec_helper/spec_helper_acceptance'
# https://blog.lorentzca.me/add-custom-matcher-of-serverspec/ # https://blog.lorentzca.me/add-custom-matcher-of-serverspec/
@ -63,13 +64,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family])
packages = [ packages = [
package('git'), package('git'),
package('build-essential'), package('build-essential'),
package('python-lxml'),
package('python-yaml'),
package('python-paramiko'),
package('python-daemon'),
package('yui-compressor'),
package('python-paste'),
package('python-webob')
] ]
packages << package('apache2') if ['ubuntu', 'debian'].include?(os[:family]) packages << package('apache2') if ['ubuntu', 'debian'].include?(os[:family])
packages << package('httpd') if ['centos', 'redhat'].include?(os[:family]) packages << package('httpd') if ['centos', 'redhat'].include?(os[:family])
@ -136,25 +130,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family])
end end
end end
describe 'public_html symlinks' do
symlinkies = {
file('/var/lib/zuul/www/images') => '/opt/zuul/etc/status/public_html/images',
file('/var/lib/zuul/www/index.html') => '/opt/zuul/etc/status/public_html/index.html',
file('/var/lib/zuul/www/jquery.zuul.js') => '/opt/zuul/etc/status/public_html/jquery.zuul.js',
file('/var/lib/zuul/www/styles') => '/opt/zuul/etc/status/public_html/styles',
file('/var/lib/zuul/www/zuul.app.js') => '/opt/zuul/etc/status/public_html/zuul.app.js',
file('/var/lib/zuul/www/lib/jquery.graphite.js') => '/opt/graphitejs/jquery.graphite.js',
file('/var/lib/zuul/www/lib/bootstrap') => '/opt/twitter-bootstrap/dist',
}
symlinkies.each do |link, destination|
describe link do
it { should be_symlink }
it { should be_linked_to destination }
end
end
end
describe file('/var/lib/zuul/ssh/id_rsa') do describe file('/var/lib/zuul/ssh/id_rsa') do
it { should be_file } it { should be_file }
it { should contain('-----BEGIN RSA PRIVATE KEY-----') } it { should contain('-----BEGIN RSA PRIVATE KEY-----') }

36
templates/zuul.vhost.erb
View File

@ -1,17 +1,7 @@
<VirtualHost *:80> <VirtualHost *:80>
ServerName <%= @vhost_name %> ServerName <%= @vhost_name %>
ServerAdmin <%= @serveradmin %> ServerAdmin <%= @serveradmin %>
DocumentRoot /var/lib/zuul/www
<Directory /var/lib/zuul/www>
<IfVersion >= 2.4>
Require all granted
</IfVersion>
<IfVersion < 2.4>
Order deny,allow
Allow from all
</IfVersion>
</Directory>
<Directory /usr/lib/git-core> <Directory /usr/lib/git-core>
<IfVersion >= 2.4> <IfVersion >= 2.4>
Require all granted Require all granted
@ -42,19 +32,8 @@
<% end %> <% end %>
<% if @zuulv3 %> <% if @zuulv3 %>
RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P]
RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P]
RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P]
RewriteRule ^/status <%= @zuul_web_full_url %>/status [P]
RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P]
RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P]
RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P]
RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P]
RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P]
RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P]
RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P]
RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P]
RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P]
RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P]
<% end %> <% end %>
@ -138,19 +117,8 @@
<% end %> <% end %>
<% if @zuulv3 %> <% if @zuulv3 %>
RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P]
RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P]
RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P]
RewriteRule ^/status <%= @zuul_web_full_url %>/status [P]
RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P]
RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P]
RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P]
RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P]
RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P]
RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P]
RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P]
RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P]
RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P]
RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P]
<% end %> <% end %>

1
templates/zuulv3.conf.erb
View File

@ -74,6 +74,7 @@ disk_limit_per_job=<%= @disk_limit_per_job %>
[web] [web]
log_config=/etc/zuul/web-logging.conf log_config=/etc/zuul/web-logging.conf
static_path=/opt/zuul-web/content
<% if @web_listen_address != "" -%> <% if @web_listen_address != "" -%>
listen_address=<%= @web_listen_address %> listen_address=<%= @web_listen_address %>
<% end -%> <% end -%>