diff --git a/manifests/init.pp b/manifests/init.pp index 01a3c9b..8881fe4 100644 --- a/manifests/init.pp +++ b/manifests/init.pp @@ -114,8 +114,6 @@ class zuul ( $packages = [ 'libffi-dev', 'libssl-dev', - 'python-paste', - 'python-webob', ] package { $packages: @@ -135,37 +133,39 @@ class zuul ( require => Class['pip'], } - # needed by python-keystoneclient, has system bindings - # Zuul and Nodepool both need it, so make it conditional - if ! defined(Package['python-lxml']) { - package { 'python-lxml': - ensure => present, + if ! $zuulv3 { + # needed by python-keystoneclient, has system bindings + # Zuul and Nodepool both need it, so make it conditional + if ! defined(Package['python-lxml']) { + package { 'python-lxml': + ensure => present, + } } - } - # A lot of things need yaml, be conservative requiring this package to avoid - # conflicts with other modules. - if ! defined(Package['python-yaml']) { - package { 'python-yaml': - ensure => present, + # A lot of things need yaml, be conservative requiring this package to avoid + # conflicts with other modules. + if ! defined(Package['python-yaml']) { + package { 'python-yaml': + ensure => present, + } } - } - if ! defined(Package['python-paramiko']) { - package { 'python-paramiko': - ensure => present, + if ! defined(Package['python-paramiko']) { + package { 'python-paramiko': + ensure => present, + } } - } - if ! defined(Package['python-daemon']) { - package { 'python-daemon': - ensure => present, + if ! defined(Package['python-daemon']) { + package { 'python-daemon': + ensure => present, + } } - } - if ! defined(Package['yui-compressor']) { - package { 'yui-compressor': - ensure => present, + if ! defined(Package['yui-compressor']) { + package { 'yui-compressor': + ensure => present, + } } } @@ -199,17 +199,19 @@ class zuul ( Package['build-essential'], Package['libffi-dev'], Package['libssl-dev'], - Package['python-daemon'], - Package['python-lxml'], - Package['python-paramiko'], - Package['python-paste'], - Package['python-webob'], - Package['python-yaml'], Package['yappi'], - Package['yui-compressor'], ], } + if ! $zuulv3 { + Exec['install_zuul'] -> Package['python-daemon'] + Exec['install_zuul'] -> Package['python-daemon'] + Exec['install_zuul'] -> Package['python-lxml'] + Exec['install_zuul'] -> Package['python-paramiko'] + Exec['install_zuul'] -> Package['python-yaml'] + Exec['install_zuul'] -> Package['yui-compressor'] + } + file { '/etc/zuul': ensure => directory, group => 'zuul', @@ -365,117 +367,135 @@ class zuul ( content => $zuul_ssh_private_key, } - file { '/var/lib/zuul/www': - ensure => directory, - require => File['/var/lib/zuul'], - } + if $zuulv3 { + package { 'libjs-jquery': + ensure => absent, + } + $v2_web_dirs = [ + '/opt/twitter-bootstrap', + '/opt/jquery-visibility', + '/var/lib/zuul/www', + '/opt/graphitejs', + ] - file { '/var/lib/zuul/www/lib': - ensure => directory, - require => File['/var/lib/zuul/www'], - } + file { $v2_web_dirs: + ensure => absent, + } - package { 'libjs-jquery': - ensure => present, - } + } else { - file { '/var/lib/zuul/www/jquery.min.js': - ensure => absent - } + file { '/var/lib/zuul/www': + ensure => directory, + require => File['/var/lib/zuul'], + } - file { '/var/lib/zuul/www/lib/jquery.min.js': - ensure => link, - target => '/usr/share/javascript/jquery/jquery.min.js', - require => [File['/var/lib/zuul/www/lib'], - Package['libjs-jquery']], - } + file { '/var/lib/zuul/www/lib': + ensure => directory, + require => File['/var/lib/zuul/www'], + } - vcsrepo { '/opt/twitter-bootstrap': - ensure => latest, - provider => git, - revision => 'v3.1.1', - source => 'https://github.com/twbs/bootstrap.git', - } + package { 'libjs-jquery': + ensure => present, + } - file { '/var/lib/zuul/www/bootstrap': - ensure => absent - } + file { '/var/lib/zuul/www/jquery.min.js': + ensure => absent + } - file { '/var/lib/zuul/www/lib/bootstrap': - ensure => link, - target => '/opt/twitter-bootstrap/dist', - require => [File['/var/lib/zuul/www/lib'], - Package['libjs-jquery'], - Vcsrepo['/opt/twitter-bootstrap']], - } + file { '/var/lib/zuul/www/lib/jquery.min.js': + ensure => link, + target => '/usr/share/javascript/jquery/jquery.min.js', + require => [File['/var/lib/zuul/www/lib'], + Package['libjs-jquery']], + } - vcsrepo { '/opt/jquery-visibility': - ensure => latest, - provider => git, - revision => 'master', - source => 'https://github.com/mathiasbynens/jquery-visibility.git', - } + vcsrepo { '/opt/twitter-bootstrap': + ensure => latest, + provider => git, + revision => 'v3.1.1', + source => 'https://github.com/twbs/bootstrap.git', + } - file { '/var/lib/zuul/www/jquery-visibility.min.js': - ensure => absent - } + file { '/var/lib/zuul/www/bootstrap': + ensure => absent + } - exec { 'install-jquery-visibility': - command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', - path => 'bin:/usr/bin', - refreshonly => true, - subscribe => Vcsrepo['/opt/jquery-visibility'], - require => [File['/var/lib/zuul/www/lib'], - Package['yui-compressor'], - Vcsrepo['/opt/jquery-visibility']], - } + file { '/var/lib/zuul/www/lib/bootstrap': + ensure => link, + target => '/opt/twitter-bootstrap/dist', + require => [File['/var/lib/zuul/www/lib'], + Package['libjs-jquery'], + Vcsrepo['/opt/twitter-bootstrap']], + } - vcsrepo { '/opt/graphitejs': - ensure => latest, - provider => git, - revision => 'master', - source => 'https://github.com/prestontimmons/graphitejs.git', - } + vcsrepo { '/opt/jquery-visibility': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://github.com/mathiasbynens/jquery-visibility.git', + } - file { '/var/lib/zuul/www/jquery.graphite.js': - ensure => absent - } + file { '/var/lib/zuul/www/jquery-visibility.min.js': + ensure => absent + } - file { '/var/lib/zuul/www/lib/jquery.graphite.js': - ensure => link, - target => '/opt/graphitejs/jquery.graphite.js', - require => [File['/var/lib/zuul/www/lib'], - Vcsrepo['/opt/graphitejs']], - } + exec { 'install-jquery-visibility': + command => 'yui-compressor -o /var/lib/zuul/www/lib/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', + path => 'bin:/usr/bin', + refreshonly => true, + subscribe => Vcsrepo['/opt/jquery-visibility'], + require => [File['/var/lib/zuul/www/lib'], + Package['yui-compressor'], + Vcsrepo['/opt/jquery-visibility']], + } - file { '/var/lib/zuul/www/index.html': - ensure => link, - target => '/opt/zuul/etc/status/public_html/index.html', - require => File['/var/lib/zuul/www'], - } + vcsrepo { '/opt/graphitejs': + ensure => latest, + provider => git, + revision => 'master', + source => 'https://github.com/prestontimmons/graphitejs.git', + } - file { '/var/lib/zuul/www/styles': - ensure => link, - target => '/opt/zuul/etc/status/public_html/styles', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/jquery.graphite.js': + ensure => absent + } - file { '/var/lib/zuul/www/zuul.app.js': - ensure => link, - target => '/opt/zuul/etc/status/public_html/zuul.app.js', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/lib/jquery.graphite.js': + ensure => link, + target => '/opt/graphitejs/jquery.graphite.js', + require => [File['/var/lib/zuul/www/lib'], + Vcsrepo['/opt/graphitejs']], + } - file { '/var/lib/zuul/www/jquery.zuul.js': - ensure => link, - target => '/opt/zuul/etc/status/public_html/jquery.zuul.js', - require => File['/var/lib/zuul/www'], - } + file { '/var/lib/zuul/www/index.html': + ensure => link, + target => '/opt/zuul/etc/status/public_html/index.html', + require => File['/var/lib/zuul/www'], + } - file { '/var/lib/zuul/www/images': - ensure => link, - target => '/opt/zuul/etc/status/public_html/images', - require => File['/var/lib/zuul/www'], + file { '/var/lib/zuul/www/styles': + ensure => link, + target => '/opt/zuul/etc/status/public_html/styles', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/zuul.app.js': + ensure => link, + target => '/opt/zuul/etc/status/public_html/zuul.app.js', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/jquery.zuul.js': + ensure => link, + target => '/opt/zuul/etc/status/public_html/jquery.zuul.js', + require => File['/var/lib/zuul/www'], + } + + file { '/var/lib/zuul/www/images': + ensure => link, + target => '/opt/zuul/etc/status/public_html/images', + require => File['/var/lib/zuul/www'], + } } if $zuulv3 { diff --git a/manifests/web.pp b/manifests/web.pp index edfb550..db83dd2 100644 --- a/manifests/web.pp +++ b/manifests/web.pp @@ -77,9 +77,9 @@ class zuul::web ( } } - file { '/var/lib/zuul/www/backup': + file { '/var/lib/zuul/backup': ensure => directory, - require => File['/var/lib/zuul/www'], + require => File['/var/lib/zuul'], } if $enable_status_backups { @@ -97,10 +97,10 @@ class zuul::web ( } cron { 'zuul_scheduler_status_backup': user => 'root', - command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/www/backup/status_$(date +\\%s).json 2>/dev/null", + command => "timeout -k 5 10 curl ${status} -o /var/lib/zuul/backup/status_$(date +\\%s).json 2>/dev/null", require => [Package['curl'], User['zuul'], - File['/var/lib/zuul/www/backup']], + File['/var/lib/zuul/backup']], } # Rotate backups and keep no more than 120 files -- or 2 hours worth of # backup if Zuul has 100% uptime. @@ -111,84 +111,85 @@ class zuul::web ( cron { 'zuul_scheduler_status_prune': user => 'root', minute => '0', - command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/www/backup/* |sed -e "1,120d" |xargs rm -f', + command => 'flock -n /var/run/status_prune.lock ls -dt -1 /var/lib/zuul/backup/* |sed -e "1,120d" |xargs rm -f', require => Cron['zuul_scheduler_status_backup'], } } file { '/var/lib/zuul/www/static': + ensure => absent, + } + + $zuul_web_root = '/opt/zuul-web' + $zuul_web_content_root = '/opt/zuul-web/content' + $zuul_web_src_root = '/opt/zuul-web/source' + $zuul_web_filename = 'zuul-content-latest.tar.gz' + $zuul_web_url = "http://tarballs.openstack.org/zuul/${zuul_web_filename}" + + file { $zuul_web_root: ensure => directory, - require => File['/var/lib/zuul/www'], + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => User['zuul'], } - file { '/var/lib/zuul/www/static/js': + file { $zuul_web_content_root: ensure => directory, - require => File['/var/lib/zuul/www/static'], + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => [ + File[$zuul_web_root], + User['zuul'], + ] } - file { '/var/lib/zuul/www/static/js/jquery.min.js': - ensure => link, - target => '/usr/share/javascript/jquery/jquery.min.js', - require => [File['/var/lib/zuul/www/static/js'], - Package['libjs-jquery']], + file { $zuul_web_src_root: + ensure => directory, + group => 'zuul', + mode => '0755', + owner => 'zuul', + require => [ + File[$zuul_web_root], + User['zuul'], + ] } - file { '/var/lib/zuul/www/static/bootstrap': - ensure => link, - target => '/opt/twitter-bootstrap/dist', - require => [File['/var/lib/zuul/www/static'], - Package['libjs-jquery'], - Vcsrepo['/opt/twitter-bootstrap']], - } - - exec { 'install-jquery-visibility-zuul-web': - command => 'yui-compressor -o /var/lib/zuul/www/static/js/jquery-visibility.js /opt/jquery-visibility/jquery-visibility.js', - path => 'bin:/usr/bin', - refreshonly => true, - subscribe => Vcsrepo['/opt/jquery-visibility'], - require => [File['/var/lib/zuul/www/static/js'], - Package['yui-compressor'], - Vcsrepo['/opt/jquery-visibility']], - } - - file { '/var/lib/zuul/www/static/js/jquery.graphite.js': - ensure => link, - target => '/opt/graphitejs/jquery.graphite.js', - require => [File['/var/lib/zuul/www/static/js'], - Vcsrepo['/opt/graphitejs']], - } - - # Download angular - # NOTE: This is using a hardcoded URL because soon this will shift to being - # based on a more javascript-native toolchain. - exec { 'get-angular-zuul-web': - command => 'curl https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js -o /var/lib/zuul/www/static/js/angular.min.js', + # Download the latest zuul-web + exec { 'get-zuul-web': + command => "curl ${zuul_web_url} -z ./${zuul_web_filename} -o ${zuul_web_filename}", path => '/bin:/usr/bin', - require => [Package[curl], - File['/var/lib/zuul/www/static/js']], - onlyif => "curl -I https://code.angularjs.org/1.5.8/angular.min.js -z /var/lib/zuul/www/static/js/angular.min.js | grep '200 OK'", - creates => '/var/lib/zuul/www/static/js/angular.min.js', + cwd => $zuul_web_root, + require => [ + File[$zuul_web_root], + File[$zuul_web_content_root], + File[$zuul_web_src_root], + ], + onlyif => "curl -I ${zuul_web_url} -z ./${zuul_web_filename} | grep '200 OK'", } - # For now, symlink in the static parts of zuul-web which are not - # tenant-scoped since they share a URL space with the external - # dependencies. - file { '/var/lib/zuul/www/static/javascripts': - ensure => link, - target => '/opt/zuul/zuul/web/static/javascripts', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], + # Unpack storyboard-zuul_web + exec { 'unpack-zuul-web': + command => "rm -rf ${zuul_web_src_root}/* && tar -C ${zuul_web_src_root} -xzf ./${zuul_web_filename}", + path => '/bin:/usr/bin', + refreshonly => true, + cwd => $zuul_web_root, + require => Exec['get-zuul-web'], + subscribe => Exec['get-zuul-web'], } - file { '/var/lib/zuul/www/static/images': - ensure => link, - target => '/opt/zuul/zuul/web/static/images', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], - } - file { '/var/lib/zuul/www/static/styles': - ensure => link, - target => '/opt/zuul/zuul/web/static/styles', - require => [File['/var/lib/zuul/www/static'], - Vcsrepo['/opt/zuul']], + + # Sync zuul-web to the directory we serve it from. This is so that we don't + # have files go missing - but also so that we can clean up old verisons of + # files. The assets built by webpack have hashes in the filenames to help + # with caching. + exec { 'sync-zuul-web': + command => "rsync -rl --delete-delay . ${zuul_web_content_root}/", + path => '/bin:/usr/bin', + refreshonly => true, + cwd => $zuul_web_src_root, + require => Exec['unpack-zuul-web'], + subscribe => Exec['unpack-zuul-web'], } + } diff --git a/spec/acceptance/basic_spec.rb b/spec/acceptance/basic_spec.rb index a83c5ac..6813f80 100644 --- a/spec/acceptance/basic_spec.rb +++ b/spec/acceptance/basic_spec.rb @@ -1,3 +1,4 @@ +# coding: utf-8 require 'puppet-openstack_infra_spec_helper/spec_helper_acceptance' # https://blog.lorentzca.me/add-custom-matcher-of-serverspec/ @@ -63,13 +64,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family]) packages = [ package('git'), package('build-essential'), - package('python-lxml'), - package('python-yaml'), - package('python-paramiko'), - package('python-daemon'), - package('yui-compressor'), - package('python-paste'), - package('python-webob') ] packages << package('apache2') if ['ubuntu', 'debian'].include?(os[:family]) packages << package('httpd') if ['centos', 'redhat'].include?(os[:family]) @@ -136,25 +130,6 @@ describe 'puppet-zuul module', :if => ['debian', 'ubuntu'].include?(os[:family]) end end - describe 'public_html symlinks' do - symlinkies = { - file('/var/lib/zuul/www/images') => '/opt/zuul/etc/status/public_html/images', - file('/var/lib/zuul/www/index.html') => '/opt/zuul/etc/status/public_html/index.html', - file('/var/lib/zuul/www/jquery.zuul.js') => '/opt/zuul/etc/status/public_html/jquery.zuul.js', - file('/var/lib/zuul/www/styles') => '/opt/zuul/etc/status/public_html/styles', - file('/var/lib/zuul/www/zuul.app.js') => '/opt/zuul/etc/status/public_html/zuul.app.js', - file('/var/lib/zuul/www/lib/jquery.graphite.js') => '/opt/graphitejs/jquery.graphite.js', - file('/var/lib/zuul/www/lib/bootstrap') => '/opt/twitter-bootstrap/dist', - } - - symlinkies.each do |link, destination| - describe link do - it { should be_symlink } - it { should be_linked_to destination } - end - end - end - describe file('/var/lib/zuul/ssh/id_rsa') do it { should be_file } it { should contain('-----BEGIN RSA PRIVATE KEY-----') } diff --git a/templates/zuul.vhost.erb b/templates/zuul.vhost.erb index 924d513..fee230a 100644 --- a/templates/zuul.vhost.erb +++ b/templates/zuul.vhost.erb @@ -1,17 +1,7 @@ ServerName <%= @vhost_name %> ServerAdmin <%= @serveradmin %> - DocumentRoot /var/lib/zuul/www - - = 2.4> - Require all granted - - - Order deny,allow - Allow from all - - = 2.4> Require all granted @@ -42,19 +32,8 @@ <% end %> <% if @zuulv3 %> - RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P] - RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P] - RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P] - RewriteRule ^/status <%= @zuul_web_full_url %>/status [P] - RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P] - RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P] - RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P] - RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P] - RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P] - RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] - RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] - RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P] + RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] <% end %> @@ -138,19 +117,8 @@ <% end %> <% if @zuulv3 %> - RewriteRule ^/connection/(.*) <%= @zuul_web_url %>/connection/$1 [P] - RewriteRule ^/status.json <%= @zuul_web_full_url %>/status.json [P] - RewriteRule ^/status.html <%= @zuul_web_full_url %>/status.html [P] - RewriteRule ^/status <%= @zuul_web_full_url %>/status [P] - RewriteRule ^/jobs.json <%= @zuul_web_full_url %>/jobs.json [P] - RewriteRule ^/jobs.html <%= @zuul_web_full_url %>/jobs.html [P] - RewriteRule ^/jobs <%= @zuul_web_full_url %>/jobs [P] - RewriteRule ^/builds.json <%= @zuul_web_full_url %>/builds.json [P] - RewriteRule ^/builds.html <%= @zuul_web_full_url %>/builds.html [P] - RewriteRule ^/builds <%= @zuul_web_full_url %>/builds [P] RewriteRule ^/console-stream <%= @zuul_web_full_url.sub('http://', 'ws://') %>/console-stream [P] - RewriteRule ^/stream.html <%= @zuul_web_full_url %>/stream.html [P] - RewriteRule ^/(.*).pub$ <%= @zuul_web_full_url %>/$1.pub [P] + RewriteRule ^/(.*)$ <%= @zuul_web_full_url %>/$1 [P] RewriteRule ^/$ <%= @zuul_web_full_url %>/status.html [P] <% end %> diff --git a/templates/zuulv3.conf.erb b/templates/zuulv3.conf.erb index 187e7e6..180c8c3 100644 --- a/templates/zuulv3.conf.erb +++ b/templates/zuulv3.conf.erb @@ -74,6 +74,7 @@ disk_limit_per_job=<%= @disk_limit_per_job %> [web] log_config=/etc/zuul/web-logging.conf +static_path=/opt/zuul-web/content <% if @web_listen_address != "" -%> listen_address=<%= @web_listen_address %> <% end -%>