opendev
/
puppet-nodepool
Code Issues Proposed changes

Enable build logs on builder, web app on launchers 

There is currently a confusing array of exporting options for
exporting build and upload logs.  The upload logs have never really
worked, and the info is better given from image-list end-points.  The
build logs since Ia7415d2fbbb320f8eddc4e46c3a055414df5f997 are logged
into separate files into /var/log/nodepool/builds.  With a separate
builder, it can not access the webapp ports provided by the launcher,
so the redirects there are wrong -- and we don't deploy apache on the
launcher to expose them.  If you're using single node anyway, it's
more than likely you have zuul which installs the main website so
can't enable this.

For these reasons deprecate and remove these old settings.

Instead, add a separate flag to each of the builder and launcher.

builder::enable_build_log_via_http will deploy apache and a config
file that exposes /var/log/nodepool/builds

launcher::enable_webapp will deploy apache and redirects to the
internal webapp

Note this does not handle a launcher and builder on the same host;
since the webapp, and indeed refactoring this module in general for
multiple daemons, is under some development, we leave this as a future
task.

Depends-On: https://review.openstack.org/543667
Change-Id: I447886dd32f7f3bc6758ffd7a1b725689d04ee68
This commit is contained in:
Ian Wienand 2018-02-13 11:44:57 +11:00
parent dae2c5809f
commit a8682baa18
6 changed files with 117 additions and 137 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
manifests/builder.pp
View File

@ -17,7 +17,10 @@
class nodepool::builder(
$statsd_host = undef,
$nodepool_ssh_public_key = undef,
$image_log_document_root = '/var/log/nodepool/image',
# If true, export build logs from $build_log_document_root via apache
$enable_build_log_via_http = false,
$build_log_document_root = '/var/log/nodepool/builds',
$vhost_name = $::fqdn,
$builder_logging_conf_template = 'nodepool/nodepool-builder.logging.conf.erb',
$environment = {},
$build_workers = '1',
@ -94,4 +97,35 @@ class nodepool::builder(
],
}
if $enable_build_log_via_http == true {
include ::httpd
::httpd::vhost { $vhost_name:
port => 80,
priority => '50',
docroot => 'MEANINGLESS_ARGUMENT',
template => 'nodepool/nodepool-builder.vhost.erb',
}
if ! defined(Httpd::Mod['rewrite']) {
httpd::mod { 'rewrite': ensure => present }
}
if ! defined(Httpd::Mod['proxy']) {
httpd::mod { 'proxy': ensure => present }
}
if ! defined(Httpd::Mod['proxy_http']) {
httpd::mod { 'proxy_http': ensure => present }
}
}
file { $build_log_document_root:
ensure => directory,
mode => '0755',
owner => 'nodepool',
group => 'nodepool',
require => [
User['nodepool'],
File['/var/log/nodepool'],
],
}
}

112
manifests/init.pp
View File

@ -25,14 +25,23 @@ class nodepool (
$git_source_repo = 'https://git.openstack.org/openstack-infra/nodepool',
$revision = 'master',
$statsd_host = undef,
# The following have all been deprecated and are left only for
# argument compatability
# - To export the image logs on builders use
# builder::enable_build_log_via_http
# - To enable the webapp on launchers use launcher::enable_webapp
# - Upload logs were never really useful, use the webapp endpoint
# to see status
# - TODO: common apache layout with config merging so launcher
# and builder on same host works
$enable_image_log_via_http = undef,
$image_log_document_root = undef,
$vhost_name = $::fqdn,
$image_log_document_root = '/var/log/nodepool/image',
$image_log_periodic_cleanup = false,
$enable_image_log_via_http = false,
$upload_log_document_root = '/var/log/nodepool/upload',
$upload_log_periodic_cleanup = false,
# note : not currently supported
$enable_upload_log_via_http = false,
$image_log_periodic_cleanup = undef,
$upload_log_document_root = undef,
$upload_log_periodic_cleanup = undef,
$enable_upload_log_via_http = undef,
# /end
$environment = {},
# enable sudo for nodepool user. Useful for using dib with nodepool
$sudo = true,
@ -333,95 +342,6 @@ class nodepool (
require => File['/etc/init.d/nodepool'],
}
if $image_log_document_root == $upload_log_document_root {
# It makes no sense to ask to not export build or upload logs, but
# then have them log to the same directory that will be exported.
if (($enable_image_log_via_http and !$enable_upload_log_via_http) or
($enable_upload_log_via_http and !$enable_image_log_via_http)
) {
fail('Unexported logs in same directory as exported logs!')
}
}
# we only need to create the upload log dir if it is separate to the
# image log.
$separate_upload_log_dir =
$image_log_document_root != $upload_log_document_root
if $enable_image_log_via_http == true or
$enable_upload_log_via_http == true {
# Setup apache for log access
include ::httpd
::httpd::vhost { $vhost_name:
port => 80,
priority => '50',
docroot => 'MEANINGLESS_ARGUMENT',
template => 'nodepool/nodepool-log.vhost.erb',
}
if ! defined(Httpd::Mod['rewrite']) {
httpd::mod { 'rewrite': ensure => present }
}
if ! defined(Httpd::Mod['proxy']) {
httpd::mod { 'proxy': ensure => present }
}
if ! defined(Httpd::Mod['proxy_http']) {
httpd::mod { 'proxy_http': ensure => present }
}
}
if $image_log_document_root != '/var/log/nodepool' {
file { $image_log_document_root:
ensure => directory,
mode => '0755',
owner => 'nodepool',
group => 'nodepool',
require => [
User['nodepool'],
File['/var/log/nodepool'],
],
}
}
# we only need this if it is different to the image_log
if $separate_upload_log_dir
{
file { $upload_log_document_root:
ensure => directory,
mode => '0755',
owner => 'nodepool',
group => 'nodepool',
require => [
User['nodepool'],
File['/var/log/nodepool'],
],
}
}
# run a cleanup on the image log directory to cleanup logs for
# images that are no longer being built
if $image_log_periodic_cleanup == true {
cron { 'image_log_cleanup':
user => 'nodepool',
hour => '1',
minute => '0',
command => "find ${image_log_document_root} \\( -name '*.log' -o -name '*.log.*' \\) -mtime +7 -execdir rm {} \\;",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
}
}
# run a cleanup on the upload log directory to cleanup logs for
# providers that are no long uploading
if $upload_log_periodic_cleanup == true {
cron { 'upload_log_cleanup':
user => 'nodepool',
hour => '1',
minute => '0',
command => "find ${upload_log_document_root} \\( -name '*.log' -o -name '*.log.*' \\) -mtime +7 -execdir rm {} \\;",
environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin',
}
}
if $sudo == true {
$sudo_file_ensure = present
}

25
manifests/launcher.pp
View File

@ -19,6 +19,10 @@ class nodepool::launcher(
$statsd_prefix = undef,
$nodepool_ssh_public_key = undef,
$launcher_logging_conf_template = 'nodepool/nodepool-launcher.logging.conf.erb',
# If true, an apache will be setup to redirect webapp end-points to
# the local webapp instance (on port 8005)
$enable_webapp = false,
$vhost_name = $::fqdn,
) {
if ! defined(File['/home/nodepool/.ssh']) {
@ -87,4 +91,25 @@ class nodepool::launcher(
File['/etc/nodepool/launcher-logging.conf'],
],
}
if $enable_webapp == true {
include ::httpd
::httpd::vhost { $vhost_name:
port => 80,
priority => '50',
docroot => 'MEANINGLESS_ARGUMENT',
template => 'nodepool/nodepool-launcher.vhost.erb',
}
if ! defined(Httpd::Mod['rewrite']) {
httpd::mod { 'rewrite': ensure => present }
}
if ! defined(Httpd::Mod['proxy']) {
httpd::mod { 'proxy': ensure => present }
}
if ! defined(Httpd::Mod['proxy_http']) {
httpd::mod { 'proxy_http': ensure => present }
}
}
}

22
templates/nodepool-builder.vhost.erb Normal file
View File

@ -0,0 +1,22 @@
<VirtualHost *:80>
ServerName <%= scope.lookupvar("nodepool::builder::vhost_name") %>
DocumentRoot <%= scope.lookupvar("nodepool::builder::build_log_document_root") %>
<Directory <%= scope.lookupvar("nodepool::builder::build_log_document_root") %>>
Options <%= scope.lookupvar("httpd::params::options") %>
AllowOverride None
Require all granted
</Directory>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined
ServerSignature Off
AddType text/plain .log
<IfModule mod_deflate.c>
SetOutputFilter DEFLATE
</IfModule>
</VirtualHost>

19
templates/nodepool-launcher.vhost.erb Normal file
View File

@ -0,0 +1,19 @@
<VirtualHost *:80>
ServerName <%= scope.lookupvar("nodepool::launcher::vhost_name") %>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined
ServerSignature Off
<IfModule mod_deflate.c>
SetOutputFilter DEFLATE
</IfModule>
RewriteEngine on
RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P]
RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P]
RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P]
RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P]
</VirtualHost>

40
templates/nodepool-log.vhost.erb
View File

@ -1,40 +0,0 @@
<VirtualHost *:80>
ServerName <%= scope.lookupvar("nodepool::vhost_name") %>
DocumentRoot <%= scope.lookupvar("nodepool::image_log_document_root") %>
<% if scope.lookupvar("nodepool::enable_image_log_via_http") -%>
<Directory <%= scope.lookupvar("nodepool::image_log_document_root") %>>
Options <%= scope.lookupvar("httpd::params::options") %>
AllowOverride None
Require all granted
</Directory>
<% end -%>
<% if scope.lookupvar("nodepool::enable_upload_log_via_http") and scope.lookupvar("nodepool::separate_upload_log_dir") -%>
Alias "/upload" <%= scope.lookupvar("nodepool::upload_log_document_root") %>
<Directory <%= scope.lookupvar("nodepool::upload_log_document_root") %> >
Options <%= scope.lookupvar("httpd::params::options") %>
AllowOverride None
Require all granted
</Directory>
<% end -%>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/nodepool_access.log combined
ServerSignature Off
AddType text/plain .log
RewriteEngine on
RewriteRule ^/image-list$ http://127.0.0.1:8005/image-list [P]
RewriteRule ^/dib-image-list$ http://127.0.0.1:8005/dib-image-list [P]
RewriteRule ^/image-list.json$ http://127.0.0.1:8005/image-list.json [P]
RewriteRule ^/dib-image-list.json$ http://127.0.0.1:8005/dib-image-list.json [P]
<IfModule mod_deflate.c>
SetOutputFilter DEFLATE
</IfModule>
</VirtualHost>