opendev/system-config
Code Issues Proposed changes

Add robots.txt to gerrit.

Browse Source 
And slow down bing (msnbot).

Change-Id: Id8361047abc2cfb52260b3d0ef01275ec3a923f5
Reviewed-on: https://review.openstack.org/32435
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Elizabeth Krumbach Joseph <lyz@princessleia.com>
Reviewed-by: Anita Kuno <anita.kuno@enovance.com>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins
This commit is contained in:
James E. Blair 2013-06-10 09:39:47 -07:00 committed by Jenkins
parent 74c7dbd6bf
commit afa87385cc
4 changed files with 30 additions and 0 deletions
modules
gerrit
manifests
init.pp
templates
gerrit.vhost.erb
openstack_project
files/gerrit
robots.txt
manifests
gerrit.pp

11
modules/gerrit/manifests/init.pp

@ -73,6 +73,7 @@ class gerrit(
$email_private_key = '', $email_private_key = '',
$vhost_name = $::fqdn, $vhost_name = $::fqdn,
$canonicalweburl = "https://${::fqdn}/", $canonicalweburl = "https://${::fqdn}/",
$robots_txt_source = '', # If left empty, the gerrit default will be used.
$serveradmin = "webmaster@${::fqdn}", $serveradmin = "webmaster@${::fqdn}",
$ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem', $ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem',
$ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key', $ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key',
@ -311,6 +312,16 @@ class gerrit(
} }
} }
if $robots_txt_source != '' {
file { '/home/gerrit2/review_site/static/robots.txt':
owner => 'root',
group => 'root',
mode => '0444',
source => $robots_txt_source,
require => File['/home/gerrit2/review_site/static'],
}
}
if $ssh_dsa_key_contents != '' { if $ssh_dsa_key_contents != '' {
file { '/home/gerrit2/review_site/etc/ssh_host_dsa_key': file { '/home/gerrit2/review_site/etc/ssh_host_dsa_key':
owner => 'gerrit2', owner => 'gerrit2',

7
modules/gerrit/templates/gerrit.vhost.erb

@ -51,11 +51,18 @@
<% end -%> <% end -%>
<% if scope.lookupvar("gerrit::contactstore") == true -%> <% if scope.lookupvar("gerrit::contactstore") == true -%>
RewriteCond %{REQUEST_URI} !^/fakestore$ RewriteCond %{REQUEST_URI} !^/fakestore$
<% end -%>
<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%>
RewriteCond %{REQUEST_URI} !^/robots.txt$
<% end -%> <% end -%>
RewriteRule ^/(.*)$ http://localhost:8081/$1 [P] RewriteRule ^/(.*)$ http://localhost:8081/$1 [P]
ProxyPassReverse / http://localhost:8081/ ProxyPassReverse / http://localhost:8081/
<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%>
Alias /robots.txt /home/gerrit2/review_site/static/robots.txt
<% end -%>
<% if scope.lookupvar("gerrit::replicate_local") -%> <% if scope.lookupvar("gerrit::replicate_local") -%>
SetEnv GIT_PROJECT_ROOT /var/lib/git/ SetEnv GIT_PROJECT_ROOT /var/lib/git/
SetEnv GIT_HTTP_EXPORT_ALL SetEnv GIT_HTTP_EXPORT_ALL

11
modules/openstack_project/files/gerrit/robots.txt Normal file

@ -0,0 +1,11 @@
# Directions for web crawlers.
# See http://www.robotstxt.org/wc/norobots.html.
User-agent: HTTrack
User-agent: puf
User-agent: MSIECrawler
User-agent: Nutch
Disallow: /
User-agent: msnbot
Crawl-delay: 1

1
modules/openstack_project/manifests/gerrit.pp

@ -97,6 +97,7 @@ class openstack_project::gerrit (
# opinions # opinions
enable_melody => true, enable_melody => true,
melody_session => true, melody_session => true,
robots_txt_source => 'puppet:///modules/openstack_project/gerrit/robots.txt',
# passthrough # passthrough
ssl_cert_file => $ssl_cert_file, ssl_cert_file => $ssl_cert_file,
ssl_key_file => $ssl_key_file, ssl_key_file => $ssl_key_file,