From afa87385cccab9da5d7ade413bdbe83d55f84324 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Mon, 10 Jun 2013 09:39:47 -0700 Subject: [PATCH] Add robots.txt to gerrit. And slow down bing (msnbot). Change-Id: Id8361047abc2cfb52260b3d0ef01275ec3a923f5 Reviewed-on: https://review.openstack.org/32435 Reviewed-by: Jeremy Stanley Reviewed-by: Elizabeth Krumbach Joseph Reviewed-by: Anita Kuno Approved: James E. Blair Tested-by: Jenkins --- modules/gerrit/manifests/init.pp | 11 +++++++++++ modules/gerrit/templates/gerrit.vhost.erb | 7 +++++++ modules/openstack_project/files/gerrit/robots.txt | 11 +++++++++++ modules/openstack_project/manifests/gerrit.pp | 1 + 4 files changed, 30 insertions(+) create mode 100644 modules/openstack_project/files/gerrit/robots.txt diff --git a/modules/gerrit/manifests/init.pp b/modules/gerrit/manifests/init.pp index 23881cba2e..2eb5f2865e 100644 --- a/modules/gerrit/manifests/init.pp +++ b/modules/gerrit/manifests/init.pp @@ -73,6 +73,7 @@ class gerrit( $email_private_key = '', $vhost_name = $::fqdn, $canonicalweburl = "https://${::fqdn}/", + $robots_txt_source = '', # If left empty, the gerrit default will be used. $serveradmin = "webmaster@${::fqdn}", $ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem', $ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key', @@ -311,6 +312,16 @@ class gerrit( } } + if $robots_txt_source != '' { + file { '/home/gerrit2/review_site/static/robots.txt': + owner => 'root', + group => 'root', + mode => '0444', + source => $robots_txt_source, + require => File['/home/gerrit2/review_site/static'], + } + } + if $ssh_dsa_key_contents != '' { file { '/home/gerrit2/review_site/etc/ssh_host_dsa_key': owner => 'gerrit2', diff --git a/modules/gerrit/templates/gerrit.vhost.erb b/modules/gerrit/templates/gerrit.vhost.erb index b26ef57aee..6ebe3c17a8 100644 --- a/modules/gerrit/templates/gerrit.vhost.erb +++ b/modules/gerrit/templates/gerrit.vhost.erb @@ -51,11 +51,18 @@ <% end -%> <% if scope.lookupvar("gerrit::contactstore") == true -%> RewriteCond %{REQUEST_URI} !^/fakestore$ +<% end -%> +<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%> + RewriteCond %{REQUEST_URI} !^/robots.txt$ <% end -%> RewriteRule ^/(.*)$ http://localhost:8081/$1 [P] ProxyPassReverse / http://localhost:8081/ +<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%> + Alias /robots.txt /home/gerrit2/review_site/static/robots.txt +<% end -%> + <% if scope.lookupvar("gerrit::replicate_local") -%> SetEnv GIT_PROJECT_ROOT /var/lib/git/ SetEnv GIT_HTTP_EXPORT_ALL diff --git a/modules/openstack_project/files/gerrit/robots.txt b/modules/openstack_project/files/gerrit/robots.txt new file mode 100644 index 0000000000..05cf74abae --- /dev/null +++ b/modules/openstack_project/files/gerrit/robots.txt @@ -0,0 +1,11 @@ +# Directions for web crawlers. +# See http://www.robotstxt.org/wc/norobots.html. + +User-agent: HTTrack +User-agent: puf +User-agent: MSIECrawler +User-agent: Nutch +Disallow: / + +User-agent: msnbot +Crawl-delay: 1 diff --git a/modules/openstack_project/manifests/gerrit.pp b/modules/openstack_project/manifests/gerrit.pp index 8530ed2222..d225ee1f95 100644 --- a/modules/openstack_project/manifests/gerrit.pp +++ b/modules/openstack_project/manifests/gerrit.pp @@ -97,6 +97,7 @@ class openstack_project::gerrit ( # opinions enable_melody => true, melody_session => true, + robots_txt_source => 'puppet:///modules/openstack_project/gerrit/robots.txt', # passthrough ssl_cert_file => $ssl_cert_file, ssl_key_file => $ssl_key_file,