Add robots.txt to gerrit.

And slow down bing (msnbot).

Change-Id: Id8361047abc2cfb52260b3d0ef01275ec3a923f5
Reviewed-on: https://review.openstack.org/32435
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Reviewed-by: Elizabeth Krumbach Joseph <lyz@princessleia.com>
Reviewed-by: Anita Kuno <anita.kuno@enovance.com>
Approved: James E. Blair <corvus@inaugust.com>
Tested-by: Jenkins

modules/gerrit/manifests/init.pp

@@ -73,6 +73,7 @@ class gerrit(
   $email_private_key = '',
   $vhost_name = $::fqdn,
   $canonicalweburl = "https://${::fqdn}/",
+  $robots_txt_source = '', # If left empty, the gerrit default will be used.
   $serveradmin = "webmaster@${::fqdn}",
   $ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem',
   $ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key',
@@ -311,6 +312,16 @@ class gerrit(
     }
   }
 
+  if $robots_txt_source != '' {
+    file { '/home/gerrit2/review_site/static/robots.txt':
+      owner    => 'root',
+      group    => 'root',
+      mode     => '0444',
+      source   => $robots_txt_source,
+      require  => File['/home/gerrit2/review_site/static'],
+    }
+  }
+
   if $ssh_dsa_key_contents != '' {
     file { '/home/gerrit2/review_site/etc/ssh_host_dsa_key':
       owner   => 'gerrit2',

modules/gerrit/templates/gerrit.vhost.erb

@@ -51,11 +51,18 @@
 <% end -%>
 <% if scope.lookupvar("gerrit::contactstore") == true -%>
   RewriteCond %{REQUEST_URI} !^/fakestore$
+<% end -%>
+<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%>
+  RewriteCond %{REQUEST_URI} !^/robots.txt$
 <% end -%>
   RewriteRule ^/(.*)$ http://localhost:8081/$1 [P]
 
   ProxyPassReverse / http://localhost:8081/
 
+<% if scope.lookupvar("gerrit::robots_txt_source") != "" -%>
+  Alias /robots.txt /home/gerrit2/review_site/static/robots.txt
+<% end -%>
+
 <% if scope.lookupvar("gerrit::replicate_local") -%>
   SetEnv GIT_PROJECT_ROOT /var/lib/git/
   SetEnv GIT_HTTP_EXPORT_ALL

modules/openstack_project/files/gerrit/robots.txt

@@ -0,0 +1,11 @@
+# Directions for web crawlers.
+# See http://www.robotstxt.org/wc/norobots.html.
+
+User-agent: HTTrack
+User-agent: puf
+User-agent: MSIECrawler
+User-agent: Nutch
+Disallow: /
+
+User-agent: msnbot
+Crawl-delay: 1

modules/openstack_project/manifests/gerrit.pp

@@ -97,6 +97,7 @@ class openstack_project::gerrit (
     # opinions
     enable_melody                   => true,
     melody_session                  => true,
+    robots_txt_source               => 'puppet:///modules/openstack_project/gerrit/robots.txt',
     # passthrough
     ssl_cert_file                   => $ssl_cert_file,
     ssl_key_file                    => $ssl_key_file,