codesearch: Add robots.txt

We don't want anything on the codesearch page indexed

Change-Id: I556b77013cf1b7ff2c03426fea92a6d445131f6d

playbooks/roles/codesearch/files/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

playbooks/roles/codesearch/tasks/main.yaml

@@ -44,6 +44,19 @@
     group: root
     mode: 0755
 
+- name: Create hound www storage area
+  file:
+    state: directory
+    path: /var/lib/hound/www
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Copy hound robots.txt
+  copy:
+    src: robots.txt
+    dest: /var/lib/hound/www/robots.txt
+
 - name: Run docker-compose pull
   shell:
     cmd: docker-compose pull

playbooks/roles/codesearch/templates/codesearch.vhost.j2

@@ -40,5 +40,13 @@
   ProxyPass  / http://localhost:6080/ retry=0
   ProxyPassReverse / http://localhost:6080/
 
+  <Location "/robots.txt">
+    ProxyPass !
+  </Location>
+  <Directory "/var/lib/hound/www">
+    Require all granted
+  </Directory>
+  Alias /robots.txt /var/lib/hound/www/robots.txt
+  
 </VirtualHost>
 

testinfra/test_codesearch.py

@@ -25,3 +25,9 @@ def test_codesearch_proxy(host):
                    '--resolve codesearch.opendev.org:443:127.0.0.1 '
                    'https://codesearch.opendev.org')
     assert '<title>Hound</title>' in cmd.stdout
+
+def test_codesearch_robots(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve codesearch.opendev.org:443:127.0.0.1 '
+                   'https://codesearch.opendev.org/robots.txt')
+    assert 'Disallow: /' in cmd.stdout