lodgeit: add robots.txt

Add robots.txt same as the old server.  While we're here and before
the db is setup, put everything under /var/lib/lodgeit/ for general
consistency.

Change-Id: Ib40cba5bb9e1f9f98769c00e2e9fe09e8fb2582f

playbooks/roles/lodgeit/files/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

playbooks/roles/lodgeit/tasks/main.yaml

@@ -15,7 +15,7 @@
     - name: Setup db directory
       file:
         state: directory
-        path: /var/lib/lodgeit_db
+        path: /var/lib/lodgeit/mariadb
         owner: root
         group: root
         mode: 0755
@@ -53,6 +53,22 @@
     mode: 0644
   notify: paste Reload apache2
 
+- name: Create www dir
+  file:
+    state: directory
+    path: /var/lib/lodgeit/www
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Install robots.txt
+  copy:
+    src: robots.txt
+    dest: /var/lib/lodgeit/www/robots.txt
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Run docker-compose pull
   shell:
     cmd: docker-compose pull

playbooks/roles/lodgeit/templates/paste.vhost.j2

@@ -37,5 +37,13 @@
   ProxyPass  / http://localhost:9000/ retry=0
   ProxyPassReverse / http://localhost:9000/
 
+  <Location "/robots.txt">
+    ProxyPass !
+  </Location>
+  <Directory "/var/lib/lodgeit/www">
+    Require all granted
+  </Directory>
+  Alias /robots.txt /var/lib/lodgeit/www/robots.txt
+
 </VirtualHost>
 

testinfra/test_paste.py

@@ -33,6 +33,12 @@ def test_paste(host):
                    'https://paste.opendev.org')
     assert 'New Paste' in cmd.stdout
 
+def test_paste_robots(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve paste.opendev.org:443:127.0.0.1 '
+                   'https://paste.opendev.org/robots.txt')
+    assert 'Disallow: /' in cmd.stdout
+
 def test_paste_screenshots(host):
     driver = webdriver.Remote(
         command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()),