From e3ab346f89dfadffde49194d0558ae3f928e612e Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Tue, 13 Jul 2021 14:15:23 +1000
Subject: [PATCH] lodgeit: add robots.txt

Add robots.txt same as the old server.  While we're here and before
the db is setup, put everything under /var/lib/lodgeit/ for general
consistency.

Change-Id: Ib40cba5bb9e1f9f98769c00e2e9fe09e8fb2582f
---
 playbooks/roles/lodgeit/files/robots.txt       |  2 ++
 playbooks/roles/lodgeit/tasks/main.yaml        | 18 +++++++++++++++++-
 .../roles/lodgeit/templates/paste.vhost.j2     |  8 ++++++++
 testinfra/test_paste.py                        |  6 ++++++
 4 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 playbooks/roles/lodgeit/files/robots.txt

diff --git a/playbooks/roles/lodgeit/files/robots.txt b/playbooks/roles/lodgeit/files/robots.txt
new file mode 100644
index 0000000000..1f53798bb4
--- /dev/null
+++ b/playbooks/roles/lodgeit/files/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/playbooks/roles/lodgeit/tasks/main.yaml b/playbooks/roles/lodgeit/tasks/main.yaml
index 6577a1244c..a9a17b4e51 100644
--- a/playbooks/roles/lodgeit/tasks/main.yaml
+++ b/playbooks/roles/lodgeit/tasks/main.yaml
@@ -15,7 +15,7 @@
     - name: Setup db directory
       file:
         state: directory
-        path: /var/lib/lodgeit_db
+        path: /var/lib/lodgeit/mariadb
         owner: root
         group: root
         mode: 0755
@@ -53,6 +53,22 @@
     mode: 0644
   notify: paste Reload apache2
 
+- name: Create www dir
+  file:
+    state: directory
+    path: /var/lib/lodgeit/www
+    owner: root
+    group: root
+    mode: 0755
+
+- name: Install robots.txt
+  copy:
+    src: robots.txt
+    dest: /var/lib/lodgeit/www/robots.txt
+    owner: root
+    group: root
+    mode: 0644
+
 - name: Run docker-compose pull
   shell:
     cmd: docker-compose pull
diff --git a/playbooks/roles/lodgeit/templates/paste.vhost.j2 b/playbooks/roles/lodgeit/templates/paste.vhost.j2
index 99ded2af40..67db387199 100644
--- a/playbooks/roles/lodgeit/templates/paste.vhost.j2
+++ b/playbooks/roles/lodgeit/templates/paste.vhost.j2
@@ -37,5 +37,13 @@
   ProxyPass  / http://localhost:9000/ retry=0
   ProxyPassReverse / http://localhost:9000/
 
+  <Location "/robots.txt">
+    ProxyPass !
+  </Location>
+  <Directory "/var/lib/lodgeit/www">
+    Require all granted
+  </Directory>
+  Alias /robots.txt /var/lib/lodgeit/www/robots.txt
+
 </VirtualHost>
 
diff --git a/testinfra/test_paste.py b/testinfra/test_paste.py
index be4fe0c90b..5124fa12eb 100644
--- a/testinfra/test_paste.py
+++ b/testinfra/test_paste.py
@@ -33,6 +33,12 @@ def test_paste(host):
                    'https://paste.opendev.org')
     assert 'New Paste' in cmd.stdout
 
+def test_paste_robots(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve paste.opendev.org:443:127.0.0.1 '
+                   'https://paste.opendev.org/robots.txt')
+    assert 'Disallow: /' in cmd.stdout
+
 def test_paste_screenshots(host):
     driver = webdriver.Remote(
         command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()),