diff --git a/playbooks/roles/lodgeit/files/robots.txt b/playbooks/roles/lodgeit/files/robots.txt
new file mode 100644
index 0000000000..1f53798bb4
--- /dev/null
+++ b/playbooks/roles/lodgeit/files/robots.txt
@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /
diff --git a/playbooks/roles/lodgeit/tasks/main.yaml b/playbooks/roles/lodgeit/tasks/main.yaml
index 6577a1244c..a9a17b4e51 100644
--- a/playbooks/roles/lodgeit/tasks/main.yaml
+++ b/playbooks/roles/lodgeit/tasks/main.yaml
@@ -15,7 +15,7 @@
- name: Setup db directory
file:
state: directory
- path: /var/lib/lodgeit_db
+ path: /var/lib/lodgeit/mariadb
owner: root
group: root
mode: 0755
@@ -53,6 +53,22 @@
mode: 0644
notify: paste Reload apache2
+- name: Create www dir
+ file:
+ state: directory
+ path: /var/lib/lodgeit/www
+ owner: root
+ group: root
+ mode: 0755
+
+- name: Install robots.txt
+ copy:
+ src: robots.txt
+ dest: /var/lib/lodgeit/www/robots.txt
+ owner: root
+ group: root
+ mode: 0644
+
- name: Run docker-compose pull
shell:
cmd: docker-compose pull
diff --git a/playbooks/roles/lodgeit/templates/paste.vhost.j2 b/playbooks/roles/lodgeit/templates/paste.vhost.j2
index 99ded2af40..67db387199 100644
--- a/playbooks/roles/lodgeit/templates/paste.vhost.j2
+++ b/playbooks/roles/lodgeit/templates/paste.vhost.j2
@@ -37,5 +37,13 @@
ProxyPass / http://localhost:9000/ retry=0
ProxyPassReverse / http://localhost:9000/
+
+ ProxyPass !
+
+
+ Require all granted
+
+ Alias /robots.txt /var/lib/lodgeit/www/robots.txt
+
diff --git a/testinfra/test_paste.py b/testinfra/test_paste.py
index be4fe0c90b..5124fa12eb 100644
--- a/testinfra/test_paste.py
+++ b/testinfra/test_paste.py
@@ -33,6 +33,12 @@ def test_paste(host):
'https://paste.opendev.org')
assert 'New Paste' in cmd.stdout
+def test_paste_robots(host):
+ cmd = host.run('curl --insecure '
+ '--resolve paste.opendev.org:443:127.0.0.1 '
+ 'https://paste.opendev.org/robots.txt')
+ assert 'Disallow: /' in cmd.stdout
+
def test_paste_screenshots(host):
driver = webdriver.Remote(
command_executor='http://%s:4444/wd/hub' % (host.backend.get_hostname()),