diff --git a/playbooks/roles/codesearch/files/robots.txt b/playbooks/roles/codesearch/files/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/playbooks/roles/codesearch/files/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/playbooks/roles/codesearch/tasks/main.yaml b/playbooks/roles/codesearch/tasks/main.yaml index b9a71ac025..c4d09bd65d 100644 --- a/playbooks/roles/codesearch/tasks/main.yaml +++ b/playbooks/roles/codesearch/tasks/main.yaml @@ -44,6 +44,19 @@ group: root mode: 0755 +- name: Create hound www storage area + file: + state: directory + path: /var/lib/hound/www + owner: root + group: root + mode: 0755 + +- name: Copy hound robots.txt + copy: + src: robots.txt + dest: /var/lib/hound/www/robots.txt + - name: Run docker-compose pull shell: cmd: docker-compose pull diff --git a/playbooks/roles/codesearch/templates/codesearch.vhost.j2 b/playbooks/roles/codesearch/templates/codesearch.vhost.j2 index de70b25c7d..e8227445a6 100644 --- a/playbooks/roles/codesearch/templates/codesearch.vhost.j2 +++ b/playbooks/roles/codesearch/templates/codesearch.vhost.j2 @@ -40,5 +40,13 @@ ProxyPass / http://localhost:6080/ retry=0 ProxyPassReverse / http://localhost:6080/ + + ProxyPass ! + + + Require all granted + + Alias /robots.txt /var/lib/hound/www/robots.txt + diff --git a/testinfra/test_codesearch.py b/testinfra/test_codesearch.py index a057caf7cd..0a575ea1ff 100644 --- a/testinfra/test_codesearch.py +++ b/testinfra/test_codesearch.py @@ -25,3 +25,9 @@ def test_codesearch_proxy(host): '--resolve codesearch.opendev.org:443:127.0.0.1 ' 'https://codesearch.opendev.org') assert 'Hound' in cmd.stdout + +def test_codesearch_robots(host): + cmd = host.run('curl --insecure ' + '--resolve codesearch.opendev.org:443:127.0.0.1 ' + 'https://codesearch.opendev.org/robots.txt') + assert 'Disallow: /' in cmd.stdout