diff --git a/playbooks/roles/zuul-web/files/robots.txt b/playbooks/roles/zuul-web/files/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/playbooks/roles/zuul-web/files/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/playbooks/roles/zuul-web/tasks/main.yaml b/playbooks/roles/zuul-web/tasks/main.yaml index 59c9f1403b..943b9e6224 100644 --- a/playbooks/roles/zuul-web/tasks/main.yaml +++ b/playbooks/roles/zuul-web/tasks/main.yaml @@ -35,6 +35,22 @@ command: a2enconf log-combined-cache when: _log_combined_cache.changed +- name: Create robots.txt location dir + file: + path: /var/www/robots + state: directory + owner: root + group: root + mode: '0755' + +- name: Copy the robots.txt + copy: + src: robots.txt + dest: /var/www/robots/robots.txt + owner: root + group: root + mode: '0644' + - name: Copy apache config template: src: zuul.vhost.j2 diff --git a/playbooks/roles/zuul-web/templates/openstack.vhost.j2 b/playbooks/roles/zuul-web/templates/openstack.vhost.j2 index 521e798b48..d55019bd70 100644 --- a/playbooks/roles/zuul-web/templates/openstack.vhost.j2 +++ b/playbooks/roles/zuul-web/templates/openstack.vhost.j2 @@ -47,6 +47,7 @@ RewriteEngine on + RewriteRule ^/robots.txt$ /var/www/robots/robots.txt [L] RewriteRule ^/api/connection/(.*)$ http://127.0.0.1:9000/api/connection/$1 [P,L] RewriteRule ^/api/console-stream ws://127.0.0.1:9000/api/tenant/openstack/console-stream [P,L] RewriteRule ^/api/(.*)$ http://127.0.0.1:9000/api/tenant/openstack/$1 [P,L] @@ -74,4 +75,7 @@ CacheEnable disk + + Require all granted + diff --git a/playbooks/roles/zuul-web/templates/zuul.vhost.j2 b/playbooks/roles/zuul-web/templates/zuul.vhost.j2 index d921230ef9..338d8e3a28 100644 --- a/playbooks/roles/zuul-web/templates/zuul.vhost.j2 +++ b/playbooks/roles/zuul-web/templates/zuul.vhost.j2 @@ -47,6 +47,7 @@ RewriteEngine on + RewriteRule ^/robots.txt$ /var/www/robots/robots.txt [L] RewriteRule ^/api/tenant/(.*)/console-stream ws://127.0.0.1:9000/api/tenant/$1/console-stream [P,L] RewriteRule ^/(.*)$ http://127.0.0.1:9000/$1 [P,L] @@ -72,4 +73,7 @@ CacheEnable disk + + Require all granted + diff --git a/testinfra/test_zuul_scheduler.py b/testinfra/test_zuul_scheduler.py index 33930f6366..deba3a0323 100644 --- a/testinfra/test_zuul_scheduler.py +++ b/testinfra/test_zuul_scheduler.py @@ -26,3 +26,9 @@ def test_iptables(host): for rule in rules: assert '--dport 7900' not in rule + +def test_zuul_robots(host): + cmd = host.run('curl --insecure ' + '--resolve zuul.opendev.org:443:127.0.0.1 ' + 'https://zuul.opendev.org/robots.txt') + assert 'Disallow: /' in cmd.stdout