Clark Boylan 7ad3125814 Add robots.txt to Zuul web
This adds a robots.txt that kindly asks bots to not crawl anything on
zuul. We've seen soem bots crawling which leads to them trolling the
build logs which seems like overkill and increases bandwidth usage in
our donor clouds. Ask them to stop and quiet everything down a bit.

Change-Id: I88d85c7a51159b5b020aa179e24acec55fb42931
2024-04-03 13:31:06 -07:00

140 lines
2.9 KiB
YAML

- name: Install apache2
apt:
name:
- apache2
- apache2-utils
state: present
- name: Add UA filter macro to apache config
# This is used in the zuul apache vhost.
include_role:
name: apache-ua-filter
- name: Apache modules
apache2_module:
state: present
name: "{{ item }}"
loop:
- rewrite
- proxy
- proxy_http
- proxy_wstunnel
- ssl
- cache
- cache_disk
- cache_socache
- headers
- name: Copy common log configuration
copy:
src: log-combined-cache.conf
dest: /etc/apache2/conf-available
register: _log_combined_cache
- name: Enable combined cache log config
command: a2enconf log-combined-cache
when: _log_combined_cache.changed
- name: Create robots.txt location dir
file:
path: /var/www/robots
state: directory
owner: root
group: root
mode: '0755'
- name: Copy the robots.txt
copy:
src: robots.txt
dest: /var/www/robots/robots.txt
owner: root
group: root
mode: '0644'
- name: Copy apache config
template:
src: zuul.vhost.j2
dest: /etc/apache2/sites-enabled/000-default.conf
owner: root
group: root
mode: 0644
notify: zuul Reload apache2
- name: Copy whitelabel config
template:
src: openstack.vhost.j2
dest: "/etc/apache2/sites-enabled/010-openstack.conf"
owner: root
group: root
mode: 0644
notify: zuul Reload apache2
- name: Copy htcacheclean config
copy:
src: apache-htcacheclean.default
dest: /etc/default/apache-htcacheclean
owner: root
group: root
mode: '0644'
- name: Install logging config
copy:
src: logging.conf
dest: /etc/zuul/web-logging.conf
- name: Install fingergw logging config
copy:
src: fingergw-logging.conf
dest: /etc/zuul/fingergw-logging.conf
- name: Rotate web logs
include_role:
name: logrotate
vars:
logrotate_file_name: /var/log/zuul/web.log
logrotate_rotate: 30
- name: Rotate web debug logs
include_role:
name: logrotate
vars:
logrotate_file_name: /var/log/zuul/web-debug.log
logrotate_rotate: 30
- name: Rotate fingergw logs
include_role:
name: logrotate
vars:
logrotate_file_name: /var/log/zuul/fingergw.log
logrotate_rotate: 30
- name: Rotate fingergw debug logs
include_role:
name: logrotate
vars:
logrotate_file_name: /var/log/zuul/fingergw-debug.log
logrotate_rotate: 30
- name: Make docker-compose directory
file:
state: directory
path: /etc/zuul-web
- name: Install docker-compose file
copy:
src: docker-compose.yaml
dest: /etc/zuul-web/docker-compose.yaml
- name: Update container images
include_tasks: pull.yaml
- name: Start containers
include_tasks: start.yaml
when: zuul_web_start | bool
# We can prune here as it should leave the "latest" tagged images
# as well as the currently running images.
- name: Run docker prune to cleanup unneeded images
shell:
cmd: docker image prune -f