From 8bf3bbba9325a362926855b02877eac4810acd03 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Wed, 11 Jan 2023 10:37:48 +1100 Subject: [PATCH] mailman: add variable for matching UAs in Apache This allows us to quickly create a list of user agents we may not want to crawl the site. Change-Id: Ibbc84e0f7b529cd029770cc8ec3a3d82477734ce --- playbooks/roles/mailman-site/README.rst | 10 ++++++++++ .../mailman-site/templates/mailman_multihost.vhost.j2 | 7 +++++++ playbooks/roles/mailman/README.rst | 10 ++++++++++ playbooks/roles/mailman/templates/mailman.vhost.j2 | 7 +++++++ playbooks/zuul/run-base.yaml | 1 + playbooks/zuul/templates/group_vars/mailman.yaml.j2 | 3 +++ zuul.d/system-config-run.yaml | 2 ++ 7 files changed, 40 insertions(+) create mode 100644 playbooks/zuul/templates/group_vars/mailman.yaml.j2 diff --git a/playbooks/roles/mailman-site/README.rst b/playbooks/roles/mailman-site/README.rst index 59fa9a7926..c05560946f 100644 --- a/playbooks/roles/mailman-site/README.rst +++ b/playbooks/roles/mailman-site/README.rst @@ -1 +1,11 @@ Role to configure a mailman site in a multihost environment + +**Role Variables** + +.. zuul:rolevar:: mailmain_block_ua + :default: unset + + Set to a list of strings of user agents to block via Apache config. + Note this is a `RewriteCond directive + `__, so + for example to exactly match a string prefix it with ``=``. diff --git a/playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2 b/playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2 index 759b341d9d..a97bb3720b 100644 --- a/playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2 +++ b/playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2 @@ -30,6 +30,13 @@ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer RewriteEngine on + + {% if mailman_block_ua %}{% for ua in mailman_block_ua %} + RewriteCond %{HTTP_USER_AGENT} "{{ ua }}" {{ loop.last | ternary('', '[OR]') }} + {% endfor %} + RewriteRule .- [R=403,L] + {% endif %} + RewriteRule ^/$ /cgi-bin/mailman/listinfo [R] RewriteCond %{HTTP_HOST} ^lists\.openstack\.org$ [nocase] RewriteRule /(cgi-bin/mailman/listinfo|pipermail)/(community|foundation|foundation-board|foundation-board-confidential|goldmembers|marketing|staff|summitsponsors)(/.*|$) %{REQUEST_SCHEME}://lists.openinfra.dev/$1/$2$3 [last,redirect=permanent] diff --git a/playbooks/roles/mailman/README.rst b/playbooks/roles/mailman/README.rst index d25870564d..f5b0c436c0 100644 --- a/playbooks/roles/mailman/README.rst +++ b/playbooks/roles/mailman/README.rst @@ -1 +1,11 @@ Role to configure mailman + +**Role Variables** + +.. zuul:rolevar:: mailmain_block_ua + :default: unset + + Set to a list of strings of user agents to block via Apache config. + Note this is a `RewriteCond directive + `__, so + for example to exactly match a string prefix it with ``=``. diff --git a/playbooks/roles/mailman/templates/mailman.vhost.j2 b/playbooks/roles/mailman/templates/mailman.vhost.j2 index 9f333b1596..2a39b04bb7 100644 --- a/playbooks/roles/mailman/templates/mailman.vhost.j2 +++ b/playbooks/roles/mailman/templates/mailman.vhost.j2 @@ -30,6 +30,13 @@ SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer RewriteEngine on + + {% if mailman_block_ua %}{% for ua in mailman_block_ua %} + RewriteCond %{HTTP_USER_AGENT} "{{ ua }}" {{ loop.last | ternary('', '[OR]') }} + {% endfor %} + RewriteRule .- [R=403,L] + {% endif %} + RewriteRule ^/$ /cgi-bin/mailman/listinfo [R] ScriptAlias /cgi-bin/mailman/ /usr/lib/cgi-bin/mailman/ diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index ba9d1ea3da..808a3450bb 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -125,6 +125,7 @@ - group_vars/kerberos-kdc.yaml - group_vars/keycloak.yaml - group_vars/letsencrypt.yaml + - group_vars/mailman.yaml - group_vars/meetpad.yaml - group_vars/jvb.yaml - group_vars/refstack.yaml diff --git a/playbooks/zuul/templates/group_vars/mailman.yaml.j2 b/playbooks/zuul/templates/group_vars/mailman.yaml.j2 new file mode 100644 index 0000000000..ad906f1caf --- /dev/null +++ b/playbooks/zuul/templates/group_vars/mailman.yaml.j2 @@ -0,0 +1,3 @@ +mailman_block_ua: + - '=Mozilla/5.0 (compatible; FooBot/1.2; +http://example.com)' + - '=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/123.45 (KHTML, like Gecko) Chrome/1.0.0.0 Safari/999.99' diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 0c9b7b1b2a..f3670e8a9c 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -312,12 +312,14 @@ '/var/log/acme.sh': logs '/var/log/apache2': logs '/var/log/mailman': logs + '/etc/apache2/sites-enabled': logs lists.openstack.org: host_copy_output: '/etc/aliases.domain': logs_txt '/var/log/acme.sh': logs '/var/log/apache2': logs '/var/log/mailman': logs + '/etc/apache2/sites-enabled': logs - job: name: system-config-run-lists3