mailman: add variable for matching UAs in Apache

This allows us to quickly create a list of user agents we may not want
to crawl the site.

Change-Id: Ibbc84e0f7b529cd029770cc8ec3a3d82477734ce

playbooks/roles/mailman-site/README.rst

@@ -1 +1,11 @@
 Role to configure a mailman site in a multihost environment
+
+**Role Variables**
+
+.. zuul:rolevar:: mailmain_block_ua
+   :default: unset
+
+   Set to a list of strings of user agents to block via Apache config.
+   Note this is a `RewriteCond directive
+   <https://httpd.apache.org/docs/2.4/mod/mod_rewrite.html>`__, so
+   for example to exactly match a string prefix it with ``=``.

playbooks/roles/mailman-site/templates/mailman_multihost.vhost.j2

@@ -30,6 +30,13 @@
   SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
 
   RewriteEngine on
+
+  {% if mailman_block_ua %}{% for ua in mailman_block_ua %}
+  RewriteCond %{HTTP_USER_AGENT} "{{ ua }}" {{ loop.last | ternary('', '[OR]') }}
+  {% endfor %}
+  RewriteRule .- [R=403,L]
+  {% endif %}
+
   RewriteRule ^/$ /cgi-bin/mailman/listinfo [R]
   RewriteCond %{HTTP_HOST} ^lists\.openstack\.org$ [nocase]
   RewriteRule /(cgi-bin/mailman/listinfo|pipermail)/(community|foundation|foundation-board|foundation-board-confidential|goldmembers|marketing|staff|summitsponsors)(/.*|$) %{REQUEST_SCHEME}://lists.openinfra.dev/$1/$2$3 [last,redirect=permanent]

playbooks/roles/mailman/README.rst

@@ -1 +1,11 @@
 Role to configure mailman
+
+**Role Variables**
+
+.. zuul:rolevar:: mailmain_block_ua
+   :default: unset
+
+   Set to a list of strings of user agents to block via Apache config.
+   Note this is a `RewriteCond directive
+   <https://httpd.apache.org/docs/2.4/mod/mod_rewrite.html>`__, so
+   for example to exactly match a string prefix it with ``=``.

playbooks/roles/mailman/templates/mailman.vhost.j2

@@ -30,6 +30,13 @@
   SSLCertificateChainFile /etc/letsencrypt-certs/{{ inventory_hostname }}/ca.cer
 
   RewriteEngine on
+
+  {% if mailman_block_ua %}{% for ua in mailman_block_ua %}
+  RewriteCond %{HTTP_USER_AGENT} "{{ ua }}" {{ loop.last | ternary('', '[OR]') }}
+  {% endfor %}
+  RewriteRule .- [R=403,L]
+  {% endif %}
+
   RewriteRule ^/$ /cgi-bin/mailman/listinfo [R]
 
   ScriptAlias /cgi-bin/mailman/ /usr/lib/cgi-bin/mailman/

playbooks/zuul/run-base.yaml

@@ -125,6 +125,7 @@
         - group_vars/kerberos-kdc.yaml
         - group_vars/keycloak.yaml
         - group_vars/letsencrypt.yaml
+        - group_vars/mailman.yaml
         - group_vars/meetpad.yaml
         - group_vars/jvb.yaml
         - group_vars/refstack.yaml

playbooks/zuul/templates/group_vars/mailman.yaml.j2

@@ -0,0 +1,3 @@
+mailman_block_ua:
+  - '=Mozilla/5.0 (compatible; FooBot/1.2; +http://example.com)'
+  - '=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/123.45 (KHTML, like Gecko) Chrome/1.0.0.0 Safari/999.99'

zuul.d/system-config-run.yaml

@@ -312,12 +312,14 @@
           '/var/log/acme.sh': logs
           '/var/log/apache2': logs
           '/var/log/mailman': logs
+          '/etc/apache2/sites-enabled': logs
       lists.openstack.org:
         host_copy_output:
           '/etc/aliases.domain': logs_txt
           '/var/log/acme.sh': logs
           '/var/log/apache2': logs
           '/var/log/mailman': logs
+          '/etc/apache2/sites-enabled': logs
 
 - job:
     name: system-config-run-lists3