Browse Source

Block restricted user agents for the tarballs site

A copy of the filter used for our Gitea farm, this same activity has
been showing up on our tarballs.opendev.org site as well which is
consuming available connection slots for all vhosts on the static
server.

This is implemented as a macro so that it can be included into
additional vhosts, and put into a separate role so that it can be
added to all playbooks which need it. A subsequent change will add
it to the Gitea servers, eliminating the redundant copy there.

Change-Id: Ic2020b753076209f7708f76744fdf746bf933bd9
changes/95/758495/7
Jeremy Stanley 2 years ago
parent
commit
deaae4f665
  1. 6
      playbooks/roles/apache-ua-filter/README.rst
  2. 52
      playbooks/roles/apache-ua-filter/files/ua-filter.conf
  3. 9
      playbooks/roles/apache-ua-filter/handlers/main.yaml
  4. 33
      playbooks/roles/apache-ua-filter/tasks/main.yaml
  5. 2
      playbooks/roles/static/files/50-tarballs.opendev.org.conf
  6. 1
      playbooks/service-static.yaml

6
playbooks/roles/apache-ua-filter/README.rst

@ -0,0 +1,6 @@
Reject requests from problematic user agent strings
This role installs and configures a filter macro called
UserAgentFilter which can be included in Apache vhosts
**Role Variables**

52
playbooks/roles/apache-ua-filter/files/ua-filter.conf

@ -0,0 +1,52 @@
# Returns a 403 Not Authorized for these user agent strings
<Macro UserAgentFilter>
RewriteEngine On
# This interesting list is taken from
#
# https://github.com/mythsman/weiboCrawler/blob/master/opener.py
#
# which appears to be a crawler for a site "weibo" which is crawling
# gitea relentlessly. It seems to rotate though these to avoid
# detection. We are seeing these very specific user-agents appear
# suggesting this code has been repurposed or has otherwise gone mad
# and started going through opendev.org
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows NT 10.0; WOW64; rv:38.0) Gecko/20100101 Firefox/38.0" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:2.0.1) Gecko/20100101 Firefox/4.0.1" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Windows NT 6.1; rv:2.0.1) Gecko/20100101 Firefox/4.0.1" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPhone; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPod; U; CPU iPhone OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (iPad; U; CPU OS 4_3_3 like Mac OS X; en-us) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8J2 Safari/6533.18.5" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" [OR]
RewriteCond %{HTTP_USER_AGENT} "=MQQBrowser/26 Mozilla/5.0 (Linux; U; Android 2.3.7; zh-cn; MB200 Build/GRJ22; CyanogenMod-7) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Opera/9.80 (Android 2.3.4; Linux; Opera Mobi/build-1107180945; U; en-GB) Presto/2.8.149 Version/11.10" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (Linux; U; Android 3.0; en-us; Xoom Build/HRI39) AppleWebKit/534.13 (KHTML, like Gecko) Version/4.0 Safari/534.13" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (BlackBerry; U; BlackBerry 9800; en) AppleWebKit/534.1+ (KHTML, like Gecko) Version/6.0.0.337 Mobile Safari/534.1+" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (hp-tablet; Linux; hpwOS/3.0.0; U; en-US) AppleWebKit/534.6 (KHTML, like Gecko) wOSBrowser/233.70 Safari/534.6 TouchPad/1.0" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (SymbianOS/9.4; Series60/5.0 NokiaN97-1/20.0.019; Profile/MIDP-2.1 Configuration/CLDC-1.1) AppleWebKit/525 (KHTML, like Gecko) BrowserNG/7.1.18124" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/5.0 (compatible; MSIE 9.0; Windows Phone OS 7.5; Trident/5.0; IEMobile/9.0; HTC; Titan)" [OR]
RewriteCond %{HTTP_USER_AGENT} "=UCWEB7.0.2.37/28/999" [OR]
RewriteCond %{HTTP_USER_AGENT} "=NOKIA5700/ UCWEB7.0.2.37/28/999" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Openwave/ UCWEB7.0.2.37/28/999" [OR]
RewriteCond %{HTTP_USER_AGENT} "=Mozilla/4.0 (compatible; MSIE 6.0; ) Opera/UCWEB7.0.2.37/28/999"
RewriteRule . - [R=403,L]
</Macro>

9
playbooks/roles/apache-ua-filter/handlers/main.yaml

@ -0,0 +1,9 @@
- name: Reload apache2
service:
name: apache2
state: reloaded
- name: Restart apache2
service:
name: apache2
state: restarted

33
playbooks/roles/apache-ua-filter/tasks/main.yaml

@ -0,0 +1,33 @@
- name: Install apache2
apt:
name:
- apache2
- apache2-utils
state: present
- name: Apache rewrite module
apache2_module:
state: present
name: rewrite
notify: Restart apache2
- name: Apache macro module
apache2_module:
state: present
name: macro
notify: Restart apache2
- name: Apache headers module
apache2_module:
state: present
name: headers
notify: Restart apache2
- name: Copy apache UA filter
copy:
src: ua-filter.conf
dest: /etc/apache2/conf-enabled/ua-filter.conf
owner: root
group: root
mode: 0644
notify: Reload apache2

2
playbooks/roles/static/files/50-tarballs.opendev.org.conf

@ -268,6 +268,8 @@ Define AFS_ROOT /afs/openstack.org/project/tarballs.opendev.org
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
SSLHonorCipherOrder on
Use UserAgentFilter
<Directory ${AFS_ROOT}>
Options Indexes FollowSymLinks MultiViews
AllowOverrideList Redirect RedirectMatch

1
playbooks/service-static.yaml

@ -11,4 +11,5 @@
- role: openafs-client
openafs_client_cache_size: "{{ afs_client_cache_size | default(50000000) }}" # 50GiB
openafs_client_cache_directory: '/opt/cache/openafs'
- role: apache-ua-filter
- role: static

Loading…
Cancel
Save