Apply UserAgentFilter to every vhost on static

AI crawlers continue to get more and more aggressive. To ensure that there are more webserver slots for actual users apply the user agent filter to every vhost on static.o.o. Note that at least one vhost already used the ua filter so we don't need to update config management beyond the vhost configs. The macro is already installed on static. Change-Id: I7c377d51f0a89272fd6fadbecc3d7923bba3cfd7
2025-07-22 10:02:55 -07:00
parent 17d92931a1
commit 25b08c09a8
29 changed files with 66 additions and 1 deletions
--- a/playbooks/roles/static/files/00-static.opendev.org.conf
+++ b/playbooks/roles/static/files/00-static.opendev.org.conf
@@ -4,6 +4,7 @@ Define AFS_ROOT /afs/openstack.org/
  ServerName static.opendev.org
  ServerAlias static.openstack.org files.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://static.opendev.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/static.opendev.org_error.log
@@ -19,6 +20,8 @@ Define AFS_ROOT /afs/openstack.org/

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/static.opendev.org/static.opendev.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/static.opendev.org/static.opendev.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/static.opendev.org/ca.cer
--- a/playbooks/roles/static/files/50-api.openstack.org.conf
+++ b/playbooks/roles/static/files/50-api.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName api.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://developer.openstack.org/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://developer.openstack.org/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-ask.openstack.org.conf
+++ b/playbooks/roles/static/files/50-ask.openstack.org.conf
@@ -1,6 +1,7 @@
 <VirtualHost *:80>
  ServerName ask.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://ask.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/ask.openstack.org_error.log
@@ -15,6 +16,8 @@

  DocumentRoot /var/www/ask.openstack.org

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/ask.openstack.org/ask.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/ask.openstack.org/ask.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/ask.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-ci.openstack.org.conf
+++ b/playbooks/roles/static/files/50-ci.openstack.org.conf
@@ -1,5 +1,6 @@
 <Macro CiRedirects>
  RewriteEngine On
+  Use UserAgentFilter

  RewriteRule ^/jenkins-job-builder(/.*|$) https://docs.openstack.org/infra/jenkins-job-builder$1 [last,redirect=permanent]
  RewriteRule ^/nodepool(/.*|$) https://docs.openstack.org/infra/nodepool$1 [last,redirect=permanent]
--- a/playbooks/roles/static/files/50-cinder.openstack.org.conf
+++ b/playbooks/roles/static/files/50-cinder.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName cinder.openstack.org

 RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/cinder/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/cinder/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-developer.openstack.org.conf
+++ b/playbooks/roles/static/files/50-developer.openstack.org.conf
@@ -2,6 +2,7 @@
    ServerName developer.openstack.org

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://developer.openstack.org/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/developer.openstack.org_error.log
@@ -15,6 +16,7 @@
    ServerName developer.openstack.org

    RewriteEngine on
+    Use UserAgentFilter

  SSLCertificateFile      /etc/letsencrypt-certs/developer.openstack.org/developer.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/developer.openstack.org/developer.openstack.org.key
--- a/playbooks/roles/static/files/50-devstack.org.conf
+++ b/playbooks/roles/static/files/50-devstack.org.conf
@@ -3,6 +3,7 @@
  ServerAlias *.devstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/devstack/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -24,6 +25,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/devstack/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-docs.airshipit.org.conf
+++ b/playbooks/roles/static/files/50-docs.airshipit.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/airshipit.org
 <VirtualHost *:80>
  ServerName docs.airshipit.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.airshipit.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/docs.airshipit.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/airshipit.org

  DocumentRoot ${AFS_ROOT}/docs

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/docs.airshipit.org/docs.airshipit.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/docs.airshipit.org/docs.airshipit.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/docs.airshipit.org/ca.cer
--- a/playbooks/roles/static/files/50-docs.opendev.org.conf
+++ b/playbooks/roles/static/files/50-docs.opendev.org.conf
@@ -2,6 +2,7 @@
    ServerName docs.opendev.org

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://docs.opendev.org/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/docs.opendev.org_error.log
@@ -15,6 +16,7 @@
    ServerName docs.opendev.org

    RewriteEngine on
+    Use UserAgentFilter

  SSLCertificateFile      /etc/letsencrypt-certs/docs.opendev.org/docs.opendev.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/docs.opendev.org/docs.opendev.org.key
--- a/playbooks/roles/static/files/50-docs.openstack.org.conf
+++ b/playbooks/roles/static/files/50-docs.openstack.org.conf
@@ -2,6 +2,7 @@
    ServerName docs.openstack.org

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://docs.openstack.org/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/docs.openstack.org_error.log
@@ -15,6 +16,7 @@
    ServerName docs.openstack.org

    RewriteEngine on
+    Use UserAgentFilter

  SSLCertificateFile      /etc/letsencrypt-certs/docs.openstack.org/docs.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/docs.openstack.org/docs.openstack.org.key
--- a/playbooks/roles/static/files/50-docs.starlingx.io.conf
+++ b/playbooks/roles/static/files/50-docs.starlingx.io.conf
@@ -2,6 +2,7 @@
    ServerName docs.starlingx.io

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://docs.starlingx.io/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/docs.starlingx.io_error.log
@@ -15,6 +16,7 @@
    ServerName docs.starlingx.io

    RewriteEngine on
+    Use UserAgentFilter

  SSLCertificateFile      /etc/letsencrypt-certs/docs.starlingx.io/docs.starlingx.io.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/docs.starlingx.io/docs.starlingx.io.key
--- a/playbooks/roles/static/files/50-eavesdrop.openstack.org.conf
+++ b/playbooks/roles/static/files/50-eavesdrop.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName eavesdrop.openstack.org

  RewriteEngine On
+  Use UserAgentFilter

  RewriteRule ^/(.*) https://meetings.opendev.org/$1 [last,redirect=permanent]

@@ -23,6 +24,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter

  RewriteRule ^/(.*) https://meetings.opendev.org/$1 [last,redirect=permanent]

--- a/playbooks/roles/static/files/50-gating.dev.conf
+++ b/playbooks/roles/static/files/50-gating.dev.conf
@@ -3,6 +3,7 @@
    ServerAlias www.gating.dev

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://gating.dev/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/gating.dev_error.log
@@ -17,6 +18,7 @@
    ServerAlias www.gating.dev

    RewriteEngine on
+    Use UserAgentFilter

    RewriteCond %{HTTP_HOST} !^gating\.dev [nocase]                               
    RewriteRule ^/(.*) https://gating.dev/$1 [last,redirect=permanent]            
--- a/playbooks/roles/static/files/50-glance.openstack.org.conf
+++ b/playbooks/roles/static/files/50-glance.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName glance.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/glance/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/glance/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-governance.openstack.org.conf
+++ b/playbooks/roles/static/files/50-governance.openstack.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/governance.openstack.org
 <VirtualHost *:80>
  ServerName governance.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://governance.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/governance.openstack.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/governance.openstack.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/governance.openstack.org/governance.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/governance.openstack.org/governance.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/governance.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-horizon.openstack.org.conf
+++ b/playbooks/roles/static/files/50-horizon.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName horizon.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/horizon/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/horizon/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-keystone.openstack.org.conf
+++ b/playbooks/roles/static/files/50-keystone.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName keystone.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/keystone/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/keystone/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-meetings.opendev.org.conf
+++ b/playbooks/roles/static/files/50-meetings.opendev.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/meetings.opendev.org
 <VirtualHost *:80>
  ServerName meetings.opendev.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://meetings.opendev.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/meetings.opendev.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/meetings.opendev.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/meetings.opendev.org/meetings.opendev.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/meetings.opendev.org/meetings.opendev.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/meetings.opendev.org/ca.cer
--- a/playbooks/roles/static/files/50-nova.openstack.org.conf
+++ b/playbooks/roles/static/files/50-nova.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName nova.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/nova/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/nova/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-planet.openstack.org.conf
+++ b/playbooks/roles/static/files/50-planet.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName planet.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://opendev.org/openstack/openstack-planet [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://opendev.org/openstack/openstack-planet [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-releases.openstack.org.conf
+++ b/playbooks/roles/static/files/50-releases.openstack.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/releases.openstack.org
 <VirtualHost *:80>
  ServerName releases.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://releases.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/releases.openstack.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/releases.openstack.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/releases.openstack.org/releases.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/releases.openstack.org/releases.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/releases.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-security.openstack.org.conf
+++ b/playbooks/roles/static/files/50-security.openstack.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/security.openstack.org
 <VirtualHost *:80>
  ServerName security.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://security.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/security.openstack.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/security.openstack.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/security.openstack.org/security.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/security.openstack.org/security.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/security.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-service-types.openstack.org.conf
+++ b/playbooks/roles/static/files/50-service-types.openstack.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/service-types.openstack.org
 <VirtualHost *:80>
  ServerName service-types.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://service-types.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/service-types.openstack.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/service-types.openstack.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/service-types.openstack.org/service-types.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/service-types.openstack.org/service-types.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/service-types.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-specs.openstack.org.conf
+++ b/playbooks/roles/static/files/50-specs.openstack.org.conf
@@ -3,6 +3,7 @@ Define AFS_ROOT /afs/openstack.org/project/specs.openstack.org
 <VirtualHost *:80>
  ServerName specs.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://specs.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/specs.openstack.org_error.log
@@ -17,6 +18,8 @@ Define AFS_ROOT /afs/openstack.org/project/specs.openstack.org

  DocumentRoot ${AFS_ROOT}

+  Use UserAgentFilter
+
  SSLCertificateFile      /etc/letsencrypt-certs/specs.openstack.org/specs.openstack.org.cer
  SSLCertificateKeyFile   /etc/letsencrypt-certs/specs.openstack.org/specs.openstack.org.key
  SSLCertificateChainFile /etc/letsencrypt-certs/specs.openstack.org/ca.cer
--- a/playbooks/roles/static/files/50-summit.openstack.org.conf
+++ b/playbooks/roles/static/files/50-summit.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName summit.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://openstack.org/summit/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://openstack.org/summit/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-swift.openstack.org.conf
+++ b/playbooks/roles/static/files/50-swift.openstack.org.conf
@@ -2,6 +2,7 @@
  ServerName swift.openstack.org

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/swift/latest/$1 [last,redirect=permanent]

  LogLevel warn
@@ -22,6 +23,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://docs.openstack.org/swift/latest/$1 [last,redirect=permanent]

  LogLevel warn
--- a/playbooks/roles/static/files/50-tarballs.opendev.org.conf
+++ b/playbooks/roles/static/files/50-tarballs.opendev.org.conf
@@ -245,6 +245,7 @@ Define AFS_ROOT /afs/openstack.org/project/tarballs.opendev.org
 <VirtualHost *:80>
  ServerName tarballs.opendev.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://tarballs.opendev.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/tarballs.opendev.org_error.log
--- a/playbooks/roles/static/files/50-tarballs.openstack.org.conf
+++ b/playbooks/roles/static/files/50-tarballs.openstack.org.conf
@@ -1,6 +1,7 @@
 <VirtualHost *:80>
  ServerName tarballs.openstack.org
  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(.*) https://tarballs.openstack.org/$1 [last,redirect=permanent]
  LogLevel warn
  ErrorLog /var/log/apache2/tarballs.openstack.org_error.log
@@ -24,6 +25,7 @@
  SSLHonorCipherOrder on

  RewriteEngine On
+  Use UserAgentFilter
  RewriteRule ^/(openstackid(/.*|$)) https://tarballs.opendev.org/osf/$1 [L]
  RewriteRule ^/?(.*)$ https://tarballs.opendev.org/openstack/$1 [L]

--- a/playbooks/roles/static/files/50-zuul-ci.org.conf
+++ b/playbooks/roles/static/files/50-zuul-ci.org.conf
@@ -5,6 +5,7 @@
    ServerAlias www.zuulci.org

    RewriteEngine on
+    Use UserAgentFilter
    RewriteRule ^/(.*) https://zuul-ci.org/$1 [last,redirect=permanent]

    ErrorLog /var/log/apache2/zuul-ci.org_error.log
@@ -21,6 +22,7 @@
    ServerAlias www.zuulci.org

    RewriteEngine on
+    Use UserAgentFilter

    SSLEngine on
    SSLProtocol All -SSLv2 -SSLv3