6f09ed62e3
AFS across long distances is slow due to its small fixed window size not working well with large RTTs. Additionally performing a vos release (even if a noop and no files change) affects the AFS cache such that some data must be reretrieved. Mitigate these things by using a regular reverse caching proxy to pypi mirrors instead. This gives us normal tcp across long distances which can scale windows appropriately and don't need to reupdate items on every vos release. Instead cache updates are handled by http. Note that we point to our mirrors rather than pypi proper because we still want to be able to control the updates to our mirrors. It is possible that in some cases this is not necessary and we can point directly upstream. Change-Id: Ifb9f28d181fca74549c2107992227c4d08150271
189 lines
6.3 KiB
Plaintext
189 lines
6.3 KiB
Plaintext
# ************************************
|
|
# Managed by Puppet
|
|
# ************************************
|
|
|
|
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
|
|
|
# Dedicated port for proxy caching, as not to affect afs mirrors.
|
|
Listen 8080
|
|
NameVirtualHost <%= @vhost_name %>:8080
|
|
|
|
Listen 8081
|
|
NameVirtualHost <%= @vhost_name %>:8081
|
|
|
|
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
|
ServerName <%= @srvname %>
|
|
<% if @serveraliases.is_a? Array -%>
|
|
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
|
|
<% elsif @serveraliases != nil -%>
|
|
<%= " ServerAlias #{@serveraliases}" -%>
|
|
<% end -%>
|
|
DocumentRoot <%= @docroot %>
|
|
<Directory <%= @docroot %>>
|
|
Options <%= @options %>
|
|
AllowOverride None
|
|
Order allow,deny
|
|
allow from all
|
|
Satisfy any
|
|
<IfVersion >= 2.4>
|
|
Require all granted
|
|
</IfVersion>
|
|
</Directory>
|
|
|
|
RewriteEngine On
|
|
|
|
# Pypi's bandersnatch URL's are:
|
|
# /pypi/simple/index.html
|
|
# /pypi/simple/a/a/(index.html)?
|
|
# /pypi/simple/a/a/a-etc.whl
|
|
# /pypi/simple/a/abcd/(index.html)?
|
|
# /pypi/simple/a/abcd/abcd-etc.whl
|
|
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
|
|
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
|
|
|
|
# Wheel URL's are:
|
|
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
|
|
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
|
|
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
|
|
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
|
|
|
|
# npm's URL's are:
|
|
# /npm/-/index.json
|
|
# /npm/a/aabc/index.json
|
|
# /npm/a/aabc/latest/index.json
|
|
# /npm/a/aabc/-/aabc-0.0.0.tgz
|
|
RewriteCond %{REQUEST_URI} ^/npm/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/npm/$1/$1$2 -d
|
|
RewriteRule ^/npm/([^/])([^/]*)(/.*)?$ /npm/$1/$1$2$3 [L]
|
|
|
|
# TODO(jhesketh): Remove this after bandersnatch implements pep503
|
|
# https://bitbucket.org/pypa/bandersnatch/pull-requests/20/fully-implement-pep-503-normalization/diff
|
|
|
|
# Special cases for openstack.nose_plugin & backports.*
|
|
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
|
|
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
|
|
|
|
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
|
|
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
|
|
|
|
# Try again but replacing -'s with .'s
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
|
|
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
|
|
RewriteRule (.*)-(.*) $1.$2 [N]
|
|
|
|
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
|
|
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
|
|
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
|
|
|
|
<DirectoryMatch "<%= @docroot %>\/npm\/[^/]+\/.*">
|
|
DirectoryIndex index.json
|
|
AddOutputFilterByType SUBSTITUTE application/json
|
|
Substitute "s|http://localhost|http://<%= @srvname %>/npm|ni"
|
|
</DirectoryMatch>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined
|
|
ServerSignature Off
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8080>
|
|
ServerName <%= @srvname %>:8080
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 3
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
|
|
# Per site caching reverse proxy rules
|
|
# Only cache specific backends, rely on afs cache otherwise.
|
|
|
|
# rdo
|
|
CacheEnable disk "/rdo"
|
|
ProxyPass "/rdo/" "https://trunk.rdoproject.org/"
|
|
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
|
|
|
|
# tarballs
|
|
CacheEnable disk "/tarballs"
|
|
ProxyPass "/tarballs/" "https://tarballs.openstack.org/"
|
|
ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
|
|
|
|
# pypi
|
|
CacheEnable disk "/pypi"
|
|
ProxyPass "/pypi/" "http://mirror.dfw.rax.openstack.org/pypi/"
|
|
ProxyPassReverse "/pypi/" "http://mirror.dfw.rax.openstack.org/pypi/"
|
|
</VirtualHost>
|
|
|
|
<VirtualHost <%= @vhost_name %>:8081>
|
|
ServerName <%= @srvname %>:8081
|
|
|
|
# Disable directory listing by default.
|
|
<Directory />
|
|
Order Deny,Allow
|
|
Deny from all
|
|
Options None
|
|
AllowOverride None
|
|
</Directory>
|
|
|
|
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
|
|
LogLevel warn
|
|
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined
|
|
ServerSignature Off
|
|
|
|
# Caching reverse proxy for things that don't make sense in AFS
|
|
#
|
|
# General cache rules
|
|
CacheRoot "/var/cache/apache2/proxy"
|
|
CacheDirLevels 5
|
|
CacheDirLength 3
|
|
# SSL support
|
|
SSLProxyEngine on
|
|
# Prevent thundering herds.
|
|
CacheLock on
|
|
CacheLockPath "/tmp/mod_cache-lock"
|
|
CacheLockMaxAge 5
|
|
# 5GiB
|
|
CacheMaxFileSize 5368709120
|
|
# Ignore expire headers as the urls use sha256 hashes.
|
|
CacheIgnoreQueryString On
|
|
CacheStoreExpired On
|
|
|
|
# registry-1.docker.io
|
|
CacheEnable disk "/registry-1.docker"
|
|
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/"
|
|
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
|
|
|
|
# dseasb33srnrn.cloudfront.net
|
|
CacheEnable disk "/cloudfront"
|
|
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
|
|
|
</VirtualHost>
|