system-config/modules/openstack_project/templates/mirror.vhost.erb
Clark Boylan 6f09ed62e3 Add caching reverse proxy for pypi mirror
AFS across long distances is slow due to its small fixed window size not
working well with large RTTs. Additionally performing a vos release
(even if a noop and no files change) affects the AFS cache such that
some data must be reretrieved.

Mitigate these things by using a regular reverse caching proxy to pypi
mirrors instead. This gives us normal tcp across long distances which
can scale windows appropriately and don't need to reupdate items on
every vos release. Instead cache updates are handled by http.

Note that we point to our mirrors rather than pypi proper because we
still want to be able to control the updates to our mirrors. It is
possible that in some cases this is not necessary and we can point
directly upstream.

Change-Id: Ifb9f28d181fca74549c2107992227c4d08150271
2017-07-13 12:36:14 -07:00

189 lines
6.3 KiB
Plaintext

# ************************************
# Managed by Puppet
# ************************************
NameVirtualHost <%= @vhost_name %>:<%= @port %>
# Dedicated port for proxy caching, as not to affect afs mirrors.
Listen 8080
NameVirtualHost <%= @vhost_name %>:8080
Listen 8081
NameVirtualHost <%= @vhost_name %>:8081
<VirtualHost <%= @vhost_name %>:<%= @port %>>
ServerName <%= @srvname %>
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%><%= " ServerAlias #{name}\n" %><% end -%>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}" -%>
<% end -%>
DocumentRoot <%= @docroot %>
<Directory <%= @docroot %>>
Options <%= @options %>
AllowOverride None
Order allow,deny
allow from all
Satisfy any
<IfVersion >= 2.4>
Require all granted
</IfVersion>
</Directory>
RewriteEngine On
# Pypi's bandersnatch URL's are:
# /pypi/simple/index.html
# /pypi/simple/a/a/(index.html)?
# /pypi/simple/a/a/a-etc.whl
# /pypi/simple/a/abcd/(index.html)?
# /pypi/simple/a/abcd/abcd-etc.whl
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
# Wheel URL's are:
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
# npm's URL's are:
# /npm/-/index.json
# /npm/a/aabc/index.json
# /npm/a/aabc/latest/index.json
# /npm/a/aabc/-/aabc-0.0.0.tgz
RewriteCond %{REQUEST_URI} ^/npm/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/npm/$1/$1$2 -d
RewriteRule ^/npm/([^/])([^/]*)(/.*)?$ /npm/$1/$1$2$3 [L]
# TODO(jhesketh): Remove this after bandersnatch implements pep503
# https://bitbucket.org/pypa/bandersnatch/pull-requests/20/fully-implement-pep-503-normalization/diff
# Special cases for openstack.nose_plugin & backports.*
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
# Try again but replacing -'s with .'s
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
RewriteRule (.*)-(.*) $1.$2 [N]
RewriteCond %{REQUEST_URI} ^/pypi/simple/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/pypi/simple/$1/$1$2 -d
RewriteRule ^/pypi/simple/([^/])([^/]*)(/.*)?$ /pypi/simple/$1/$1$2$3 [L]
<DirectoryMatch "<%= @docroot %>\/npm\/[^/]+\/.*">
DirectoryIndex index.json
AddOutputFilterByType SUBSTITUTE application/json
Substitute "s|http://localhost|http://<%= @srvname %>/npm|ni"
</DirectoryMatch>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined
ServerSignature Off
</VirtualHost>
<VirtualHost <%= @vhost_name %>:8080>
ServerName <%= @srvname %>:8080
# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined
ServerSignature Off
# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 3
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Per site caching reverse proxy rules
# Only cache specific backends, rely on afs cache otherwise.
# rdo
CacheEnable disk "/rdo"
ProxyPass "/rdo/" "https://trunk.rdoproject.org/"
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
# tarballs
CacheEnable disk "/tarballs"
ProxyPass "/tarballs/" "https://tarballs.openstack.org/"
ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
# pypi
CacheEnable disk "/pypi"
ProxyPass "/pypi/" "http://mirror.dfw.rax.openstack.org/pypi/"
ProxyPassReverse "/pypi/" "http://mirror.dfw.rax.openstack.org/pypi/"
</VirtualHost>
<VirtualHost <%= @vhost_name %>:8081>
ServerName <%= @srvname %>:8081
# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined
ServerSignature Off
# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 3
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Ignore expire headers as the urls use sha256 hashes.
CacheIgnoreQueryString On
CacheStoreExpired On
# registry-1.docker.io
CacheEnable disk "/registry-1.docker"
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/"
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
# dseasb33srnrn.cloudfront.net
CacheEnable disk "/cloudfront"
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
</VirtualHost>