Remove puppet mirror support
Remove the separate "mirror_opendev" group and rename it to just "mirror". Update various parts to reflect that change. We no longer deploy any mirror hosts with puppet, remove the various configuration files. Depends-On: https://review.opendev.org/728345 Change-Id: Ia982fe9cb4357447989664f033df976b528aaf84
This commit is contained in:
Ian Wienand
parent
6acc2dca98
commit
45201f3d66
@ -4,7 +4,7 @@ groups:
|
||||
afs: afs[0-9]*.open*.org
|
||||
afs-client:
|
||||
- review-dev[0-9]*.open*.org
|
||||
- mirror[0-9]*.open*.org
|
||||
- mirror[0-9]*.opendev.org
|
||||
- mirror-update[0-9]*.opendev.org
|
||||
- ze[0-9]*.open*.org
|
||||
- afsdb*.open*.org
|
||||
@ -40,7 +40,6 @@ groups:
|
||||
- ethercalc01.openstack.org
|
||||
- idp.openstackid.org
|
||||
- lists-dev01.openstack.org
|
||||
- mirror01.sto2.citycloud.openstack.org
|
||||
- wiki-dev01.openstack.org
|
||||
- wiki-upgrade-test.openstack.org
|
||||
dns:
|
||||
@ -90,8 +89,6 @@ groups:
|
||||
meetpad:
|
||||
- meetpad[0-9]*.opendev.org
|
||||
mirror:
|
||||
- mirror[0-9]*.openstack.org
|
||||
mirror_opendev:
|
||||
- mirror[0-9]*.opendev.org
|
||||
mirror-update:
|
||||
- mirror-update[0-9]*.opendev.org
|
||||
@ -136,7 +133,6 @@ groups:
|
||||
- logstash-worker[0-9]*.open*.org
|
||||
- logstash[0-9]*.open*.org
|
||||
- mirror-update[0-9]*.openstack.org
|
||||
- mirror[0-9]*.openstack.org
|
||||
- nb[0-9]*.openstack.org
|
||||
- openstackid-dev*.openstack.org
|
||||
- openstackid.org
|
||||
@ -175,7 +171,6 @@ groups:
|
||||
- logstash-worker[0-9]*.open*.org
|
||||
- logstash[0-9]*.open*.org
|
||||
- mirror-update[0-9]*.openstack.org
|
||||
- ^mirror[0-9].*\..*\.(?!linaro|linaro-london|linaro-us).*\.openstack\.org
|
||||
- ^nb(?!03)[0-9]*\.openstack\.org
|
||||
- openstackid[0-9]*.openstack.org
|
||||
- openstackid-dev[0-9]*.openstack.org
|
||||
|
||||
@ -246,22 +246,6 @@ node /^mirror-update\d*\.open.*\.org$/ {
|
||||
}
|
||||
}
|
||||
|
||||
# Machines in each region to serve AFS mirrors.
|
||||
# Node-OS: xenial
|
||||
node /^mirror\d*\..*\.open.*\.org$/ {
|
||||
$group = "mirror"
|
||||
|
||||
class { 'openstack_project::server':
|
||||
afs => true,
|
||||
afs_cache_size => 50000000, # 50GB
|
||||
}
|
||||
|
||||
class { 'openstack_project::mirror':
|
||||
vhost_name => $::fqdn,
|
||||
require => Class['Openstack_project::Server'],
|
||||
}
|
||||
}
|
||||
|
||||
# Node-OS: trusty
|
||||
# Node-OS: xenial
|
||||
node /^refstack\d*\.open.*\.org$/ {
|
||||
|
||||
@ -1,378 +0,0 @@
|
||||
# == Class: openstack_project::mirror
|
||||
#
|
||||
class openstack_project::mirror (
|
||||
$vhost_name = $::fqdn,
|
||||
) {
|
||||
|
||||
# Some hosts are mirror01, but we need the host to respond to
|
||||
# "mirror." Re-evaluate this if we end up doing multiple
|
||||
# mirrors/load balancing etc.
|
||||
$alias_name = regsubst($vhost_name, 'mirror\d*\.', 'mirror.')
|
||||
if $alias_name != $vhost_name {
|
||||
$serveraliases = [$alias_name]
|
||||
} else {
|
||||
$serveraliases = undef
|
||||
}
|
||||
|
||||
$mirror_root = '/afs/openstack.org/mirror'
|
||||
$wheel_root = "${mirror_root}/wheel"
|
||||
$ceph_deb_hammer_root = "${mirror_root}/ceph-deb-hammer"
|
||||
$ceph_deb_jewel_root = "${mirror_root}/ceph-deb-jewel"
|
||||
$ceph_deb_luminous_root = "${mirror_root}/ceph-deb-luminous"
|
||||
$ceph_deb_mimic_root = "${mirror_root}/ceph-deb-mimic"
|
||||
$ceph_deb_nautilus_root = "${mirror_root}/ceph-deb-nautilus"
|
||||
$gem_root = "${mirror_root}/gem"
|
||||
|
||||
$www_base = '/var/www'
|
||||
$www_root = "${www_base}/mirror"
|
||||
|
||||
#####################################################
|
||||
# Build Apache Webroot
|
||||
file { "${www_base}":
|
||||
ensure => directory,
|
||||
owner => root,
|
||||
group => root,
|
||||
}
|
||||
|
||||
file { "${www_root}":
|
||||
ensure => directory,
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_base}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to pypi.
|
||||
# NOTE(ianw) : retired 2018-11-22; see
|
||||
# https://review.opendev.org/#/c/618326/
|
||||
file { "${www_root}/pypi":
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
# Create the symlink to wheel.
|
||||
file { "${www_root}/wheel":
|
||||
ensure => link,
|
||||
target => "${wheel_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to centos
|
||||
file { "${www_root}/centos":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/centos",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to debian
|
||||
file { "${www_root}/debian":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/debian",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to debian-security
|
||||
file { "${www_root}/debian-security":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/debian-security",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to Debian OpenStack Packaging Team reprepro.
|
||||
file { "${www_root}/debian-openstack":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/debian-openstack",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to rdo
|
||||
file { "${www_root}/rdo":
|
||||
ensure => absent,
|
||||
}
|
||||
|
||||
# Create the symlink to epel
|
||||
file { "${www_root}/epel":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/epel",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to yum-puppetlabs
|
||||
file { "${www_root}/yum-puppetlabs":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/yum-puppetlabs",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to fedora
|
||||
file { "${www_root}/fedora":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/fedora",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to openSUSE
|
||||
file { "${www_root}/opensuse":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/opensuse",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to Ubuntu
|
||||
file { "${www_root}/ubuntu":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/ubuntu",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to Ubuntu ports
|
||||
file { "${www_root}/ubuntu-ports":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/ubuntu-ports",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to ceph-deb-hammer.
|
||||
file { "${www_root}/ceph-deb-hammer":
|
||||
ensure => link,
|
||||
target => "${ceph_deb_hammer_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to ceph-deb-jewel.
|
||||
file { "${www_root}/ceph-deb-jewel":
|
||||
ensure => link,
|
||||
target => "${ceph_deb_jewel_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to ceph-deb-luminous.
|
||||
file { "${www_root}/ceph-deb-luminous":
|
||||
ensure => link,
|
||||
target => "${ceph_deb_luminous_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to ceph-deb-mimic.
|
||||
file { "${www_root}/ceph-deb-mimic":
|
||||
ensure => link,
|
||||
target => "${ceph_deb_mimic_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to ceph-deb-nautilus.
|
||||
file { "${www_root}/ceph-deb-nautilus":
|
||||
ensure => link,
|
||||
target => "${ceph_deb_nautilus_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to Ubuntu Cloud Archive.
|
||||
file { "${www_root}/ubuntu-cloud-archive":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/ubuntu-cloud-archive",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to deb-docker.
|
||||
file { "${www_root}/deb-docker":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/deb-docker",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Create the symlink to Ubuntu Puppetlabs.
|
||||
file { "${www_root}/apt-puppetlabs":
|
||||
ensure => link,
|
||||
target => "${mirror_root}/apt-puppetlabs",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
file { "${www_root}/gem":
|
||||
ensure => link,
|
||||
target => "${gem_root}",
|
||||
owner => root,
|
||||
group => root,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
file { "${www_root}/robots.txt":
|
||||
ensure => present,
|
||||
owner => 'root',
|
||||
group => 'root',
|
||||
mode => '0444',
|
||||
source => 'puppet:///modules/openstack_project/disallow_robots.txt',
|
||||
require => File["${www_root}"],
|
||||
}
|
||||
|
||||
#####################################################
|
||||
# Build VHost
|
||||
include ::httpd
|
||||
|
||||
file { '/opt/apache_cache':
|
||||
ensure => absent,
|
||||
force => true,
|
||||
}
|
||||
|
||||
file { '/var/cache/apache2/proxy':
|
||||
ensure => directory,
|
||||
owner => 'www-data',
|
||||
group => 'www-data',
|
||||
mode => '0755',
|
||||
require => Class['httpd']
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['rewrite']) {
|
||||
httpd::mod { 'rewrite':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['substitute']) {
|
||||
httpd::mod { 'substitute':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['cache']) {
|
||||
httpd::mod { 'cache':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['cache_disk']) {
|
||||
httpd::mod { 'cache_disk':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['proxy']) {
|
||||
httpd::mod { 'proxy':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
if ! defined(Httpd::Mod['proxy_http']) {
|
||||
httpd::mod { 'proxy_http':
|
||||
ensure => present,
|
||||
}
|
||||
}
|
||||
|
||||
::httpd::vhost { $vhost_name:
|
||||
port => 80,
|
||||
priority => '50',
|
||||
docroot => "${www_root}",
|
||||
template => 'openstack_project/mirror.vhost.erb',
|
||||
serveraliases => $serveraliases,
|
||||
require => [
|
||||
File["${www_root}"],
|
||||
]
|
||||
}
|
||||
|
||||
# Cache cleanup
|
||||
package { 'apache2-utils':
|
||||
ensure => present,
|
||||
}
|
||||
|
||||
cron { 'apache-cache-cleanup':
|
||||
# Clean apache cache once an hour, keep size down to 70GiB.
|
||||
minute => '0',
|
||||
hour => '*',
|
||||
command => 'flock -n /var/run/htcacheclean.lock htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null',
|
||||
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
|
||||
require => [
|
||||
File['/var/cache/apache2/proxy'],
|
||||
Package['apache2-utils'],
|
||||
],
|
||||
}
|
||||
|
||||
class { '::httpd::logrotate':
|
||||
options => [
|
||||
'daily',
|
||||
'missingok',
|
||||
'rotate 7',
|
||||
'compress',
|
||||
'delaycompress',
|
||||
'notifempty',
|
||||
'create 640 root adm',
|
||||
],
|
||||
}
|
||||
}
|
||||
@ -1,525 +0,0 @@
|
||||
# ************************************
|
||||
# Managed by Puppet
|
||||
# ************************************
|
||||
|
||||
NameVirtualHost <%= @vhost_name %>:<%= @port %>
|
||||
|
||||
# Dedicated port for proxy caching, as not to affect afs mirrors.
|
||||
Listen 8080
|
||||
NameVirtualHost <%= @vhost_name %>:8080
|
||||
|
||||
Listen 8081
|
||||
NameVirtualHost <%= @vhost_name %>:8081
|
||||
|
||||
Listen 8082
|
||||
NameVirtualHost <%= @vhost_name %>:8082
|
||||
|
||||
Listen 8083
|
||||
NameVirtualHost <%= @vhost_name %>:8083
|
||||
|
||||
LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
|
||||
|
||||
<VirtualHost <%= @vhost_name %>:<%= @port %>>
|
||||
ServerName <%= @srvname %>
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}" -%>
|
||||
<% end %>
|
||||
DocumentRoot <%= @docroot %>
|
||||
<Directory <%= @docroot %>>
|
||||
Options <%= @options %>
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
Satisfy any
|
||||
<IfVersion >= 2.4>
|
||||
Require all granted
|
||||
</IfVersion>
|
||||
</Directory>
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
CacheStoreExpired On
|
||||
# Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
|
||||
# This means we don't use the cache for those requests. This setting
|
||||
# should force the proxy to ignore cache-control on the request side
|
||||
# but we should still cache things based on the cache-control responses
|
||||
# from the backed servers.
|
||||
CacheIgnoreCacheControl On
|
||||
|
||||
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
||||
# 0.05% of requests) of the type:
|
||||
#
|
||||
# End of file found: ... AH01102: error reading status line from remote server ...
|
||||
#
|
||||
# Per [1]:
|
||||
#
|
||||
# This avoids the "proxy: error reading status line from remote
|
||||
# server" error message caused by the race condition that the backend
|
||||
# server closed the pooled connection after the connection check by the
|
||||
# proxy and before data sent by the proxy reached the backend.
|
||||
#
|
||||
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
||||
SetEnv proxy-initial-not-pooled 1
|
||||
|
||||
RewriteEngine On
|
||||
# pypi
|
||||
CacheEnable disk "/pypi"
|
||||
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypi/" "https://pypi.org/
|
||||
|
||||
# files.pythonhosted.org
|
||||
CacheEnable disk "/pypifiles"
|
||||
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
||||
|
||||
# Rewrite the locations of the actual files
|
||||
<Location /pypi>
|
||||
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
||||
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
||||
</Location>
|
||||
|
||||
# Wheel URL's are:
|
||||
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
|
||||
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
|
||||
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
|
||||
|
||||
# Special cases for openstack.nose_plugin & backports.*
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
|
||||
|
||||
# Try again but replacing -'s with .'s
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
|
||||
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
|
||||
RewriteRule (.*)-(.*) $1.$2 [N]
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache
|
||||
ServerSignature Off
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost <%= @vhost_name %>:8080>
|
||||
ServerName <%= @srvname %>:8080
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}:8080\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}:8080" -%>
|
||||
<% end %>
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Let upstreams decide on encoded slash handling.
|
||||
# The default is 'Off' which returns 404 for URLs with encoded slashes,
|
||||
# i.e. '%2f' instead of '/'.
|
||||
AllowEncodedSlashes NoDecode
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
CacheStoreExpired On
|
||||
|
||||
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
||||
# 0.05% of requests) of the type:
|
||||
#
|
||||
# End of file found: ... AH01102: error reading status line from remote server ...
|
||||
#
|
||||
# Per [1]:
|
||||
#
|
||||
# This avoids the "proxy: error reading status line from remote
|
||||
# server" error message caused by the race condition that the backend
|
||||
# server closed the pooled connection after the connection check by the
|
||||
# proxy and before data sent by the proxy reached the backend.
|
||||
#
|
||||
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
||||
SetEnv proxy-initial-not-pooled 1
|
||||
|
||||
# Per site caching reverse proxy rules
|
||||
# Only cache specific backends, rely on afs cache otherwise.
|
||||
|
||||
# buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
|
||||
CacheEnable disk "/buildlogs.centos"
|
||||
ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
|
||||
ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
|
||||
|
||||
# buildlogs.cdn.centos.org
|
||||
CacheEnable disk "/buildlogs.cdn.centos"
|
||||
ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
|
||||
ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
|
||||
|
||||
# rdo
|
||||
CacheEnable disk "/rdo"
|
||||
ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
|
||||
|
||||
# cbs.centos.org
|
||||
CacheEnable disk "/cbs.centos"
|
||||
ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"
|
||||
|
||||
# pypi
|
||||
CacheEnable disk "/pypi"
|
||||
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypi/" "https://pypi.org/
|
||||
|
||||
# files.pythonhosted.org
|
||||
CacheEnable disk "/pypifiles"
|
||||
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
||||
|
||||
# Rewrite the locations of the actual files
|
||||
<Location /pypi>
|
||||
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
||||
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
||||
</Location>
|
||||
|
||||
# images.linuxcontainers.org
|
||||
CacheEnable disk "/images.linuxcontainers"
|
||||
ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
|
||||
|
||||
# registry.npmjs.org
|
||||
CacheEnable disk "/registry.npmjs"
|
||||
ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0 nocanon
|
||||
ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
|
||||
|
||||
# api.rubygems.org
|
||||
CacheEnable disk "/api.rubygems"
|
||||
ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
|
||||
|
||||
# rubygems.org
|
||||
CacheEnable disk "/rubygems"
|
||||
ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/rubygems/" "https://rubygems.org/"
|
||||
|
||||
# opendaylight
|
||||
CacheEnable disk "/opendaylight"
|
||||
ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
|
||||
|
||||
# elastico
|
||||
CacheEnable disk "/elastic"
|
||||
ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
|
||||
|
||||
# grafana
|
||||
CacheEnable disk "/grafana"
|
||||
ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
|
||||
|
||||
# OracleLinux
|
||||
CacheEnable disk "/oraclelinux"
|
||||
ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
|
||||
|
||||
# Percona
|
||||
CacheEnable disk "/percona"
|
||||
ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/percona/" "https://repo.percona.com/"
|
||||
|
||||
# MariaDB
|
||||
CacheEnable disk "/MariaDB"
|
||||
ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
|
||||
|
||||
# Docker
|
||||
CacheEnable disk "/docker"
|
||||
ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
|
||||
|
||||
# Alpine
|
||||
CacheEnable disk "/alpine"
|
||||
ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"
|
||||
|
||||
# LXC (copr)
|
||||
CacheEnable disk "/copr-lxc2"
|
||||
ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
|
||||
|
||||
</VirtualHost>
|
||||
|
||||
# Docker registry v1 proxy.
|
||||
<VirtualHost <%= @vhost_name %>:8081>
|
||||
ServerName <%= @srvname %>:8081
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}:8081\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}:8081" -%>
|
||||
<% end %>
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
||||
# last-modified date are provided default expire to 1 day. This is up from
|
||||
# 1 hour.
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# registry-1.docker.io
|
||||
CacheEnable disk "/registry-1.docker"
|
||||
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
|
||||
|
||||
# dseasb33srnrn.cloudfront.net
|
||||
CacheEnable disk "/cloudfront"
|
||||
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
||||
|
||||
# production.cloudflare.docker.com
|
||||
CacheEnable disk "/cloudflare"
|
||||
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
||||
|
||||
</VirtualHost>
|
||||
|
||||
# Docker registry v2 proxy.
|
||||
<VirtualHost <%= @vhost_name %>:8082>
|
||||
ServerName <%= @srvname %>:8082
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}:8082\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}:8082" -%>
|
||||
<% end %>
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
||||
# last-modified date are provided default expire to 1 day. This is up from
|
||||
# 1 hour.
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# dseasb33srnrn.cloudfront.net
|
||||
CacheEnable disk "/cloudfront"
|
||||
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
||||
|
||||
# production.cloudflare.docker.com
|
||||
CacheEnable disk "/cloudflare"
|
||||
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
||||
|
||||
# NOTE(corvus): Ensure this stanza is last since it's the most
|
||||
# greedy match.
|
||||
CacheEnable disk "/"
|
||||
ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/" "https://registry-1.docker.io/"
|
||||
</VirtualHost>
|
||||
|
||||
# Redhat registry proxy.
|
||||
<VirtualHost <%= @vhost_name %>:8083>
|
||||
ServerName <%= @srvname %>:8083
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}:8083\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}:8083" -%>
|
||||
<% end %>
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# e14353.d.akamaiedge.net
|
||||
CacheEnable disk "/e14353.d.akamaiedge"
|
||||
ProxyPass "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/"
|
||||
|
||||
# edgekey.net
|
||||
CacheEnable disk "/redhat.com.edgekey"
|
||||
ProxyPass "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/"
|
||||
|
||||
# registry.access.redhat.com
|
||||
CacheEnable disk "/"
|
||||
ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/" "https://registry.access.redhat.com/"
|
||||
</VirtualHost>
|
||||
|
||||
# Quay registry proxy.
|
||||
<VirtualHost <%= @vhost_name %>:8084>
|
||||
ServerName <%= @srvname %>:8084
|
||||
<% if @serveraliases.is_a? Array -%>
|
||||
<% @serveraliases.each do |name| -%>
|
||||
<%= " ServerAlias #{name}:8084\n" %>
|
||||
<% end %>
|
||||
<% elsif @serveraliases != nil -%>
|
||||
<%= " ServerAlias #{@serveraliases}:8084" -%>
|
||||
<% end %>
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# iah50.r.cloudfront.net
|
||||
CacheEnable disk "/iah50.r.cloudfront.net"
|
||||
ProxyPass "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/"
|
||||
|
||||
# quay.io
|
||||
CacheEnable disk "/"
|
||||
ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/" "https://quay.io/"
|
||||
</VirtualHost>
|
||||
@ -1,5 +1,6 @@
|
||||
iptables_extra_public_tcp_ports:
|
||||
- 80
|
||||
- 443
|
||||
- 8080
|
||||
- 8081
|
||||
- 8082
|
||||
|
||||
@ -1,8 +0,0 @@
|
||||
iptables_extra_public_tcp_ports:
|
||||
- 80
|
||||
- 443
|
||||
- 8080
|
||||
- 8081
|
||||
- 8082
|
||||
- 8083
|
||||
- 8084
|
||||
@ -33,16 +33,10 @@ results:
|
||||
- puppet
|
||||
- puppet4
|
||||
|
||||
mirror02.dfw.rax.openstack.org:
|
||||
- afs-client
|
||||
- mirror
|
||||
- puppet
|
||||
- puppet4
|
||||
|
||||
mirror01.regionone.linaro-us.opendev.org:
|
||||
- afs-client
|
||||
- letsencrypt
|
||||
- mirror_opendev
|
||||
- mirror
|
||||
|
||||
mirror-update01.openstack.org:
|
||||
- afsadmin
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
- hosts: "mirror_opendev:!disabled"
|
||||
- hosts: "mirror:!disabled"
|
||||
name: "Configure per region opendev mirrors"
|
||||
roles:
|
||||
- role: kerberos-client
|
||||
|
||||
@ -252,7 +252,7 @@
|
||||
files:
|
||||
- inventory/
|
||||
- playbooks/service-mirror.yaml
|
||||
- playbooks/group_vars/mirror_opendev.yaml
|
||||
- playbooks/group_vars/mirror.yaml
|
||||
- playbooks/roles/kerberos-client/
|
||||
- playbooks/roles/openafs-client/
|
||||
- playbooks/roles/mirror/
|
||||
|
||||
@ -350,12 +350,12 @@
|
||||
files:
|
||||
- playbooks/install-ansible.yaml
|
||||
- roles/
|
||||
- playbooks/group_vars/mirror_opendev.yaml
|
||||
- playbooks/group_vars/mirror.yaml
|
||||
- playbooks/roles/mirror/
|
||||
- playbooks/roles/letsencrypt
|
||||
- playbooks/service-letsencrypt.yaml
|
||||
- playbooks/service-mirror.yaml
|
||||
- playbooks/zuul/templates/group_vars/mirror_opendev.yaml.j2
|
||||
- playbooks/zuul/templates/group_vars/mirror.yaml.j2
|
||||
- testinfra/test_mirror.py
|
||||
|
||||
- job:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user