Browse Source

Remove puppet mirror support

Remove the separate "mirror_opendev" group and rename it to just
"mirror".  Update various parts to reflect that change.

We no longer deploy any mirror hosts with puppet, remove the various
configuration files.

Depends-On: https://review.opendev.org/728345
Change-Id: Ia982fe9cb4357447989664f033df976b528aaf84
changes/50/728350/4
Ian Wienand 2 weeks ago
parent
commit
45201f3d66
10 changed files with 7 additions and 944 deletions
  1. +1
    -6
      inventory/groups.yaml
  2. +0
    -16
      manifests/site.pp
  3. +0
    -378
      modules/openstack_project/manifests/mirror.pp
  4. +0
    -525
      modules/openstack_project/templates/mirror.vhost.erb
  5. +1
    -0
      playbooks/group_vars/mirror.yaml
  6. +0
    -8
      playbooks/group_vars/mirror_opendev.yaml
  7. +1
    -7
      playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml
  8. +1
    -1
      playbooks/service-mirror.yaml
  9. +1
    -1
      zuul.d/infra-prod.yaml
  10. +2
    -2
      zuul.d/system-config-run.yaml

+ 1
- 6
inventory/groups.yaml View File

@@ -4,7 +4,7 @@ groups:
afs: afs[0-9]*.open*.org
afs-client:
- review-dev[0-9]*.open*.org
- mirror[0-9]*.open*.org
- mirror[0-9]*.opendev.org
- mirror-update[0-9]*.opendev.org
- ze[0-9]*.open*.org
- afsdb*.open*.org
@@ -40,7 +40,6 @@ groups:
- ethercalc01.openstack.org
- idp.openstackid.org
- lists-dev01.openstack.org
- mirror01.sto2.citycloud.openstack.org
- wiki-dev01.openstack.org
- wiki-upgrade-test.openstack.org
dns:
@@ -90,8 +89,6 @@ groups:
meetpad:
- meetpad[0-9]*.opendev.org
mirror:
- mirror[0-9]*.openstack.org
mirror_opendev:
- mirror[0-9]*.opendev.org
mirror-update:
- mirror-update[0-9]*.opendev.org
@@ -136,7 +133,6 @@ groups:
- logstash-worker[0-9]*.open*.org
- logstash[0-9]*.open*.org
- mirror-update[0-9]*.openstack.org
- mirror[0-9]*.openstack.org
- nb[0-9]*.openstack.org
- openstackid-dev*.openstack.org
- openstackid.org
@@ -175,7 +171,6 @@ groups:
- logstash-worker[0-9]*.open*.org
- logstash[0-9]*.open*.org
- mirror-update[0-9]*.openstack.org
- ^mirror[0-9].*\..*\.(?!linaro|linaro-london|linaro-us).*\.openstack\.org
- ^nb(?!03)[0-9]*\.openstack\.org
- openstackid[0-9]*.openstack.org
- openstackid-dev[0-9]*.openstack.org


+ 0
- 16
manifests/site.pp View File

@@ -246,22 +246,6 @@ node /^mirror-update\d*\.open.*\.org$/ {
}
}

# Machines in each region to serve AFS mirrors.
# Node-OS: xenial
node /^mirror\d*\..*\.open.*\.org$/ {
$group = "mirror"

class { 'openstack_project::server':
afs => true,
afs_cache_size => 50000000, # 50GB
}

class { 'openstack_project::mirror':
vhost_name => $::fqdn,
require => Class['Openstack_project::Server'],
}
}

# Node-OS: trusty
# Node-OS: xenial
node /^refstack\d*\.open.*\.org$/ {


+ 0
- 378
modules/openstack_project/manifests/mirror.pp View File

@@ -1,378 +0,0 @@
# == Class: openstack_project::mirror
#
class openstack_project::mirror (
$vhost_name = $::fqdn,
) {

# Some hosts are mirror01, but we need the host to respond to
# "mirror." Re-evaluate this if we end up doing multiple
# mirrors/load balancing etc.
$alias_name = regsubst($vhost_name, 'mirror\d*\.', 'mirror.')
if $alias_name != $vhost_name {
$serveraliases = [$alias_name]
} else {
$serveraliases = undef
}

$mirror_root = '/afs/openstack.org/mirror'
$wheel_root = "${mirror_root}/wheel"
$ceph_deb_hammer_root = "${mirror_root}/ceph-deb-hammer"
$ceph_deb_jewel_root = "${mirror_root}/ceph-deb-jewel"
$ceph_deb_luminous_root = "${mirror_root}/ceph-deb-luminous"
$ceph_deb_mimic_root = "${mirror_root}/ceph-deb-mimic"
$ceph_deb_nautilus_root = "${mirror_root}/ceph-deb-nautilus"
$gem_root = "${mirror_root}/gem"

$www_base = '/var/www'
$www_root = "${www_base}/mirror"

#####################################################
# Build Apache Webroot
file { "${www_base}":
ensure => directory,
owner => root,
group => root,
}

file { "${www_root}":
ensure => directory,
owner => root,
group => root,
require => [
File["${www_base}"],
]
}

# Create the symlink to pypi.
# NOTE(ianw) : retired 2018-11-22; see
# https://review.opendev.org/#/c/618326/
file { "${www_root}/pypi":
ensure => absent,
}

# Create the symlink to wheel.
file { "${www_root}/wheel":
ensure => link,
target => "${wheel_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to centos
file { "${www_root}/centos":
ensure => link,
target => "${mirror_root}/centos",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to debian
file { "${www_root}/debian":
ensure => link,
target => "${mirror_root}/debian",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to debian-security
file { "${www_root}/debian-security":
ensure => link,
target => "${mirror_root}/debian-security",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to Debian OpenStack Packaging Team reprepro.
file { "${www_root}/debian-openstack":
ensure => link,
target => "${mirror_root}/debian-openstack",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to rdo
file { "${www_root}/rdo":
ensure => absent,
}

# Create the symlink to epel
file { "${www_root}/epel":
ensure => link,
target => "${mirror_root}/epel",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to yum-puppetlabs
file { "${www_root}/yum-puppetlabs":
ensure => link,
target => "${mirror_root}/yum-puppetlabs",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to fedora
file { "${www_root}/fedora":
ensure => link,
target => "${mirror_root}/fedora",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to openSUSE
file { "${www_root}/opensuse":
ensure => link,
target => "${mirror_root}/opensuse",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to Ubuntu
file { "${www_root}/ubuntu":
ensure => link,
target => "${mirror_root}/ubuntu",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to Ubuntu ports
file { "${www_root}/ubuntu-ports":
ensure => link,
target => "${mirror_root}/ubuntu-ports",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to ceph-deb-hammer.
file { "${www_root}/ceph-deb-hammer":
ensure => link,
target => "${ceph_deb_hammer_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to ceph-deb-jewel.
file { "${www_root}/ceph-deb-jewel":
ensure => link,
target => "${ceph_deb_jewel_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to ceph-deb-luminous.
file { "${www_root}/ceph-deb-luminous":
ensure => link,
target => "${ceph_deb_luminous_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to ceph-deb-mimic.
file { "${www_root}/ceph-deb-mimic":
ensure => link,
target => "${ceph_deb_mimic_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to ceph-deb-nautilus.
file { "${www_root}/ceph-deb-nautilus":
ensure => link,
target => "${ceph_deb_nautilus_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to Ubuntu Cloud Archive.
file { "${www_root}/ubuntu-cloud-archive":
ensure => link,
target => "${mirror_root}/ubuntu-cloud-archive",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to deb-docker.
file { "${www_root}/deb-docker":
ensure => link,
target => "${mirror_root}/deb-docker",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

# Create the symlink to Ubuntu Puppetlabs.
file { "${www_root}/apt-puppetlabs":
ensure => link,
target => "${mirror_root}/apt-puppetlabs",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

file { "${www_root}/gem":
ensure => link,
target => "${gem_root}",
owner => root,
group => root,
require => [
File["${www_root}"],
]
}

file { "${www_root}/robots.txt":
ensure => present,
owner => 'root',
group => 'root',
mode => '0444',
source => 'puppet:///modules/openstack_project/disallow_robots.txt',
require => File["${www_root}"],
}

#####################################################
# Build VHost
include ::httpd

file { '/opt/apache_cache':
ensure => absent,
force => true,
}

file { '/var/cache/apache2/proxy':
ensure => directory,
owner => 'www-data',
group => 'www-data',
mode => '0755',
require => Class['httpd']
}

if ! defined(Httpd::Mod['rewrite']) {
httpd::mod { 'rewrite':
ensure => present,
}
}

if ! defined(Httpd::Mod['substitute']) {
httpd::mod { 'substitute':
ensure => present,
}
}

if ! defined(Httpd::Mod['cache']) {
httpd::mod { 'cache':
ensure => present,
}
}

if ! defined(Httpd::Mod['cache_disk']) {
httpd::mod { 'cache_disk':
ensure => present,
}
}

if ! defined(Httpd::Mod['proxy']) {
httpd::mod { 'proxy':
ensure => present,
}
}

if ! defined(Httpd::Mod['proxy_http']) {
httpd::mod { 'proxy_http':
ensure => present,
}
}

::httpd::vhost { $vhost_name:
port => 80,
priority => '50',
docroot => "${www_root}",
template => 'openstack_project/mirror.vhost.erb',
serveraliases => $serveraliases,
require => [
File["${www_root}"],
]
}

# Cache cleanup
package { 'apache2-utils':
ensure => present,
}

cron { 'apache-cache-cleanup':
# Clean apache cache once an hour, keep size down to 70GiB.
minute => '0',
hour => '*',
command => 'flock -n /var/run/htcacheclean.lock htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null',
environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin',
require => [
File['/var/cache/apache2/proxy'],
Package['apache2-utils'],
],
}

class { '::httpd::logrotate':
options => [
'daily',
'missingok',
'rotate 7',
'compress',
'delaycompress',
'notifempty',
'create 640 root adm',
],
}
}

+ 0
- 525
modules/openstack_project/templates/mirror.vhost.erb View File

@@ -1,525 +0,0 @@
# ************************************
# Managed by Puppet
# ************************************

NameVirtualHost <%= @vhost_name %>:<%= @port %>

# Dedicated port for proxy caching, as not to affect afs mirrors.
Listen 8080
NameVirtualHost <%= @vhost_name %>:8080

Listen 8081
NameVirtualHost <%= @vhost_name %>:8081

Listen 8082
NameVirtualHost <%= @vhost_name %>:8082

Listen 8083
NameVirtualHost <%= @vhost_name %>:8083

LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache

<VirtualHost <%= @vhost_name %>:<%= @port %>>
ServerName <%= @srvname %>
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}" -%>
<% end %>
DocumentRoot <%= @docroot %>
<Directory <%= @docroot %>>
Options <%= @options %>
AllowOverride None
Order allow,deny
allow from all
Satisfy any
<IfVersion >= 2.4>
Require all granted
</IfVersion>
</Directory>

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
CacheStoreExpired On
# Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
# This means we don't use the cache for those requests. This setting
# should force the proxy to ignore cache-control on the request side
# but we should still cache things based on the cache-control responses
# from the backed servers.
CacheIgnoreCacheControl On

# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
# 0.05% of requests) of the type:
#
# End of file found: ... AH01102: error reading status line from remote server ...
#
# Per [1]:
#
# This avoids the "proxy: error reading status line from remote
# server" error message caused by the race condition that the backend
# server closed the pooled connection after the connection check by the
# proxy and before data sent by the proxy reached the backend.
#
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
SetEnv proxy-initial-not-pooled 1

RewriteEngine On
# pypi
CacheEnable disk "/pypi"
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/pypi/" "https://pypi.org/

# files.pythonhosted.org
CacheEnable disk "/pypifiles"
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"

# Rewrite the locations of the actual files
<Location /pypi>
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
</Location>

# Wheel URL's are:
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]

# Special cases for openstack.nose_plugin & backports.*
RewriteCond %{REQUEST_URI} ^/wheel/
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
RewriteCond %{REQUEST_URI} ^/wheel/
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2

# Try again but replacing -'s with .'s
RewriteCond %{REQUEST_URI} ^/wheel/
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
RewriteRule (.*)-(.*) $1.$2 [N]

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache
ServerSignature Off
</VirtualHost>

<VirtualHost <%= @vhost_name %>:8080>
ServerName <%= @srvname %>:8080
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}:8080\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}:8080" -%>
<% end %>

# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined-cache
ServerSignature Off

# Let upstreams decide on encoded slash handling.
# The default is 'Off' which returns 404 for URLs with encoded slashes,
# i.e. '%2f' instead of '/'.
AllowEncodedSlashes NoDecode

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
CacheStoreExpired On

# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
# 0.05% of requests) of the type:
#
# End of file found: ... AH01102: error reading status line from remote server ...
#
# Per [1]:
#
# This avoids the "proxy: error reading status line from remote
# server" error message caused by the race condition that the backend
# server closed the pooled connection after the connection check by the
# proxy and before data sent by the proxy reached the backend.
#
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
SetEnv proxy-initial-not-pooled 1

# Per site caching reverse proxy rules
# Only cache specific backends, rely on afs cache otherwise.

# buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
CacheEnable disk "/buildlogs.centos"
ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"

# buildlogs.cdn.centos.org
CacheEnable disk "/buildlogs.cdn.centos"
ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"

# rdo
CacheEnable disk "/rdo"
ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"

# cbs.centos.org
CacheEnable disk "/cbs.centos"
ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"

# pypi
CacheEnable disk "/pypi"
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/pypi/" "https://pypi.org/

# files.pythonhosted.org
CacheEnable disk "/pypifiles"
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"

# Rewrite the locations of the actual files
<Location /pypi>
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
</Location>

# images.linuxcontainers.org
CacheEnable disk "/images.linuxcontainers"
ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"

# registry.npmjs.org
CacheEnable disk "/registry.npmjs"
ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0 nocanon
ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"

# api.rubygems.org
CacheEnable disk "/api.rubygems"
ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"

# rubygems.org
CacheEnable disk "/rubygems"
ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/rubygems/" "https://rubygems.org/"

# opendaylight
CacheEnable disk "/opendaylight"
ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"

# elastico
CacheEnable disk "/elastic"
ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/elastic/" "https://packages.elastic.co/"

# grafana
CacheEnable disk "/grafana"
ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"

# OracleLinux
CacheEnable disk "/oraclelinux"
ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"

# Percona
CacheEnable disk "/percona"
ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/percona/" "https://repo.percona.com/"

# MariaDB
CacheEnable disk "/MariaDB"
ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"

# Docker
CacheEnable disk "/docker"
ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/docker/" "https://download.docker.com/linux/"

# Alpine
CacheEnable disk "/alpine"
ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"

# LXC (copr)
CacheEnable disk "/copr-lxc2"
ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"

</VirtualHost>

# Docker registry v1 proxy.
<VirtualHost <%= @vhost_name %>:8081>
ServerName <%= @srvname %>:8081
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}:8081\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}:8081" -%>
<% end %>

# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined-cache
ServerSignature Off

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Ignore expire headers as the urls use sha256 hashes.
CacheIgnoreQueryString On
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
# last-modified date are provided default expire to 1 day. This is up from
# 1 hour.
CacheDefaultExpire 86400
CacheStoreExpired On

# registry-1.docker.io
CacheEnable disk "/registry-1.docker"
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"

# dseasb33srnrn.cloudfront.net
CacheEnable disk "/cloudfront"
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"

# production.cloudflare.docker.com
CacheEnable disk "/cloudflare"
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"

</VirtualHost>

# Docker registry v2 proxy.
<VirtualHost <%= @vhost_name %>:8082>
ServerName <%= @srvname %>:8082
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}:8082\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}:8082" -%>
<% end %>

# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_access.log combined-cache
ServerSignature Off

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Ignore expire headers as the urls use sha256 hashes.
CacheIgnoreQueryString On
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
# last-modified date are provided default expire to 1 day. This is up from
# 1 hour.
CacheDefaultExpire 86400
CacheStoreExpired On

# dseasb33srnrn.cloudfront.net
CacheEnable disk "/cloudfront"
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"

# production.cloudflare.docker.com
CacheEnable disk "/cloudflare"
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"

# NOTE(corvus): Ensure this stanza is last since it's the most
# greedy match.
CacheEnable disk "/"
ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/" "https://registry-1.docker.io/"
</VirtualHost>

# Redhat registry proxy.
<VirtualHost <%= @vhost_name %>:8083>
ServerName <%= @srvname %>:8083
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}:8083\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}:8083" -%>
<% end %>

# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_access.log combined-cache
ServerSignature Off

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Ignore expire headers as the urls use sha256 hashes.
CacheIgnoreQueryString On
CacheDefaultExpire 86400
CacheStoreExpired On

# e14353.d.akamaiedge.net
CacheEnable disk "/e14353.d.akamaiedge"
ProxyPass "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/"

# edgekey.net
CacheEnable disk "/redhat.com.edgekey"
ProxyPass "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/"

# registry.access.redhat.com
CacheEnable disk "/"
ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/" "https://registry.access.redhat.com/"
</VirtualHost>

# Quay registry proxy.
<VirtualHost <%= @vhost_name %>:8084>
ServerName <%= @srvname %>:8084
<% if @serveraliases.is_a? Array -%>
<% @serveraliases.each do |name| -%>
<%= " ServerAlias #{name}:8084\n" %>
<% end %>
<% elsif @serveraliases != nil -%>
<%= " ServerAlias #{@serveraliases}:8084" -%>
<% end %>

# Disable directory listing by default.
<Directory />
Order Deny,Allow
Deny from all
Options None
AllowOverride None
</Directory>

ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_error.log
LogLevel warn
CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_access.log combined-cache
ServerSignature Off

# Caching reverse proxy for things that don't make sense in AFS
#
# General cache rules
CacheRoot "/var/cache/apache2/proxy"
CacheDirLevels 5
CacheDirLength 2
# SSL support
SSLProxyEngine on
# Prevent thundering herds.
CacheLock on
CacheLockPath "/tmp/mod_cache-lock"
CacheLockMaxAge 5
# 5GiB
CacheMaxFileSize 5368709120
# Ignore expire headers as the urls use sha256 hashes.
CacheIgnoreQueryString On
CacheDefaultExpire 86400
CacheStoreExpired On

# iah50.r.cloudfront.net
CacheEnable disk "/iah50.r.cloudfront.net"
ProxyPass "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/"

# quay.io
CacheEnable disk "/"
ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0
ProxyPassReverse "/" "https://quay.io/"
</VirtualHost>

+ 1
- 0
playbooks/group_vars/mirror.yaml View File

@@ -1,5 +1,6 @@
iptables_extra_public_tcp_ports:
- 80
- 443
- 8080
- 8081
- 8082


+ 0
- 8
playbooks/group_vars/mirror_opendev.yaml View File

@@ -1,8 +0,0 @@
iptables_extra_public_tcp_ports:
- 80
- 443
- 8080
- 8081
- 8082
- 8083
- 8084

+ 1
- 7
playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml View File

@@ -33,16 +33,10 @@ results:
- puppet
- puppet4

mirror02.dfw.rax.openstack.org:
- afs-client
- mirror
- puppet
- puppet4

mirror01.regionone.linaro-us.opendev.org:
- afs-client
- letsencrypt
- mirror_opendev
- mirror

mirror-update01.openstack.org:
- afsadmin


+ 1
- 1
playbooks/service-mirror.yaml View File

@@ -1,4 +1,4 @@
- hosts: "mirror_opendev:!disabled"
- hosts: "mirror:!disabled"
name: "Configure per region opendev mirrors"
roles:
- role: kerberos-client


+ 1
- 1
zuul.d/infra-prod.yaml View File

@@ -252,7 +252,7 @@
files:
- inventory/
- playbooks/service-mirror.yaml
- playbooks/group_vars/mirror_opendev.yaml
- playbooks/group_vars/mirror.yaml
- playbooks/roles/kerberos-client/
- playbooks/roles/openafs-client/
- playbooks/roles/mirror/


+ 2
- 2
zuul.d/system-config-run.yaml View File

@@ -350,12 +350,12 @@
files:
- playbooks/install-ansible.yaml
- roles/
- playbooks/group_vars/mirror_opendev.yaml
- playbooks/group_vars/mirror.yaml
- playbooks/roles/mirror/
- playbooks/roles/letsencrypt
- playbooks/service-letsencrypt.yaml
- playbooks/service-mirror.yaml
- playbooks/zuul/templates/group_vars/mirror_opendev.yaml.j2
- playbooks/zuul/templates/group_vars/mirror.yaml.j2
- testinfra/test_mirror.py

- job:


Loading…
Cancel
Save