diff --git a/inventory/groups.yaml b/inventory/groups.yaml index a468fc35ef..e4361a066c 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -4,7 +4,7 @@ groups: afs: afs[0-9]*.open*.org afs-client: - review-dev[0-9]*.open*.org - - mirror[0-9]*.open*.org + - mirror[0-9]*.opendev.org - mirror-update[0-9]*.opendev.org - ze[0-9]*.open*.org - afsdb*.open*.org @@ -40,7 +40,6 @@ groups: - ethercalc01.openstack.org - idp.openstackid.org - lists-dev01.openstack.org - - mirror01.sto2.citycloud.openstack.org - wiki-dev01.openstack.org - wiki-upgrade-test.openstack.org dns: @@ -90,8 +89,6 @@ groups: meetpad: - meetpad[0-9]*.opendev.org mirror: - - mirror[0-9]*.openstack.org - mirror_opendev: - mirror[0-9]*.opendev.org mirror-update: - mirror-update[0-9]*.opendev.org @@ -136,7 +133,6 @@ groups: - logstash-worker[0-9]*.open*.org - logstash[0-9]*.open*.org - mirror-update[0-9]*.openstack.org - - mirror[0-9]*.openstack.org - nb[0-9]*.openstack.org - openstackid-dev*.openstack.org - openstackid.org @@ -175,7 +171,6 @@ groups: - logstash-worker[0-9]*.open*.org - logstash[0-9]*.open*.org - mirror-update[0-9]*.openstack.org - - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|linaro-us).*\.openstack\.org - ^nb(?!03)[0-9]*\.openstack\.org - openstackid[0-9]*.openstack.org - openstackid-dev[0-9]*.openstack.org diff --git a/manifests/site.pp b/manifests/site.pp index 3a4f7e6210..cd6ac92154 100644 --- a/manifests/site.pp +++ b/manifests/site.pp @@ -246,22 +246,6 @@ node /^mirror-update\d*\.open.*\.org$/ { } } -# Machines in each region to serve AFS mirrors. -# Node-OS: xenial -node /^mirror\d*\..*\.open.*\.org$/ { - $group = "mirror" - - class { 'openstack_project::server': - afs => true, - afs_cache_size => 50000000, # 50GB - } - - class { 'openstack_project::mirror': - vhost_name => $::fqdn, - require => Class['Openstack_project::Server'], - } -} - # Node-OS: trusty # Node-OS: xenial node /^refstack\d*\.open.*\.org$/ { diff --git a/modules/openstack_project/manifests/mirror.pp b/modules/openstack_project/manifests/mirror.pp deleted file mode 100644 index 688ad86cfe..0000000000 --- a/modules/openstack_project/manifests/mirror.pp +++ /dev/null @@ -1,378 +0,0 @@ -# == Class: openstack_project::mirror -# -class openstack_project::mirror ( - $vhost_name = $::fqdn, -) { - - # Some hosts are mirror01, but we need the host to respond to - # "mirror." Re-evaluate this if we end up doing multiple - # mirrors/load balancing etc. - $alias_name = regsubst($vhost_name, 'mirror\d*\.', 'mirror.') - if $alias_name != $vhost_name { - $serveraliases = [$alias_name] - } else { - $serveraliases = undef - } - - $mirror_root = '/afs/openstack.org/mirror' - $wheel_root = "${mirror_root}/wheel" - $ceph_deb_hammer_root = "${mirror_root}/ceph-deb-hammer" - $ceph_deb_jewel_root = "${mirror_root}/ceph-deb-jewel" - $ceph_deb_luminous_root = "${mirror_root}/ceph-deb-luminous" - $ceph_deb_mimic_root = "${mirror_root}/ceph-deb-mimic" - $ceph_deb_nautilus_root = "${mirror_root}/ceph-deb-nautilus" - $gem_root = "${mirror_root}/gem" - - $www_base = '/var/www' - $www_root = "${www_base}/mirror" - - ##################################################### - # Build Apache Webroot - file { "${www_base}": - ensure => directory, - owner => root, - group => root, - } - - file { "${www_root}": - ensure => directory, - owner => root, - group => root, - require => [ - File["${www_base}"], - ] - } - - # Create the symlink to pypi. - # NOTE(ianw) : retired 2018-11-22; see - # https://review.opendev.org/#/c/618326/ - file { "${www_root}/pypi": - ensure => absent, - } - - # Create the symlink to wheel. - file { "${www_root}/wheel": - ensure => link, - target => "${wheel_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to centos - file { "${www_root}/centos": - ensure => link, - target => "${mirror_root}/centos", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to debian - file { "${www_root}/debian": - ensure => link, - target => "${mirror_root}/debian", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to debian-security - file { "${www_root}/debian-security": - ensure => link, - target => "${mirror_root}/debian-security", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to Debian OpenStack Packaging Team reprepro. - file { "${www_root}/debian-openstack": - ensure => link, - target => "${mirror_root}/debian-openstack", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to rdo - file { "${www_root}/rdo": - ensure => absent, - } - - # Create the symlink to epel - file { "${www_root}/epel": - ensure => link, - target => "${mirror_root}/epel", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to yum-puppetlabs - file { "${www_root}/yum-puppetlabs": - ensure => link, - target => "${mirror_root}/yum-puppetlabs", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to fedora - file { "${www_root}/fedora": - ensure => link, - target => "${mirror_root}/fedora", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to openSUSE - file { "${www_root}/opensuse": - ensure => link, - target => "${mirror_root}/opensuse", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to Ubuntu - file { "${www_root}/ubuntu": - ensure => link, - target => "${mirror_root}/ubuntu", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to Ubuntu ports - file { "${www_root}/ubuntu-ports": - ensure => link, - target => "${mirror_root}/ubuntu-ports", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to ceph-deb-hammer. - file { "${www_root}/ceph-deb-hammer": - ensure => link, - target => "${ceph_deb_hammer_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to ceph-deb-jewel. - file { "${www_root}/ceph-deb-jewel": - ensure => link, - target => "${ceph_deb_jewel_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to ceph-deb-luminous. - file { "${www_root}/ceph-deb-luminous": - ensure => link, - target => "${ceph_deb_luminous_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to ceph-deb-mimic. - file { "${www_root}/ceph-deb-mimic": - ensure => link, - target => "${ceph_deb_mimic_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to ceph-deb-nautilus. - file { "${www_root}/ceph-deb-nautilus": - ensure => link, - target => "${ceph_deb_nautilus_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to Ubuntu Cloud Archive. - file { "${www_root}/ubuntu-cloud-archive": - ensure => link, - target => "${mirror_root}/ubuntu-cloud-archive", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to deb-docker. - file { "${www_root}/deb-docker": - ensure => link, - target => "${mirror_root}/deb-docker", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - # Create the symlink to Ubuntu Puppetlabs. - file { "${www_root}/apt-puppetlabs": - ensure => link, - target => "${mirror_root}/apt-puppetlabs", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - file { "${www_root}/gem": - ensure => link, - target => "${gem_root}", - owner => root, - group => root, - require => [ - File["${www_root}"], - ] - } - - file { "${www_root}/robots.txt": - ensure => present, - owner => 'root', - group => 'root', - mode => '0444', - source => 'puppet:///modules/openstack_project/disallow_robots.txt', - require => File["${www_root}"], - } - - ##################################################### - # Build VHost - include ::httpd - - file { '/opt/apache_cache': - ensure => absent, - force => true, - } - - file { '/var/cache/apache2/proxy': - ensure => directory, - owner => 'www-data', - group => 'www-data', - mode => '0755', - require => Class['httpd'] - } - - if ! defined(Httpd::Mod['rewrite']) { - httpd::mod { 'rewrite': - ensure => present, - } - } - - if ! defined(Httpd::Mod['substitute']) { - httpd::mod { 'substitute': - ensure => present, - } - } - - if ! defined(Httpd::Mod['cache']) { - httpd::mod { 'cache': - ensure => present, - } - } - - if ! defined(Httpd::Mod['cache_disk']) { - httpd::mod { 'cache_disk': - ensure => present, - } - } - - if ! defined(Httpd::Mod['proxy']) { - httpd::mod { 'proxy': - ensure => present, - } - } - - if ! defined(Httpd::Mod['proxy_http']) { - httpd::mod { 'proxy_http': - ensure => present, - } - } - - ::httpd::vhost { $vhost_name: - port => 80, - priority => '50', - docroot => "${www_root}", - template => 'openstack_project/mirror.vhost.erb', - serveraliases => $serveraliases, - require => [ - File["${www_root}"], - ] - } - - # Cache cleanup - package { 'apache2-utils': - ensure => present, - } - - cron { 'apache-cache-cleanup': - # Clean apache cache once an hour, keep size down to 70GiB. - minute => '0', - hour => '*', - command => 'flock -n /var/run/htcacheclean.lock htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null', - environment => 'PATH=/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin', - require => [ - File['/var/cache/apache2/proxy'], - Package['apache2-utils'], - ], - } - - class { '::httpd::logrotate': - options => [ - 'daily', - 'missingok', - 'rotate 7', - 'compress', - 'delaycompress', - 'notifempty', - 'create 640 root adm', - ], - } -} diff --git a/modules/openstack_project/templates/mirror.vhost.erb b/modules/openstack_project/templates/mirror.vhost.erb deleted file mode 100644 index 2a9cd10d01..0000000000 --- a/modules/openstack_project/templates/mirror.vhost.erb +++ /dev/null @@ -1,525 +0,0 @@ -# ************************************ -# Managed by Puppet -# ************************************ - -NameVirtualHost <%= @vhost_name %>:<%= @port %> - -# Dedicated port for proxy caching, as not to affect afs mirrors. -Listen 8080 -NameVirtualHost <%= @vhost_name %>:8080 - -Listen 8081 -NameVirtualHost <%= @vhost_name %>:8081 - -Listen 8082 -NameVirtualHost <%= @vhost_name %>:8082 - -Listen 8083 -NameVirtualHost <%= @vhost_name %>:8083 - -LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache - -:<%= @port %>> - ServerName <%= @srvname %> - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}" -%> - <% end %> - DocumentRoot <%= @docroot %> - > - Options <%= @options %> - AllowOverride None - Order allow,deny - allow from all - Satisfy any - = 2.4> - Require all granted - - - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - CacheStoreExpired On - # Pip sets Cache-Control: max-age=0 on requests for pypi index pages. - # This means we don't use the cache for those requests. This setting - # should force the proxy to ignore cache-control on the request side - # but we should still cache things based on the cache-control responses - # from the backed servers. - CacheIgnoreCacheControl On - - # Added Aug 2017 in an attempt to avoid occasional 502 errors (around - # 0.05% of requests) of the type: - # - # End of file found: ... AH01102: error reading status line from remote server ... - # - # Per [1]: - # - # This avoids the "proxy: error reading status line from remote - # server" error message caused by the race condition that the backend - # server closed the pooled connection after the connection check by the - # proxy and before data sent by the proxy reached the backend. - # - # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html - SetEnv proxy-initial-not-pooled 1 - - RewriteEngine On - # pypi - CacheEnable disk "/pypi" - ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/pypi/" "https://pypi.org/ - - # files.pythonhosted.org - CacheEnable disk "/pypifiles" - ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/" - - # Rewrite the locations of the actual files - - SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE - Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni" - - - # Wheel URL's are: - # /wheel/{distro}-{distro-version}/a/a/a-etc.whl - # /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl - # /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl - RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*) - RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d - RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L] - - # Special cases for openstack.nose_plugin & backports.* - RewriteCond %{REQUEST_URI} ^/wheel/ - RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2 - RewriteCond %{REQUEST_URI} ^/wheel/ - RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2 - - # Try again but replacing -'s with .'s - RewriteCond %{REQUEST_URI} ^/wheel/ - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f - RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d - RewriteRule (.*)-(.*) $1.$2 [N] - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined-cache - ServerSignature Off - - -:8080> - ServerName <%= @srvname %>:8080 - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}:8080\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}:8080" -%> - <% end %> - - # Disable directory listing by default. - - Order Deny,Allow - Deny from all - Options None - AllowOverride None - - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8080_access.log combined-cache - ServerSignature Off - - # Let upstreams decide on encoded slash handling. - # The default is 'Off' which returns 404 for URLs with encoded slashes, - # i.e. '%2f' instead of '/'. - AllowEncodedSlashes NoDecode - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - CacheStoreExpired On - - # Added Aug 2017 in an attempt to avoid occasional 502 errors (around - # 0.05% of requests) of the type: - # - # End of file found: ... AH01102: error reading status line from remote server ... - # - # Per [1]: - # - # This avoids the "proxy: error reading status line from remote - # server" error message caused by the race condition that the backend - # server closed the pooled connection after the connection check by the - # proxy and before data sent by the proxy reached the backend. - # - # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html - SetEnv proxy-initial-not-pooled 1 - - # Per site caching reverse proxy rules - # Only cache specific backends, rely on afs cache otherwise. - - # buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org) - CacheEnable disk "/buildlogs.centos" - ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0 - ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/" - - # buildlogs.cdn.centos.org - CacheEnable disk "/buildlogs.cdn.centos" - ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0 - ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" - - # rdo - CacheEnable disk "/rdo" - ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/" - - # cbs.centos.org - CacheEnable disk "/cbs.centos" - ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/" - - # pypi - CacheEnable disk "/pypi" - ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/pypi/" "https://pypi.org/ - - # files.pythonhosted.org - CacheEnable disk "/pypifiles" - ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/" - - # Rewrite the locations of the actual files - - SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE - Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni" - - - # images.linuxcontainers.org - CacheEnable disk "/images.linuxcontainers" - ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" - - # registry.npmjs.org - CacheEnable disk "/registry.npmjs" - ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0 nocanon - ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/" - - # api.rubygems.org - CacheEnable disk "/api.rubygems" - ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/" - - # rubygems.org - CacheEnable disk "/rubygems" - ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/rubygems/" "https://rubygems.org/" - - # opendaylight - CacheEnable disk "/opendaylight" - ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/" - - # elastico - CacheEnable disk "/elastic" - ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/elastic/" "https://packages.elastic.co/" - - # grafana - CacheEnable disk "/grafana" - ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/" - - # OracleLinux - CacheEnable disk "/oraclelinux" - ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" - - # Percona - CacheEnable disk "/percona" - ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/percona/" "https://repo.percona.com/" - - # MariaDB - CacheEnable disk "/MariaDB" - ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" - - # Docker - CacheEnable disk "/docker" - ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/docker/" "https://download.docker.com/linux/" - - # Alpine - CacheEnable disk "/alpine" - ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" - - # LXC (copr) - CacheEnable disk "/copr-lxc2" - ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" - - - -# Docker registry v1 proxy. -:8081> - ServerName <%= @srvname %>:8081 - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}:8081\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}:8081" -%> - <% end %> - - # Disable directory listing by default. - - Order Deny,Allow - Deny from all - Options None - AllowOverride None - - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8081_access.log combined-cache - ServerSignature Off - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - # Ignore expire headers as the urls use sha256 hashes. - CacheIgnoreQueryString On - # NOTE(pabelanger): In the case of docker, if neither an expiry date nor - # last-modified date are provided default expire to 1 day. This is up from - # 1 hour. - CacheDefaultExpire 86400 - CacheStoreExpired On - - # registry-1.docker.io - CacheEnable disk "/registry-1.docker" - ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/" - - # dseasb33srnrn.cloudfront.net - CacheEnable disk "/cloudfront" - ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" - - # production.cloudflare.docker.com - CacheEnable disk "/cloudflare" - ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" - - - -# Docker registry v2 proxy. -:8082> - ServerName <%= @srvname %>:8082 - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}:8082\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}:8082" -%> - <% end %> - - # Disable directory listing by default. - - Order Deny,Allow - Deny from all - Options None - AllowOverride None - - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8082_access.log combined-cache - ServerSignature Off - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - # Ignore expire headers as the urls use sha256 hashes. - CacheIgnoreQueryString On - # NOTE(pabelanger): In the case of docker, if neither an expiry date nor - # last-modified date are provided default expire to 1 day. This is up from - # 1 hour. - CacheDefaultExpire 86400 - CacheStoreExpired On - - # dseasb33srnrn.cloudfront.net - CacheEnable disk "/cloudfront" - ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" - - # production.cloudflare.docker.com - CacheEnable disk "/cloudflare" - ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" - - # NOTE(corvus): Ensure this stanza is last since it's the most - # greedy match. - CacheEnable disk "/" - ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/" "https://registry-1.docker.io/" - - -# Redhat registry proxy. -:8083> - ServerName <%= @srvname %>:8083 - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}:8083\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}:8083" -%> - <% end %> - - # Disable directory listing by default. - - Order Deny,Allow - Deny from all - Options None - AllowOverride None - - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8083_access.log combined-cache - ServerSignature Off - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - # Ignore expire headers as the urls use sha256 hashes. - CacheIgnoreQueryString On - CacheDefaultExpire 86400 - CacheStoreExpired On - - # e14353.d.akamaiedge.net - CacheEnable disk "/e14353.d.akamaiedge" - ProxyPass "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/e14353.d.akamaiedge/" "https://e14353.d.akamaiedge.net/" - - # edgekey.net - CacheEnable disk "/redhat.com.edgekey" - ProxyPass "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/redhat.com.edgekey/" "https://registry.access.redhat.com.edgekey.net/" - - # registry.access.redhat.com - CacheEnable disk "/" - ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/" "https://registry.access.redhat.com/" - - -# Quay registry proxy. -:8084> - ServerName <%= @srvname %>:8084 - <% if @serveraliases.is_a? Array -%> - <% @serveraliases.each do |name| -%> - <%= " ServerAlias #{name}:8084\n" %> - <% end %> - <% elsif @serveraliases != nil -%> - <%= " ServerAlias #{@serveraliases}:8084" -%> - <% end %> - - # Disable directory listing by default. - - Order Deny,Allow - Deny from all - Options None - AllowOverride None - - - ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_error.log - LogLevel warn - CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_8084_access.log combined-cache - ServerSignature Off - - # Caching reverse proxy for things that don't make sense in AFS - # - # General cache rules - CacheRoot "/var/cache/apache2/proxy" - CacheDirLevels 5 - CacheDirLength 2 - # SSL support - SSLProxyEngine on - # Prevent thundering herds. - CacheLock on - CacheLockPath "/tmp/mod_cache-lock" - CacheLockMaxAge 5 - # 5GiB - CacheMaxFileSize 5368709120 - # Ignore expire headers as the urls use sha256 hashes. - CacheIgnoreQueryString On - CacheDefaultExpire 86400 - CacheStoreExpired On - - # iah50.r.cloudfront.net - CacheEnable disk "/iah50.r.cloudfront.net" - ProxyPass "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/iah50.r.cloudfront.net/" "https://iah50.r.cloudfront.net/" - - # quay.io - CacheEnable disk "/" - ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/" "https://quay.io/" - diff --git a/playbooks/group_vars/mirror.yaml b/playbooks/group_vars/mirror.yaml index beae37d749..1306e83cf7 100644 --- a/playbooks/group_vars/mirror.yaml +++ b/playbooks/group_vars/mirror.yaml @@ -1,5 +1,6 @@ iptables_extra_public_tcp_ports: - 80 + - 443 - 8080 - 8081 - 8082 diff --git a/playbooks/group_vars/mirror_opendev.yaml b/playbooks/group_vars/mirror_opendev.yaml deleted file mode 100644 index 1306e83cf7..0000000000 --- a/playbooks/group_vars/mirror_opendev.yaml +++ /dev/null @@ -1,8 +0,0 @@ -iptables_extra_public_tcp_ports: - - 80 - - 443 - - 8080 - - 8081 - - 8082 - - 8083 - - 8084 diff --git a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml index 975b81d478..91571a98ab 100644 --- a/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml +++ b/playbooks/roles/install-ansible/files/inventory_plugins/test-fixtures/results.yaml @@ -33,16 +33,10 @@ results: - puppet - puppet4 - mirror02.dfw.rax.openstack.org: - - afs-client - - mirror - - puppet - - puppet4 - mirror01.regionone.linaro-us.opendev.org: - afs-client - letsencrypt - - mirror_opendev + - mirror mirror-update01.openstack.org: - afsadmin diff --git a/playbooks/service-mirror.yaml b/playbooks/service-mirror.yaml index 0dae8db3a4..961c9d2479 100644 --- a/playbooks/service-mirror.yaml +++ b/playbooks/service-mirror.yaml @@ -1,4 +1,4 @@ -- hosts: "mirror_opendev:!disabled" +- hosts: "mirror:!disabled" name: "Configure per region opendev mirrors" roles: - role: kerberos-client diff --git a/zuul.d/infra-prod.yaml b/zuul.d/infra-prod.yaml index f32c459d7b..5a36d666e1 100644 --- a/zuul.d/infra-prod.yaml +++ b/zuul.d/infra-prod.yaml @@ -252,7 +252,7 @@ files: - inventory/ - playbooks/service-mirror.yaml - - playbooks/group_vars/mirror_opendev.yaml + - playbooks/group_vars/mirror.yaml - playbooks/roles/kerberos-client/ - playbooks/roles/openafs-client/ - playbooks/roles/mirror/ diff --git a/zuul.d/system-config-run.yaml b/zuul.d/system-config-run.yaml index 9dcc1e763a..35715a06e9 100644 --- a/zuul.d/system-config-run.yaml +++ b/zuul.d/system-config-run.yaml @@ -350,12 +350,12 @@ files: - playbooks/install-ansible.yaml - roles/ - - playbooks/group_vars/mirror_opendev.yaml + - playbooks/group_vars/mirror.yaml - playbooks/roles/mirror/ - playbooks/roles/letsencrypt - playbooks/service-letsencrypt.yaml - playbooks/service-mirror.yaml - - playbooks/zuul/templates/group_vars/mirror_opendev.yaml.j2 + - playbooks/zuul/templates/group_vars/mirror.yaml.j2 - testinfra/test_mirror.py - job: