Create opendev mirrors
This impelements mirrors to live in the opendev.org namespace. The implementation is Ansible native for deployment on a Bionic node. The hostname prefix remains the same (mirrorXX.region.provider.) but the groups.yaml splits the opendev.org mirrors into a separate group. The matches in the puppet group are also updated so to not run puppet on the hosts. The kerberos and openafs client parts do not need any updating and works on the Bionic host. The hosts are setup to provision certificates for themselves from letsencrypt. Note we've added a new handler for mirror nodes to use that restarts apache on certificate issue/renewal. The new "mirror" role is a port of the existing puppet mirror.pp. It installs apache, sets up some modules, makes some symlinks, sets up a cleanup cron job and installs the apache vhost configuration. The vhost configuration is also ported from the extant puppet. It is simplified somewhat; but the biggest change is that we have extracted the main port 80 configuration into a macro which is applied to both port 80 and 443; i.e. the host will have SSL support. The other ports are left alone for now, but can be updated in due course. Thus we should be able to CNAME the existing mirrors to new nodes, and any existing http access can continue. We can update our mirror setup scripts to point to https resources as appropriate. Change-Id: Iec576d631dd5b02f6b9fb445ee600be060f9cf1e
This commit is contained in:
Ian Wienand
parent
a4566bab0c
commit
670107045a
30
.zuul.yaml
30
.zuul.yaml
|
|
@ -481,6 +481,35 @@
|
|||
- testinfra/test_adns.py
|
||||
- testinfra/test_ns.py
|
||||
|
||||
- job:
|
||||
name: system-config-run-mirror
|
||||
parent: system-config-run
|
||||
description: |
|
||||
Run the playbook for a mirror node
|
||||
nodeset:
|
||||
nodes:
|
||||
- name: bridge.openstack.org
|
||||
label: ubuntu-bionic
|
||||
- name: mirror01.region.provider.opendev.org
|
||||
label: ubuntu-bionic
|
||||
vars:
|
||||
run_playbooks:
|
||||
- playbooks/service-letsencrypt.yaml
|
||||
- playbooks/service-mirror.yaml
|
||||
files:
|
||||
- .zuul.yaml
|
||||
- roles/
|
||||
- playbooks/roles/mirror/
|
||||
- playbooks/roles/letsencrypt.*
|
||||
- playbooks/service-letsencrypt.yaml
|
||||
- playbooks/service-mirror.yaml
|
||||
- testinfra/test_mirror.py
|
||||
|
||||
host-vars:
|
||||
mirror.region.provider.opendev.org:
|
||||
host_copy_output:
|
||||
'/var/log/apache2/': logs
|
||||
|
||||
- job:
|
||||
name: system-config-run-docker-registry
|
||||
parent: system-config-run
|
||||
|
|
@ -615,6 +644,7 @@
|
|||
- system-config-run-dns
|
||||
- system-config-run-eavesdrop
|
||||
- system-config-run-nodepool
|
||||
- system-config-run-mirror
|
||||
- system-config-run-docker-registry
|
||||
- system-config-run-gitea:
|
||||
dependencies:
|
||||
|
|
|
|||
|
|
@ -57,6 +57,7 @@ groups:
|
|||
- opendev-k8s*.opendev.org
|
||||
letsencrypt:
|
||||
- graphite01.opendev.org
|
||||
- mirror[0-9]*.opendev.org
|
||||
logstash:
|
||||
- logstash[0-9]*.open*.org
|
||||
logstash-worker:
|
||||
|
|
@ -65,7 +66,9 @@ groups:
|
|||
- lists*.katacontainers.io
|
||||
- lists*.open*.org
|
||||
mirror:
|
||||
- mirror[0-9]*.open*.org
|
||||
- mirror[0-9]*.openstack.org
|
||||
mirror_opendev:
|
||||
- mirror[0-9]*.opendev.org
|
||||
nodepool:
|
||||
- nb[0-9]*.open*.org
|
||||
- nl[0-9]*.open*.org
|
||||
|
|
@ -110,7 +113,7 @@ groups:
|
|||
- logstash-worker[0-9]*.open*.org
|
||||
- logstash[0-9]*.open*.org
|
||||
- mirror-update[0-9]*.open*.org
|
||||
- mirror[0-9]*.open*.org
|
||||
- mirror[0-9]*.openstack.org
|
||||
- nb[0-9]*.open*.org
|
||||
- nl[0-9]*.open*.org
|
||||
- openstackid-dev*.openstack.org
|
||||
|
|
@ -161,7 +164,7 @@ groups:
|
|||
- logstash-worker[0-9]*.open*.org
|
||||
- logstash[0-9]*.open*.org
|
||||
- mirror-update[0-9]*.open*.org
|
||||
- ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org
|
||||
- ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.openstack\.org
|
||||
- ^nb(?!03)[0-9]*\.open.*\.org
|
||||
- nl[0-9]*.open*.org
|
||||
- openstackid[0-9]*.openstack.org
|
||||
|
|
|
|||
|
|
@ -0,0 +1,6 @@
|
|||
iptables_extra_public_tcp_ports:
|
||||
- 80
|
||||
- 443
|
||||
- 8080
|
||||
- 8081
|
||||
- 8082
|
||||
|
|
@ -30,3 +30,6 @@
|
|||
import_tasks: touch_file.yaml
|
||||
vars:
|
||||
touch_file: '/tmp/letsencrypt02-main-service.stamp'
|
||||
|
||||
- name: letsencrypt updated mirror01-region-provider-opendev-org-main
|
||||
import_tasks: restart_apache.yaml
|
||||
|
|
|
|||
|
|
@ -0,0 +1,8 @@
|
|||
- name: Populate service facts
|
||||
service_facts:
|
||||
|
||||
- name: Restart apache
|
||||
service:
|
||||
name: apache2
|
||||
state: restarted
|
||||
when: "'apache2' in ansible_facts.services"
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
Configure an opendev mirror
|
||||
|
||||
This role installs and configures a mirror node
|
||||
|
||||
**Role Variables**
|
||||
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
mirror_root: '/afs/openstack.org/mirror'
|
||||
www_base: '/var/www'
|
||||
www_root: '{{ www_base }}/mirror'
|
||||
|
|
@ -0,0 +1,2 @@
|
|||
User-agent: *
|
||||
Disallow: /
|
||||
|
|
@ -0,0 +1,4 @@
|
|||
- name: restart apache2
|
||||
service:
|
||||
name: apache2
|
||||
state: restarted
|
||||
|
|
@ -0,0 +1,151 @@
|
|||
- name: Check AFS mounted
|
||||
stat:
|
||||
path: "/afs/openstack.org/mirror"
|
||||
register: afs_mirror
|
||||
- name: Sanity check AFS
|
||||
assert:
|
||||
that:
|
||||
- afs_mirror.stat.exists
|
||||
|
||||
- name: Install apache2
|
||||
apt:
|
||||
name:
|
||||
- apache2
|
||||
- apache2-utils
|
||||
state: present
|
||||
|
||||
- name: Rewrite module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: rewrite
|
||||
|
||||
- name: Substitute module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: substitute
|
||||
|
||||
- name: Cache module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: cache
|
||||
|
||||
- name: Cache disk module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: cache_disk
|
||||
|
||||
- name: Proxy module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: proxy
|
||||
|
||||
- name: HTTP Proxy module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: proxy_http
|
||||
|
||||
- name: Apache macro module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: macro
|
||||
|
||||
- name: Apache 2 ssl module
|
||||
apache2_module:
|
||||
state: present
|
||||
name: ssl
|
||||
|
||||
- name: Apache webroot
|
||||
file:
|
||||
path: '{{ www_base }}'
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
- name: Apache www root
|
||||
file:
|
||||
path: '{{ www_root }}'
|
||||
state: directory
|
||||
owner: root
|
||||
group: root
|
||||
|
||||
- name: AFS content symlinks
|
||||
file:
|
||||
src: '{{ mirror_root }}/{{ item }}'
|
||||
dest: '{{ www_root }}/{{ item }}'
|
||||
state: link
|
||||
owner: root
|
||||
group: root
|
||||
with_items:
|
||||
- centos
|
||||
- ceph-deb-hammer
|
||||
- ceph-deb-jewel
|
||||
- ceph-deb-luminous
|
||||
- ceph-deb-mimic
|
||||
- deb-docker
|
||||
- debian
|
||||
- debian-security
|
||||
- debian-openstack
|
||||
- epel
|
||||
- fedora
|
||||
- opensuse
|
||||
- ubuntu-ports
|
||||
- ubuntu-cloud-archive
|
||||
- wheel
|
||||
- yum-puppetlabs
|
||||
|
||||
- name: Install robots.txt
|
||||
copy:
|
||||
src: robots.txt
|
||||
dest: '{{ www_root }}'
|
||||
owner: root
|
||||
group: root
|
||||
mode: 0444
|
||||
|
||||
- name: Apache proxy cache
|
||||
file:
|
||||
path: /var/cache/apache2/proxy
|
||||
owner: www-data
|
||||
group: www-data
|
||||
mode: 0755
|
||||
state: directory
|
||||
|
||||
- name: Set mirror servername and alias
|
||||
set_fact:
|
||||
apache_server_name: '{{ inventory_hostname }}'
|
||||
# Strip the numeric host value from mirror01.region.provider.o.o
|
||||
# for the serveralias
|
||||
apache_server_alias: '{{ inventory_hostname | regex_replace("^mirror\d\d\.", "mirror.") }}'
|
||||
|
||||
- name: Create mirror virtual host
|
||||
template:
|
||||
src: mirror.vhost.j2
|
||||
dest: /etc/apache2/sites-available/mirror.conf
|
||||
|
||||
- name: Make sure default site disabled
|
||||
command: a2dissite 000-default.conf
|
||||
args:
|
||||
removes: /etc/apache2/sites-enabled/000-default.conf
|
||||
|
||||
- name: Enable mirror virtual host
|
||||
command: a2ensite mirror
|
||||
args:
|
||||
creates: /etc/apache2/sites-enabled/mirror.conf
|
||||
notify:
|
||||
- restart apache2
|
||||
|
||||
- name: Debug config
|
||||
slurp:
|
||||
src: /etc/apache2/sites-available/mirror.conf
|
||||
register: http_config
|
||||
- name: Show config
|
||||
debug:
|
||||
msg: '{{ http_config["content"] | b64decode }}'
|
||||
|
||||
# Clean apache cache once an hour, keep size down to 70GiB.
|
||||
- name: Proxy cleanup cron job
|
||||
cron:
|
||||
name: Apache cache cleanup
|
||||
state: present
|
||||
job: /usr/bin/flock -n /var/run/htcacheclean.lock /usr/bin/htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null
|
||||
minute: '0'
|
||||
hour: '*'
|
||||
|
|
@ -0,0 +1,404 @@
|
|||
NameVirtualHost *:80
|
||||
NameVirtualHost *:443
|
||||
|
||||
# Dedicated port for proxy caching, as not to affect afs mirrors.
|
||||
Listen 8080
|
||||
NameVirtualHost *:8080
|
||||
|
||||
Listen 8081
|
||||
NameVirtualHost *:8081
|
||||
|
||||
Listen 8082
|
||||
NameVirtualHost *:8082
|
||||
|
||||
LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
|
||||
|
||||
<Macro BaseProxy $port>
|
||||
|
||||
DocumentRoot /var/www/mirror
|
||||
<Directory /var/www/mirror>
|
||||
Options Indexes FollowSymLinks MultiViews
|
||||
AllowOverride None
|
||||
Order allow,deny
|
||||
allow from all
|
||||
Satisfy any
|
||||
<IfVersion >= 2.4>
|
||||
Require all granted
|
||||
</IfVersion>
|
||||
</Directory>
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
CacheStoreExpired On
|
||||
# Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
|
||||
# This means we don't use the cache for those requests. This setting
|
||||
# should force the proxy to ignore cache-control on the request side
|
||||
# but we should still cache things based on the cache-control responses
|
||||
# from the backed servers.
|
||||
CacheIgnoreCacheControl On
|
||||
|
||||
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
||||
# 0.05% of requests) of the type:
|
||||
#
|
||||
# End of file found: ... AH01102: error reading status line from remote server ...
|
||||
#
|
||||
# Per [1]:
|
||||
#
|
||||
# This avoids the "proxy: error reading status line from remote
|
||||
# server" error message caused by the race condition that the backend
|
||||
# server closed the pooled connection after the connection check by the
|
||||
# proxy and before data sent by the proxy reached the backend.
|
||||
#
|
||||
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
||||
SetEnv proxy-initial-not-pooled 1
|
||||
|
||||
RewriteEngine On
|
||||
# pypi
|
||||
CacheEnable disk "/pypi"
|
||||
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypi/" "https://pypi.org/
|
||||
|
||||
# files.pythonhosted.org
|
||||
CacheEnable disk "/pypifiles"
|
||||
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
||||
|
||||
# Rewrite the locations of the actual files
|
||||
<Location /pypi>
|
||||
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
||||
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
||||
</Location>
|
||||
|
||||
# Wheel URL's are:
|
||||
# /wheel/{distro}-{distro-version}/a/a/a-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
|
||||
# /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
|
||||
RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
|
||||
RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
|
||||
|
||||
# Special cases for openstack.nose_plugin & backports.*
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
|
||||
|
||||
# Try again but replacing -'s with .'s
|
||||
RewriteCond %{REQUEST_URI} ^/wheel/
|
||||
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
|
||||
RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
|
||||
RewriteRule (.*)-(.*) $1.$2 [N]
|
||||
|
||||
ErrorLog /var/log/apache2/proxy_$port_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
</Macro>
|
||||
|
||||
<VirtualHost *:80>
|
||||
ServerName {{ apache_server_name }}
|
||||
ServerAlias {{ apache_server_alias }}
|
||||
|
||||
Use BaseProxy 80
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName {{ apache_server_name }}
|
||||
ServerAlias {{ apache_server_alias }}
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer
|
||||
SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key
|
||||
SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer
|
||||
SSLProtocol All -SSLv2 -SSLv3
|
||||
# Note: this list should ensure ciphers that provide forward secrecy
|
||||
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
|
||||
SSLHonorCipherOrder on
|
||||
|
||||
Use BaseProxy 443
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:8080>
|
||||
ServerName {{ apache_server_name }}:8080
|
||||
ServerAlias {{ apache_server_alias }}:8080
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/apache2/proxy_8080_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/proxy_8080_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
CacheStoreExpired On
|
||||
|
||||
# Added Aug 2017 in an attempt to avoid occasional 502 errors (around
|
||||
# 0.05% of requests) of the type:
|
||||
#
|
||||
# End of file found: ... AH01102: error reading status line from remote server ...
|
||||
#
|
||||
# Per [1]:
|
||||
#
|
||||
# This avoids the "proxy: error reading status line from remote
|
||||
# server" error message caused by the race condition that the backend
|
||||
# server closed the pooled connection after the connection check by the
|
||||
# proxy and before data sent by the proxy reached the backend.
|
||||
#
|
||||
# [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
|
||||
SetEnv proxy-initial-not-pooled 1
|
||||
|
||||
# Per site caching reverse proxy rules
|
||||
# Only cache specific backends, rely on afs cache otherwise.
|
||||
|
||||
# buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
|
||||
CacheEnable disk "/buildlogs.centos"
|
||||
ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
|
||||
ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
|
||||
|
||||
# buildlogs.cdn.centos.org
|
||||
CacheEnable disk "/buildlogs.cdn.centos"
|
||||
ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
|
||||
ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
|
||||
|
||||
# rdo
|
||||
CacheEnable disk "/rdo"
|
||||
ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
|
||||
|
||||
# cbs.centos.org
|
||||
CacheEnable disk "/cbs.centos"
|
||||
ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"
|
||||
|
||||
# tarballs
|
||||
CacheEnable disk "/tarballs"
|
||||
ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
|
||||
|
||||
# pypi
|
||||
CacheEnable disk "/pypi"
|
||||
ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypi/" "https://pypi.org/
|
||||
|
||||
# files.pythonhosted.org
|
||||
CacheEnable disk "/pypifiles"
|
||||
ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
|
||||
|
||||
# Rewrite the locations of the actual files
|
||||
<Location /pypi>
|
||||
SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
|
||||
Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
|
||||
</Location>
|
||||
|
||||
# images.linuxcontainers.org
|
||||
CacheEnable disk "/images.linuxcontainers"
|
||||
ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
|
||||
|
||||
# registry.npmjs.org
|
||||
CacheEnable disk "/registry.npmjs"
|
||||
ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
|
||||
|
||||
# api.rubygems.org
|
||||
CacheEnable disk "/api.rubygems"
|
||||
ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
|
||||
|
||||
# rubygems.org
|
||||
CacheEnable disk "/rubygems"
|
||||
ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/rubygems/" "https://rubygems.org/"
|
||||
|
||||
# opendaylight
|
||||
CacheEnable disk "/opendaylight"
|
||||
ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
|
||||
|
||||
# elastico
|
||||
CacheEnable disk "/elastic"
|
||||
ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
|
||||
|
||||
# grafana
|
||||
CacheEnable disk "/grafana"
|
||||
ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
|
||||
|
||||
# OracleLinux
|
||||
CacheEnable disk "/oraclelinux"
|
||||
ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
|
||||
|
||||
# Percona
|
||||
CacheEnable disk "/percona"
|
||||
ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/percona/" "https://repo.percona.com/"
|
||||
|
||||
# MariaDB
|
||||
CacheEnable disk "/MariaDB"
|
||||
ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
|
||||
|
||||
# Docker
|
||||
CacheEnable disk "/docker"
|
||||
ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
|
||||
|
||||
# Alpine
|
||||
CacheEnable disk "/alpine"
|
||||
ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"
|
||||
|
||||
# LXC (copr)
|
||||
CacheEnable disk "/copr-lxc2"
|
||||
ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
|
||||
|
||||
</VirtualHost>
|
||||
|
||||
# Docker registry v1 proxy.
|
||||
<VirtualHost *:8081>
|
||||
ServerName {{ apache_server_name }}:8081
|
||||
ServerAlias {{ apache_server_alias }}:8081
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/apache2/proxy_8081_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/proxy_8081_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
||||
# last-modified date are provided default expire to 1 day. This is up from
|
||||
# 1 hour.
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# registry-1.docker.io
|
||||
CacheEnable disk "/registry-1.docker"
|
||||
ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
|
||||
|
||||
# dseasb33srnrn.cloudfront.net
|
||||
CacheEnable disk "/cloudfront"
|
||||
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
||||
|
||||
# production.cloudflare.docker.com
|
||||
CacheEnable disk "/cloudflare"
|
||||
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
||||
|
||||
</VirtualHost>
|
||||
|
||||
# Docker registry v2 proxy.
|
||||
<VirtualHost *:8082>
|
||||
ServerName {{ apache_server_name }}:8082
|
||||
ServerAlias {{ apache_server_alias }}:8082
|
||||
|
||||
# Disable directory listing by default.
|
||||
<Directory />
|
||||
Order Deny,Allow
|
||||
Deny from all
|
||||
Options None
|
||||
AllowOverride None
|
||||
</Directory>
|
||||
|
||||
ErrorLog /var/log/apache2/proxy_8082_error.log
|
||||
LogLevel warn
|
||||
CustomLog /var/log/apache2/proxy_8082_access.log combined-cache
|
||||
ServerSignature Off
|
||||
|
||||
# Caching reverse proxy for things that don't make sense in AFS
|
||||
#
|
||||
# General cache rules
|
||||
CacheRoot "/var/cache/apache2/proxy"
|
||||
CacheDirLevels 5
|
||||
CacheDirLength 2
|
||||
# SSL support
|
||||
SSLProxyEngine on
|
||||
# Prevent thundering herds.
|
||||
CacheLock on
|
||||
CacheLockPath "/tmp/mod_cache-lock"
|
||||
CacheLockMaxAge 5
|
||||
# 5GiB
|
||||
CacheMaxFileSize 5368709120
|
||||
# Ignore expire headers as the urls use sha256 hashes.
|
||||
CacheIgnoreQueryString On
|
||||
# NOTE(pabelanger): In the case of docker, if neither an expiry date nor
|
||||
# last-modified date are provided default expire to 1 day. This is up from
|
||||
# 1 hour.
|
||||
CacheDefaultExpire 86400
|
||||
CacheStoreExpired On
|
||||
|
||||
# dseasb33srnrn.cloudfront.net
|
||||
CacheEnable disk "/cloudfront"
|
||||
ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
|
||||
|
||||
# production.cloudflare.docker.com
|
||||
CacheEnable disk "/cloudflare"
|
||||
ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
|
||||
|
||||
# NOTE(corvus): Ensure this stanza is last since it's the most
|
||||
# greedy match.
|
||||
CacheEnable disk "/"
|
||||
ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
|
||||
ProxyPassReverse "/" "https://registry-1.docker.io/"
|
||||
</VirtualHost>
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
- hosts: "mirror_opendev:!disabled"
|
||||
name: "Configure per region opendev mirrors"
|
||||
roles:
|
||||
- role: kerberos-client
|
||||
kerberos_realm: 'OPENSTACK.ORG'
|
||||
kerberos_admin_server: 'kdc.openstack.org'
|
||||
kerberos_kdcs:
|
||||
- kdc03.openstack.org
|
||||
- kdc04.openstack.org
|
||||
- role: openafs-client
|
||||
- role: mirror
|
||||
|
|
@ -81,6 +81,7 @@
|
|||
- host_vars/bridge.openstack.org.yaml
|
||||
- host_vars/letsencrypt01.opendev.org.yaml
|
||||
- host_vars/letsencrypt02.opendev.org.yaml
|
||||
- host_vars/mirror01.region.provider.opendev.org.yaml
|
||||
- name: Display group membership
|
||||
command: ansible localhost -m debug -a 'var=groups'
|
||||
- name: Run base.yaml
|
||||
|
|
|
|||
|
|
@ -8,3 +8,4 @@ groups:
|
|||
letsencrypt:
|
||||
- letsencrypt01.opendev.org
|
||||
- letsencrypt02.opendev.org
|
||||
- mirror01.region.provider.opendev.org
|
||||
|
|
|
|||
|
|
@ -0,0 +1,4 @@
|
|||
letsencrypt_certs:
|
||||
mirror01-region-provider-opendev-org-main:
|
||||
- mirror01.region.provider.opendev.org
|
||||
- mirror.region.provider.opendev.org
|
||||
|
|
@ -100,6 +100,10 @@ start_timer
|
|||
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
|
||||
send_timer nodepool
|
||||
|
||||
start_timer
|
||||
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
|
||||
send_timer nodepool
|
||||
|
||||
start_timer
|
||||
timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
|
||||
send_timer registry
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
# Copyright 2019 Red Hat, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
|
||||
testinfra_hosts = ['mirror01.region.provider.opendev.org']
|
||||
|
||||
|
||||
def test_apache(host):
|
||||
apache = host.service('apache2')
|
||||
assert apache.is_running
|
||||
|
||||
def test_mirror_indexes(host):
|
||||
cmd = host.run("wget --no-check-certificate -qO- https://localhost/")
|
||||
assert '<a href="debian/">' in cmd.stdout
|
||||
|
||||
cmd = host.run("wget -qO- http://localhost/")
|
||||
assert '<a href="debian/">' in cmd.stdout
|
||||
|
||||
# NOTE(ianw): further testing idea for anyone interested; get the
|
||||
# actual IP address of the mirror node and connect via that, and then
|
||||
# also poke at the other proxy ports
|
||||
Loading…
Reference in New Issue