diff --git a/inventory/service/group_vars/mirror.yaml b/inventory/service/group_vars/mirror.yaml index 578e8f72cb..c0db9e5de1 100644 --- a/inventory/service/group_vars/mirror.yaml +++ b/inventory/service/group_vars/mirror.yaml @@ -5,7 +5,9 @@ iptables_extra_public_tcp_ports: - 4445 - 4446 - 4447 + - 4448 - 8080 - 8082 - 8083 - 8084 + - 8085 diff --git a/playbooks/roles/mirror/templates/mirror.vhost.j2 b/playbooks/roles/mirror/templates/mirror.vhost.j2 index 660e9304ae..e42c003361 100644 --- a/playbooks/roles/mirror/templates/mirror.vhost.j2 +++ b/playbooks/roles/mirror/templates/mirror.vhost.j2 @@ -22,6 +22,11 @@ NameVirtualHost *:8084 Listen 4447 NameVirtualHost *:4447 +Listen 8085 +NameVirtualHost *:8085 +Listen 4448 +NameVirtualHost *:4448 + {% raw %} LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ referer\ %{Referer}i" @@ -124,13 +129,6 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d RewriteRule (.*)-(.*) $1.$2 [N] - # Ansible Galaxy - CacheEnable disk "/galaxy" - ProxyPass "/galaxy/" "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/galaxy/" "https://galaxy.ansible.com/" - CacheEnable disk "/galaxy-s3" - ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0 - ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ErrorLog /var/log/apache2/mirror_$port_error.log LogLevel warn @@ -544,3 +542,70 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ Use SSLConfig Use QuayRegistryMirror 4447 + +# ansible-galaxy has some non-proxy-friendly redirects, so we need to get a +# dedicated vhost on a dedicated port, in order to use its / instead of a +# subdirectory. + + # Let upstreams decide on encoded slash handling. + # The default is 'Off' which returns 404 for URLs with encoded slashes, + # i.e. '%2f' instead of '/'. + AllowEncodedSlashes NoDecode + + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + CacheStoreExpired On + + + ProxyPass "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "https://galaxy.ansible.com/" + SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE + SubstituteMaxLineLength 20m + # ansible-galaxy CLI needs a fully qualified URI. So we must take care + # of the REQUEST_SCHEME. Note that mod_substitute can't use parameters... + + Substitute "s|https://galaxy.ansible.com/|https://{{ apache_server_name }}:$port/|ni" + Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|https://{{ apache_server_name }}:$port/galaxy-s3/|ni" + + + Substitute "s|https://galaxy.ansible.com/|http://{{ apache_server_name }}:$port/|ni" + Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|http://{{ apache_server_name }}:$port/galaxy-s3/|ni" + + + ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" + + ErrorLog /var/log/apache2/proxy_$port_error.log + LogLevel warn + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache + ServerSignature Off + + AddType text/plain .log .log.1 + + + + ServerName {{ apache_server_name }}:8085 + ServerAlias {{ apache_server_alias }}:8085 + + Use AnsibleGalaxy 8085 + + + + ServerName {{ apache_server_name }}:4448 + ServerAlias {{ apache_server_alias }}:4448 + + Use SSLConfig + Use AnsibleGalaxy 4448 + diff --git a/testinfra/test_mirror.py b/testinfra/test_mirror.py index dc9609cc48..f5489924dc 100644 --- a/testinfra/test_mirror.py +++ b/testinfra/test_mirror.py @@ -13,10 +13,12 @@ # under the License. +import json + + testinfra_hosts = ['mirror01.openafs.provider.opendev.org', 'mirror02.openafs.provider.opendev.org'] - def test_apache(host): apache = host.service('apache2') assert apache.is_running @@ -64,9 +66,24 @@ def test_quay_mirror(host): # TODO test RHRegistryMirror def test_galaxy_mirror(host): - cmd = host.run(_run_cmd(host, 443, url='/galaxy/')) + cmd = host.run(_run_cmd(host, 4448, url='/')) assert 'Ansible Galaxy' in cmd.stdout - cmd = host.run(_run_cmd(host, 80, scheme='http', url='/galaxy/')) + cmd = host.run(_run_cmd(host, 8085, scheme='http', url='/')) assert 'Ansible Galaxy' in cmd.stdout + hostname = host.backend.get_hostname() + # Ensure API properly answers + cmd = host.run(_run_cmd(host, 4448, url='/api/')) + assert 'GALAXY REST API' in cmd.stdout + # Ensure we get data out of a specific collection + cmd = host.run(_run_cmd(host, 4448, url='/api/v2/collections/community/general/')) + assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout + answer = json.loads(cmd.stdout) + version_uri = answer['latest_version']['href'].replace('https://{}:4448'.format(hostname), '') + # Ensure we get a correct download URI + cmd = host.run(_run_cmd(host, 4448, url=version_uri)) + assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout + answer = json.loads(cmd.stdout) + download_uri = answer['download_url'] + assert download_uri.startswith('https://{}:4448/download/community-general'.format(hostname))