Merge "Correct how ansible-galaxy is proxified"

inventory/service/group_vars/mirror.yaml

@@ -5,7 +5,9 @@ iptables_extra_public_tcp_ports:
   - 4445
   - 4446
   - 4447
+  - 4448
   - 8080
   - 8082
   - 8083
   - 8084
+  - 8085

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -22,6 +22,11 @@ NameVirtualHost *:8084
 Listen 4447
 NameVirtualHost *:4447
 
+Listen 8085
+NameVirtualHost *:8085
+Listen 4448
+NameVirtualHost *:4448
+
 {% raw %}
 LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
 ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ referer\ %{Referer}i"
@@ -124,13 +129,6 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
     RewriteRule (.*)-(.*) $1.$2 [N]
 
-    # Ansible Galaxy
-    CacheEnable disk "/galaxy"
-    ProxyPass "/galaxy/" "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0
-    ProxyPassReverse "/galaxy/" "https://galaxy.ansible.com/"
-    CacheEnable disk "/galaxy-s3"
-    ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
-    ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/"
 
     ErrorLog /var/log/apache2/mirror_$port_error.log
     LogLevel warn
@@ -544,3 +542,70 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     Use SSLConfig
     Use QuayRegistryMirror 4447
 </VirtualHost>
+
+# ansible-galaxy has some non-proxy-friendly redirects, so we need to get a
+# dedicated vhost on a dedicated port, in order to use its / instead of a
+# subdirectory.
+<Macro AnsibleGalaxy $port>
+    # Let upstreams decide on encoded slash handling.
+    # The default is 'Off' which returns 404 for URLs with encoded slashes,
+    # i.e. '%2f' instead of '/'.
+    AllowEncodedSlashes NoDecode
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+
+    <Location "/">
+      ProxyPass "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0
+      ProxyPassReverse "https://galaxy.ansible.com/"
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      SubstituteMaxLineLength 20m
+      # ansible-galaxy CLI needs a fully qualified URI. So we must take care
+      # of the REQUEST_SCHEME. Note that mod_substitute can't use parameters...
+      <If "-T %{HTTPS}">
+        Substitute "s|https://galaxy.ansible.com/|https://{{ apache_server_name }}:$port/|ni"
+        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|https://{{ apache_server_name }}:$port/galaxy-s3/|ni"
+      </If>
+      <If "! -T %{HTTPS}">
+        Substitute "s|https://galaxy.ansible.com/|http://{{ apache_server_name }}:$port/|ni"
+        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|http://{{ apache_server_name }}:$port/galaxy-s3/|ni"
+      </If>
+    </Location>
+    ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/"
+
+    ErrorLog /var/log/apache2/proxy_$port_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
+    ServerSignature Off
+
+    AddType text/plain .log .log.1
+</Macro>
+
+<VirtualHost *:8085>
+    ServerName {{ apache_server_name }}:8085
+    ServerAlias {{ apache_server_alias }}:8085
+
+    Use AnsibleGalaxy 8085
+</VirtualHost>
+
+<VirtualHost *:4448>
+    ServerName {{ apache_server_name }}:4448
+    ServerAlias {{ apache_server_alias }}:4448
+
+    Use SSLConfig
+    Use AnsibleGalaxy 4448
+</VirtualHost>

testinfra/test_mirror.py

@@ -13,10 +13,12 @@
 # under the License.
 
 
+import json
+
+
 testinfra_hosts = ['mirror01.openafs.provider.opendev.org',
                    'mirror02.openafs.provider.opendev.org']
 
-
 def test_apache(host):
     apache = host.service('apache2')
     assert apache.is_running
@@ -64,9 +66,24 @@ def test_quay_mirror(host):
 # TODO test RHRegistryMirror
 
 def test_galaxy_mirror(host):
-    cmd = host.run(_run_cmd(host, 443, url='/galaxy/'))
+    cmd = host.run(_run_cmd(host, 4448, url='/'))
     assert 'Ansible Galaxy' in cmd.stdout
 
-    cmd = host.run(_run_cmd(host, 80, scheme='http', url='/galaxy/'))
+    cmd = host.run(_run_cmd(host, 8085, scheme='http', url='/'))
     assert 'Ansible Galaxy' in cmd.stdout
 
+    hostname = host.backend.get_hostname()
+    # Ensure API properly answers
+    cmd = host.run(_run_cmd(host, 4448, url='/api/'))
+    assert 'GALAXY REST API' in cmd.stdout
+    # Ensure we get data out of a specific collection
+    cmd = host.run(_run_cmd(host, 4448, url='/api/v2/collections/community/general/'))
+    assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout
+    answer = json.loads(cmd.stdout)
+    version_uri = answer['latest_version']['href'].replace('https://{}:4448'.format(hostname), '')
+    # Ensure we get a correct download URI
+    cmd = host.run(_run_cmd(host, 4448, url=version_uri))
+    assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout
+    answer = json.loads(cmd.stdout)
+    download_uri = answer['download_url']
+    assert download_uri.startswith('https://{}:4448/download/community-general'.format(hostname))