Correct how ansible-galaxy is proxified

ansible-galaxy CLI makes multiple calls to the remote server, with
various API endpoint, and expects JSON containing fully qualified URI
(scheme://host/path), meaning we must inspect the different files and
ensure we're rewriting the content so that it points to the proxy all
the time.

Also, the remote galaxy.ansible.com has some redirects with absolute
paths, breaking for some reason the ProxyPassReverse - this is why we
get yet a new pair of dedicated ports for this proxy (TLS/non-TLS).

Then, there's the protocol issue: since mod_substitute is apparently
unable to take httpd variables such as the REQUEST_SCHEME, we have to
use some If statement in order to ensure we're passing the correct
scheme, being http or https. Note that ansible-galaxy doesn't understand
the "//host/path".

This patch also adds some more tests in order to ensure the API answers
as expected through the proxy.

Change-Id: Icf6f5c83554b51854fabde6e4cc2d646d120c0e9

inventory/service/group_vars/mirror.yaml

@@ -5,7 +5,9 @@ iptables_extra_public_tcp_ports:
   - 4445
   - 4446
   - 4447
+  - 4448
   - 8080
   - 8082
   - 8083
   - 8084
+  - 8085

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -22,6 +22,11 @@ NameVirtualHost *:8084
 Listen 4447
 NameVirtualHost *:4447
 
+Listen 8085
+NameVirtualHost *:8085
+Listen 4448
+NameVirtualHost *:4448
+
 {% raw %}
 LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
 ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \ referer\ %{Referer}i"
@@ -124,13 +129,6 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
     RewriteRule (.*)-(.*) $1.$2 [N]
 
-    # Ansible Galaxy
-    CacheEnable disk "/galaxy"
-    ProxyPass "/galaxy/" "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0
-    ProxyPassReverse "/galaxy/" "https://galaxy.ansible.com/"
-    CacheEnable disk "/galaxy-s3"
-    ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
-    ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/"
 
     ErrorLog /var/log/apache2/mirror_$port_error.log
     LogLevel warn
@@ -544,3 +542,70 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     Use SSLConfig
     Use QuayRegistryMirror 4447
 </VirtualHost>
+
+# ansible-galaxy has some non-proxy-friendly redirects, so we need to get a
+# dedicated vhost on a dedicated port, in order to use its / instead of a
+# subdirectory.
+<Macro AnsibleGalaxy $port>
+    # Let upstreams decide on encoded slash handling.
+    # The default is 'Off' which returns 404 for URLs with encoded slashes,
+    # i.e. '%2f' instead of '/'.
+    AllowEncodedSlashes NoDecode
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+
+    <Location "/">
+      ProxyPass "https://galaxy.ansible.com/" ttl=120 keepalive=On retry=0
+      ProxyPassReverse "https://galaxy.ansible.com/"
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      SubstituteMaxLineLength 20m
+      # ansible-galaxy CLI needs a fully qualified URI. So we must take care
+      # of the REQUEST_SCHEME. Note that mod_substitute can't use parameters...
+      <If "-T %{HTTPS}">
+        Substitute "s|https://galaxy.ansible.com/|https://{{ apache_server_name }}:$port/|ni"
+        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|https://{{ apache_server_name }}:$port/galaxy-s3/|ni"
+      </If>
+      <If "! -T %{HTTPS}">
+        Substitute "s|https://galaxy.ansible.com/|http://{{ apache_server_name }}:$port/|ni"
+        Substitute "s|https://ansible-galaxy.s3.amazonaws.com/|http://{{ apache_server_name }}:$port/galaxy-s3/|ni"
+      </If>
+    </Location>
+    ProxyPass "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/galaxy-s3/" "https://ansible-galaxy.s3.amazonaws.com/"
+
+    ErrorLog /var/log/apache2/proxy_$port_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
+    ServerSignature Off
+
+    AddType text/plain .log .log.1
+</Macro>
+
+<VirtualHost *:8085>
+    ServerName {{ apache_server_name }}:8085
+    ServerAlias {{ apache_server_alias }}:8085
+
+    Use AnsibleGalaxy 8085
+</VirtualHost>
+
+<VirtualHost *:4448>
+    ServerName {{ apache_server_name }}:4448
+    ServerAlias {{ apache_server_alias }}:4448
+
+    Use SSLConfig
+    Use AnsibleGalaxy 4448
+</VirtualHost>

testinfra/test_mirror.py

@@ -13,10 +13,12 @@
 # under the License.
 
 
+import json
+
+
 testinfra_hosts = ['mirror01.openafs.provider.opendev.org',
                    'mirror02.openafs.provider.opendev.org']
 
-
 def test_apache(host):
     apache = host.service('apache2')
     assert apache.is_running
@@ -64,9 +66,24 @@ def test_quay_mirror(host):
 # TODO test RHRegistryMirror
 
 def test_galaxy_mirror(host):
-    cmd = host.run(_run_cmd(host, 443, url='/galaxy/'))
+    cmd = host.run(_run_cmd(host, 4448, url='/'))
     assert 'Ansible Galaxy' in cmd.stdout
 
-    cmd = host.run(_run_cmd(host, 80, scheme='http', url='/galaxy/'))
+    cmd = host.run(_run_cmd(host, 8085, scheme='http', url='/'))
     assert 'Ansible Galaxy' in cmd.stdout
 
+    hostname = host.backend.get_hostname()
+    # Ensure API properly answers
+    cmd = host.run(_run_cmd(host, 4448, url='/api/'))
+    assert 'GALAXY REST API' in cmd.stdout
+    # Ensure we get data out of a specific collection
+    cmd = host.run(_run_cmd(host, 4448, url='/api/v2/collections/community/general/'))
+    assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout
+    answer = json.loads(cmd.stdout)
+    version_uri = answer['latest_version']['href'].replace('https://{}:4448'.format(hostname), '')
+    # Ensure we get a correct download URI
+    cmd = host.run(_run_cmd(host, 4448, url=version_uri))
+    assert 'https://{}:4448/api/'.format(hostname) in cmd.stdout
+    answer = json.loads(cmd.stdout)
+    download_uri = answer['download_url']
+    assert download_uri.startswith('https://{}:4448/download/community-general'.format(hostname))