Enable ssl on all mirror vhosts

Previously we had enabled SSL on our main vhost for the mirrors. Do
similar for all of the proxy cache vhosts for docker and other external
resources.

As part of this change we improve the testing to ensure that the new
vhosts are working as expected. One testing specific change to note is
the testinfra node names did not match our existing system-config-run
job nodenames. This has been corrected.

Additionally RHRegistryMirror and QuayMirror may not be working and
fixing those is left as a followup.

Change-Id: I9dbbd4080c3a2cce4acc39d63244f7a645503553

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -4,15 +4,23 @@ NameVirtualHost *:443
 # Dedicated port for proxy caching, as not to affect afs mirrors.
 Listen 8080
 NameVirtualHost *:8080
+Listen 4443
+NameVirtualHost *:4443
 
 Listen 8081
 NameVirtualHost *:8081
+Listen 4444
+NameVirtualHost *:4444
 
 Listen 8082
 NameVirtualHost *:8082
+Listen 4445
+NameVirtualHost *:4445
 
 Listen 8083
 NameVirtualHost *:8083
+Listen 4446
+NameVirtualHost *:4446
 
 {% raw %}
 LogFormat "%h %l %u [%{%F %T}t.%{msec_frac}t] \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
@@ -116,6 +124,17 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
 
 </Macro>
 
+<Macro SSLConfig>
+    SSLEngine On
+    SSLCertificateFile      /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer
+    SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key
+    SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer
+    SSLProtocol All -SSLv2 -SSLv3
+    # Note: this list should ensure ciphers that provide forward secrecy
+    SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+    SSLHonorCipherOrder on
+</Macro>
+
 <VirtualHost *:80>
     ServerName {{ apache_server_name }}
     ServerAlias {{ apache_server_alias }}
@@ -127,21 +146,11 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     ServerName {{ apache_server_name }}
     ServerAlias {{ apache_server_alias }}
 
-    SSLCertificateFile      /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer
+    Use SSLConfig
-    SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key
-    SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer
-    SSLProtocol All -SSLv2 -SSLv3
-    # Note: this list should ensure ciphers that provide forward secrecy
-    SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
-    SSLHonorCipherOrder on
-
     Use BaseMirror 443
 </VirtualHost>
 
-<VirtualHost *:8080>
+<Macro ProxyMirror $port>
-    ServerName {{ apache_server_name }}:8080
-    ServerAlias {{ apache_server_alias }}:8080
-
     # Disable directory listing by default.
     <Directory />
       Order Deny,Allow
@@ -150,9 +159,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       AllowOverride None
     </Directory>
 
-    ErrorLog /var/log/apache2/proxy_8080_error.log
+    ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn
-    CustomLog /var/log/apache2/proxy_8080_access.log combined-cache
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
     ServerSignature Off
 
     # Let upstreams decide on encoded slash handling.
@@ -294,14 +303,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     CacheEnable disk "/copr-lxc2"
     ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
     ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
+</Macro>
 
+<VirtualHost *:8080>
+    ServerName {{ apache_server_name }}:8080
+    ServerAlias {{ apache_server_alias }}:8080
+
+    Use ProxyMirror 8080
+</VirtualHost>
+
+<VirtualHost *:4443>
+    ServerName {{ apache_server_name }}:4443
+    ServerAlias {{ apache_server_alias }}:4443
+
+    Use SSLConfig
+    Use ProxyMirror 4443
 </VirtualHost>
 
 # Docker registry v1 proxy.
-<VirtualHost *:8081>
+<Macro Dockerv1Mirror $port>
-    ServerName {{ apache_server_name }}:8081
-    ServerAlias {{ apache_server_alias }}:8081
-
     # Disable directory listing by default.
     <Directory />
       Order Deny,Allow
@@ -310,9 +330,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       AllowOverride None
     </Directory>
 
-    ErrorLog /var/log/apache2/proxy_8081_error.log
+    ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn
-    CustomLog /var/log/apache2/proxy_8081_access.log combined-cache
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
     ServerSignature Off
 
     # Caching reverse proxy for things that don't make sense in AFS
@@ -351,14 +371,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     CacheEnable disk "/cloudflare"
     ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
     ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
+</Macro>
 
+<VirtualHost *:8081>
+    ServerName {{ apache_server_name }}:8081
+    ServerAlias {{ apache_server_alias }}:8081
+
+    Use Dockerv1Mirror 8081
+</VirtualHost>
+
+<VirtualHost *:4444>
+    ServerName {{ apache_server_name }}:4444
+    ServerAlias {{ apache_server_alias }}:4444
+
+    Use SSLConfig
+    Use Dockerv1Mirror 4444
 </VirtualHost>
 
 # Docker registry v2 proxy.
-<VirtualHost *:8082>
+<Macro Dockerv2Mirror $port>
-    ServerName {{ apache_server_name }}:8082
-    ServerAlias {{ apache_server_alias }}:8082
-
     # Disable directory listing by default.
     <Directory />
       Order Deny,Allow
@@ -367,9 +398,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       AllowOverride None
     </Directory>
 
-    ErrorLog /var/log/apache2/proxy_8082_error.log
+    ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn
-    CustomLog /var/log/apache2/proxy_8082_access.log combined-cache
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
     ServerSignature Off
 
     # Caching reverse proxy for things that don't make sense in AFS
@@ -409,13 +440,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     CacheEnable disk  "/"
     ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
     ProxyPassReverse "/" "https://registry-1.docker.io/"
+</Macro>
+
+<VirtualHost *:8082>
+    ServerName {{ apache_server_name }}:8082
+    ServerAlias {{ apache_server_alias }}:8082
+
+    Use Dockerv2Mirror 8082
+</VirtualHost>
+
+<VirtualHost *:4445>
+    ServerName {{ apache_server_name }}:4445
+    ServerAlias {{ apache_server_alias }}:4445
+
+    Use SSLConfig
+    Use Dockerv2Mirror 4445
 </VirtualHost>
 
 # Redhat registry proxy.
-<VirtualHost *:8083>
+<Macro RHRegistryMirror $port>
-    ServerName {{ apache_server_name }}:8083
-    ServerAlias {{ apache_server_alias }}:8083
-
     # Disable directory listing by default.
     <Directory />
       Order Deny,Allow
@@ -424,9 +467,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       AllowOverride None
     </Directory>
 
-    ErrorLog /var/log/apache2/proxy_8083_error.log
+    ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn
-    CustomLog /var/log/apache2/proxy_8083_access.log combined-cache
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
     ServerSignature Off
 
     # Caching reverse proxy for things that don't make sense in AFS
@@ -462,12 +505,25 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     CacheEnable disk "/"
     ProxyPass "/" "https://registry.access.redhat.com/" ttl=120 keepalive=On retry=0
     ProxyPassReverse "/" "https://registry.access.redhat.com/"
+</Macro>
+
+<VirtualHost *:8083>
+    ServerName {{ apache_server_name }}:8083
+    ServerAlias {{ apache_server_alias }}:8083
+
+    Use RHRegistryMirror 8083
+</VirtualHost>
+
+<VirtualHost *:4446>
+    ServerName {{ apache_server_name }}:4446
+    ServerAlias {{ apache_server_alias }}:4446
+
+    Use SSLConfig
+    Use RHRegistryMirror 4446
 </VirtualHost>
 
 # Quay registry proxy.
-<VirtualHost *:8084>
+<Macro QuayRegistryMirror $port>
-    ServerName {{ apache_server_name }}:8084
-    ServerAlias {{ apache_server_alias }}:8084
 
     # Disable directory listing by default.
     <Directory />
@@ -477,9 +533,9 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
       AllowOverride None
     </Directory>
 
-    ErrorLog /var/log/apache2/proxy_8083_error.log
+    ErrorLog /var/log/apache2/proxy_$port_error.log
     LogLevel warn
-    CustomLog /var/log/apache2/proxy_8083_access.log combined-cache
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
     ServerSignature Off
 
     # Caching reverse proxy for things that don't make sense in AFS
@@ -510,4 +566,19 @@ ErrorLogFormat "[%{cu}t] [%-m:%l] [pid %P:tid %T] %7F: %E: [client\ %a] %M% , \
     CacheEnable disk "/"
     ProxyPass "/" "https://quay.io/" ttl=120 keepalive=On retry=0
     ProxyPassReverse "/" "https://quay.io/"
+</Macro>
+
+<VirtualHost *:8084>
+    ServerName {{ apache_server_name }}:8084
+    ServerAlias {{ apache_server_alias }}:8084
+
+    Use QuayRegistryMirror 8084
+</VirtualHost>
+
+<VirtualHost *:4447>
+    ServerName {{ apache_server_name }}:4447
+    ServerAlias {{ apache_server_alias }}:4447
+
+    Use SSLConfig
+    Use QuayRegistryMirror 4447
 </VirtualHost>

testinfra/test_mirror.py

@@ -13,21 +13,52 @@
 # under the License.
 
 
-testinfra_hosts = ['mirror01.region.provider.opendev.org',
+testinfra_hosts = ['mirror01.openafs.provider.opendev.org',
-                   'mirror02.region.provider.opendev.org']
+                   'mirror02.openafs.provider.opendev.org']
 
 
 def test_apache(host):
     apache = host.service('apache2')
     assert apache.is_running
 
-def test_mirror_indexes(host):
+def test_base_mirror(host):
+    # BaseMirror
     cmd = host.run("wget --no-check-certificate -qO- https://localhost/")
     assert '<a href="debian/">' in cmd.stdout
 
     cmd = host.run("wget -qO- http://localhost/")
     assert '<a href="debian/">' in cmd.stdout
 
+def test_proxy_mirror(host):
+    # ProxyMirror
+    cmd = host.run("wget --no-check-certificate -qO- "
+                   "https://localhost:4443/pypi/simple/setuptools")
+    assert 'setuptools' in cmd.stdout
+
+    cmd = host.run("wget -qO- http://localhost:8080/pypi/simple/setuptools")
+    assert 'setuptools' in cmd.stdout
+
+def test_dockerv1_mirror(host):
+    # Dockerv1Mirror
+    cmd = host.run("wget --no-check-certificate -O- "
+                   "https://localhost:4444/registry-1.docker")
+    # TODO assert that this proxy cache is working more properly
+    assert '403 Forbidden' in cmd.stderr
+
+    cmd = host.run("wget -O- http://localhost:8081/registry-1.docker")
+    # TODO assert that this proxy cache is working more properly
+    assert '403 Forbidden' in cmd.stderr
+
+def test_dockerv2_mirror(host):
+    # Dockerv2Mirror
+    cmd = host.run("wget --no-check-certificate -O- "
+                   "https://localhost:4445/v2/")
+    assert '401 Unauthorized' in cmd.stderr
+
+    cmd = host.run("wget -O- http://localhost:8082/v2/")
+    assert '401 Unauthorized' in cmd.stderr
+
+# TODO test RHRegistryMirror and QuayMirror
+
 # NOTE(ianw): further testing idea for anyone interested; get the
-# actual IP address of the mirror node and connect via that, and then
+# actual IP address of the mirror node and connect via that
-# also poke at the other proxy ports

zuul.d/system-config-run.yaml

@@ -347,6 +347,12 @@
         host_copy_output:
           '/var/log/apache2/': logs
           '/var/log/acme.sh': logs
+          '/etc/apache2/sites-available/mirror.conf': logs
+      mirror02.openafs.provider.opendev.org:
+        host_copy_output:
+          '/var/log/apache2/': logs
+          '/var/log/acme.sh': logs
+          '/etc/apache2/sites-available/mirror.conf': logs
     files:
       - playbooks/install-ansible.yaml
       - roles/