Create opendev mirrors

This impelements mirrors to live in the opendev.org namespace.  The
implementation is Ansible native for deployment on a Bionic node.

The hostname prefix remains the same (mirrorXX.region.provider.) but
the groups.yaml splits the opendev.org mirrors into a separate group.
The matches in the puppet group are also updated so to not run puppet
on the hosts.

The kerberos and openafs client parts do not need any updating and
works on the Bionic host.

The hosts are setup to provision certificates for themselves from
letsencrypt.  Note we've added a new handler for mirror nodes to use
that restarts apache on certificate issue/renewal.

The new "mirror" role is a port of the existing puppet mirror.pp.  It
installs apache, sets up some modules, makes some symlinks, sets up a
cleanup cron job and installs the apache vhost configuration.

The vhost configuration is also ported from the extant puppet.  It is
simplified somewhat; but the biggest change is that we have extracted
the main port 80 configuration into a macro which is applied to both
port 80 and 443; i.e. the host will have SSL support.  The other ports
are left alone for now, but can be updated in due course.

Thus we should be able to CNAME the existing mirrors to new nodes, and
any existing http access can continue.  We can update our mirror setup
scripts to point to https resources as appropriate.

Change-Id: Iec576d631dd5b02f6b9fb445ee600be060f9cf1e

.zuul.yaml

@@ -481,6 +481,35 @@
       - testinfra/test_adns.py
       - testinfra/test_ns.py
 
+- job:
+    name: system-config-run-mirror
+    parent: system-config-run
+    description: |
+      Run the playbook for a mirror node
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: mirror01.region.provider.opendev.org
+          label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-letsencrypt.yaml
+        - playbooks/service-mirror.yaml
+    files:
+      - .zuul.yaml
+      - roles/
+      - playbooks/roles/mirror/
+      - playbooks/roles/letsencrypt.*
+      - playbooks/service-letsencrypt.yaml
+      - playbooks/service-mirror.yaml
+      - testinfra/test_mirror.py
+
+    host-vars:
+      mirror.region.provider.opendev.org:
+        host_copy_output:
+          '/var/log/apache2/': logs
+
 - job:
     name: system-config-run-docker-registry
     parent: system-config-run
@@ -615,6 +644,7 @@
         - system-config-run-dns
         - system-config-run-eavesdrop
         - system-config-run-nodepool
+        - system-config-run-mirror
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:

inventory/groups.yaml

@@ -57,6 +57,7 @@ groups:
     - opendev-k8s*.opendev.org
   letsencrypt:
     - graphite01.opendev.org
+    - mirror[0-9]*.opendev.org
   logstash:
     - logstash[0-9]*.open*.org
   logstash-worker:
@@ -65,7 +66,9 @@ groups:
     - lists*.katacontainers.io
     - lists*.open*.org
   mirror:
-    - mirror[0-9]*.open*.org
+    - mirror[0-9]*.openstack.org
+  mirror_opendev:
+    - mirror[0-9]*.opendev.org
   nodepool:
     - nb[0-9]*.open*.org
     - nl[0-9]*.open*.org
@@ -110,7 +113,7 @@ groups:
     - logstash-worker[0-9]*.open*.org
     - logstash[0-9]*.open*.org
     - mirror-update[0-9]*.open*.org
-    - mirror[0-9]*.open*.org
+    - mirror[0-9]*.openstack.org
     - nb[0-9]*.open*.org
     - nl[0-9]*.open*.org
     - openstackid-dev*.openstack.org
@@ -161,7 +164,7 @@ groups:
     - logstash-worker[0-9]*.open*.org
     - logstash[0-9]*.open*.org
     - mirror-update[0-9]*.open*.org
-    - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org
+    - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.openstack\.org
     - ^nb(?!03)[0-9]*\.open.*\.org
     - nl[0-9]*.open*.org
     - openstackid[0-9]*.openstack.org

playbooks/group_vars/mirror_opendev.yaml

@@ -0,0 +1,6 @@
+iptables_extra_public_tcp_ports:
+  - 80
+  - 443
+  - 8080
+  - 8081
+  - 8082

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -30,3 +30,6 @@
   import_tasks: touch_file.yaml
   vars:
     touch_file: '/tmp/letsencrypt02-main-service.stamp'
+
+- name: letsencrypt updated mirror01-region-provider-opendev-org-main
+  import_tasks: restart_apache.yaml

playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml

@@ -0,0 +1,8 @@
+- name: Populate service facts
+  service_facts:
+
+- name: Restart apache
+  service:
+    name: apache2
+    state: restarted
+  when: "'apache2' in ansible_facts.services"

playbooks/roles/mirror/README.rst

@@ -0,0 +1,6 @@
+Configure an opendev mirror
+
+This role installs and configures a mirror node
+
+**Role Variables**
+

playbooks/roles/mirror/defaults/main.yaml

@@ -0,0 +1,3 @@
+mirror_root: '/afs/openstack.org/mirror'
+www_base: '/var/www'
+www_root: '{{ www_base }}/mirror'

playbooks/roles/mirror/files/robots.txt

@@ -0,0 +1,2 @@
+User-agent: *
+Disallow: /

playbooks/roles/mirror/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: restart apache2
+  service:
+    name: apache2
+    state: restarted

playbooks/roles/mirror/tasks/main.yaml

@@ -0,0 +1,151 @@
+- name: Check AFS mounted
+  stat:
+    path: "/afs/openstack.org/mirror"
+  register: afs_mirror
+- name: Sanity check AFS
+  assert:
+    that:
+      - afs_mirror.stat.exists
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Rewrite module
+  apache2_module:
+    state: present
+    name: rewrite
+
+- name: Substitute module
+  apache2_module:
+    state: present
+    name: substitute
+
+- name: Cache module
+  apache2_module:
+    state: present
+    name: cache
+
+- name: Cache disk module
+  apache2_module:
+    state: present
+    name: cache_disk
+
+- name: Proxy module
+  apache2_module:
+    state: present
+    name: proxy
+
+- name: HTTP Proxy module
+  apache2_module:
+    state: present
+    name: proxy_http
+
+- name: Apache macro module
+  apache2_module:
+    state: present
+    name: macro
+
+- name: Apache 2 ssl module
+  apache2_module:
+    state: present
+    name: ssl
+
+- name: Apache webroot
+  file:
+    path: '{{ www_base }}'
+    state: directory
+    owner: root
+    group: root
+
+- name: Apache www root
+  file:
+    path: '{{ www_root }}'
+    state: directory
+    owner: root
+    group: root
+
+- name: AFS content symlinks
+  file:
+    src: '{{ mirror_root }}/{{ item }}'
+    dest: '{{ www_root }}/{{ item }}'
+    state: link
+    owner: root
+    group: root
+  with_items:
+    - centos
+    - ceph-deb-hammer
+    - ceph-deb-jewel
+    - ceph-deb-luminous
+    - ceph-deb-mimic
+    - deb-docker
+    - debian
+    - debian-security
+    - debian-openstack
+    - epel
+    - fedora
+    - opensuse
+    - ubuntu-ports
+    - ubuntu-cloud-archive
+    - wheel
+    - yum-puppetlabs
+
+- name: Install robots.txt
+  copy:
+    src: robots.txt
+    dest: '{{ www_root }}'
+    owner: root
+    group: root
+    mode: 0444
+
+- name: Apache proxy cache
+  file:
+    path: /var/cache/apache2/proxy
+    owner: www-data
+    group: www-data
+    mode: 0755
+    state: directory
+
+- name: Set mirror servername and alias
+  set_fact:
+    apache_server_name: '{{ inventory_hostname }}'
+    # Strip the numeric host value from mirror01.region.provider.o.o
+    # for the serveralias
+    apache_server_alias: '{{ inventory_hostname | regex_replace("^mirror\d\d\.", "mirror.") }}'
+
+- name: Create mirror virtual host
+  template:
+    src: mirror.vhost.j2
+    dest: /etc/apache2/sites-available/mirror.conf
+
+- name: Make sure default site disabled
+  command: a2dissite 000-default.conf
+  args:
+    removes: /etc/apache2/sites-enabled/000-default.conf
+
+- name: Enable mirror virtual host
+  command: a2ensite mirror
+  args:
+    creates: /etc/apache2/sites-enabled/mirror.conf
+  notify:
+    - restart apache2
+
+- name: Debug config
+  slurp:
+    src: /etc/apache2/sites-available/mirror.conf
+  register: http_config
+- name: Show config
+  debug:
+    msg: '{{ http_config["content"] | b64decode }}'
+
+# Clean apache cache once an hour, keep size down to 70GiB.
+- name: Proxy cleanup cron job
+  cron:
+    name: Apache cache cleanup
+    state: present
+    job: /usr/bin/flock -n /var/run/htcacheclean.lock /usr/bin/htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null
+    minute: '0'
+    hour: '*'

playbooks/roles/mirror/templates/mirror.vhost.j2

@@ -0,0 +1,404 @@
+NameVirtualHost *:80
+NameVirtualHost *:443
+
+# Dedicated port for proxy caching, as not to affect afs mirrors.
+Listen 8080
+NameVirtualHost *:8080
+
+Listen 8081
+NameVirtualHost *:8081
+
+Listen 8082
+NameVirtualHost *:8082
+
+LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache
+
+<Macro BaseProxy $port>
+
+    DocumentRoot /var/www/mirror
+    <Directory /var/www/mirror>
+        Options Indexes FollowSymLinks MultiViews
+        AllowOverride None
+        Order allow,deny
+        allow from all
+        Satisfy any
+        <IfVersion >= 2.4>
+            Require all granted
+        </IfVersion>
+    </Directory>
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+    # Pip sets Cache-Control: max-age=0 on requests for pypi index pages.
+    # This means we don't use the cache for those requests. This setting
+    # should force the proxy to ignore cache-control on the request side
+    # but we should still cache things based on the cache-control responses
+    # from the backed servers.
+    CacheIgnoreCacheControl On
+
+    # Added Aug 2017 in an attempt to avoid occasional 502 errors (around
+    # 0.05% of requests) of the type:
+    #
+    #  End of file found: ... AH01102: error reading status line from remote server ...
+    #
+    # Per [1]:
+    #
+    #  This avoids the "proxy: error reading status line from remote
+    #  server" error message caused by the race condition that the backend
+    #  server closed the pooled connection after the connection check by the
+    #  proxy and before data sent by the proxy reached the backend.
+    #
+    # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
+    SetEnv proxy-initial-not-pooled 1
+
+    RewriteEngine On
+    # pypi
+    CacheEnable disk  "/pypi"
+    ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypi/" "https://pypi.org/
+
+    # files.pythonhosted.org
+    CacheEnable disk  "/pypifiles"
+    ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
+
+    # Rewrite the locations of the actual files
+    <Location /pypi>
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
+    </Location>
+
+    # Wheel URL's are:
+    # /wheel/{distro}-{distro-version}/a/a/a-etc.whl
+    # /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl
+    # /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl
+    RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*)
+    RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d
+    RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L]
+
+    # Special cases for openstack.nose_plugin & backports.*
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2
+
+    # Try again but replacing -'s with .'s
+    RewriteCond %{REQUEST_URI} ^/wheel/
+    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f
+    RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d
+    RewriteRule (.*)-(.*) $1.$2 [N]
+
+    ErrorLog /var/log/apache2/proxy_$port_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_$port_access.log combined-cache
+    ServerSignature Off
+
+</Macro>
+
+<VirtualHost *:80>
+    ServerName {{ apache_server_name }}
+    ServerAlias {{ apache_server_alias }}
+
+    Use BaseProxy 80
+</VirtualHost>
+
+<VirtualHost *:443>
+    ServerName {{ apache_server_name }}
+    ServerAlias {{ apache_server_alias }}
+
+    SSLCertificateFile      /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer
+    SSLCertificateKeyFile   /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key
+    SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer
+    SSLProtocol All -SSLv2 -SSLv3
+    # Note: this list should ensure ciphers that provide forward secrecy
+    SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+    SSLHonorCipherOrder on
+
+    Use BaseProxy 443
+</VirtualHost>
+
+<VirtualHost *:8080>
+    ServerName {{ apache_server_name }}:8080
+    ServerAlias {{ apache_server_alias }}:8080
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8080_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8080_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    CacheStoreExpired On
+
+    # Added Aug 2017 in an attempt to avoid occasional 502 errors (around
+    # 0.05% of requests) of the type:
+    #
+    #  End of file found: ... AH01102: error reading status line from remote server ...
+    #
+    # Per [1]:
+    #
+    #  This avoids the "proxy: error reading status line from remote
+    #  server" error message caused by the race condition that the backend
+    #  server closed the pooled connection after the connection check by the
+    #  proxy and before data sent by the proxy reached the backend.
+    #
+    # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html
+    SetEnv proxy-initial-not-pooled 1
+
+    # Per site caching reverse proxy rules
+    # Only cache specific backends, rely on afs cache otherwise.
+
+    # buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org)
+    CacheEnable disk  "/buildlogs.centos"
+    ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0
+    ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/"
+
+    # buildlogs.cdn.centos.org
+    CacheEnable disk  "/buildlogs.cdn.centos"
+    ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0
+    ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/"
+
+    # rdo
+    CacheEnable disk  "/rdo"
+    ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/"
+
+    # cbs.centos.org
+    CacheEnable disk  "/cbs.centos"
+    ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/"
+
+    # tarballs
+    CacheEnable disk  "/tarballs"
+    ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/"
+
+    # pypi
+    CacheEnable disk  "/pypi"
+    ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypi/" "https://pypi.org/
+
+    # files.pythonhosted.org
+    CacheEnable disk  "/pypifiles"
+    ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/"
+
+    # Rewrite the locations of the actual files
+    <Location /pypi>
+      SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE
+      Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni"
+    </Location>
+
+    # images.linuxcontainers.org
+    CacheEnable disk "/images.linuxcontainers"
+    ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/"
+
+    # registry.npmjs.org
+    CacheEnable disk  "/registry.npmjs"
+    ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/"
+
+    # api.rubygems.org
+    CacheEnable disk  "/api.rubygems"
+    ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/"
+
+    # rubygems.org
+    CacheEnable disk  "/rubygems"
+    ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/rubygems/" "https://rubygems.org/"
+
+    # opendaylight
+    CacheEnable disk "/opendaylight"
+    ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/"
+
+    # elastico
+    CacheEnable disk "/elastic"
+    ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/elastic/" "https://packages.elastic.co/"
+
+    # grafana
+    CacheEnable disk "/grafana"
+    ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/"
+
+    # OracleLinux
+    CacheEnable disk "/oraclelinux"
+    ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/"
+
+    # Percona
+    CacheEnable disk "/percona"
+    ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/percona/" "https://repo.percona.com/"
+
+    # MariaDB
+    CacheEnable disk "/MariaDB"
+    ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/"
+
+    # Docker
+    CacheEnable disk "/docker"
+    ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/docker/" "https://download.docker.com/linux/"
+
+    # Alpine
+    CacheEnable disk "/alpine"
+    ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/"
+
+    # LXC (copr)
+    CacheEnable disk "/copr-lxc2"
+    ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/"
+
+</VirtualHost>
+
+# Docker registry v1 proxy.
+<VirtualHost *:8081>
+    ServerName {{ apache_server_name }}:8081
+    ServerAlias {{ apache_server_alias }}:8081
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8081_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8081_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    # Ignore expire headers as the urls use sha256 hashes.
+    CacheIgnoreQueryString On
+    # NOTE(pabelanger): In the case of docker, if neither an expiry date nor
+    # last-modified date are provided default expire to 1 day. This is up from
+    # 1 hour.
+    CacheDefaultExpire 86400
+    CacheStoreExpired On
+
+    # registry-1.docker.io
+    CacheEnable disk  "/registry-1.docker"
+    ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/"
+
+    # dseasb33srnrn.cloudfront.net
+    CacheEnable disk "/cloudfront"
+    ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
+
+    # production.cloudflare.docker.com
+    CacheEnable disk "/cloudflare"
+    ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
+
+</VirtualHost>
+
+# Docker registry v2 proxy.
+<VirtualHost *:8082>
+    ServerName {{ apache_server_name }}:8082
+    ServerAlias {{ apache_server_alias }}:8082
+
+    # Disable directory listing by default.
+    <Directory />
+      Order Deny,Allow
+      Deny from all
+      Options None
+      AllowOverride None
+    </Directory>
+
+    ErrorLog /var/log/apache2/proxy_8082_error.log
+    LogLevel warn
+    CustomLog /var/log/apache2/proxy_8082_access.log combined-cache
+    ServerSignature Off
+
+    # Caching reverse proxy for things that don't make sense in AFS
+    #
+    # General cache rules
+    CacheRoot "/var/cache/apache2/proxy"
+    CacheDirLevels 5
+    CacheDirLength 2
+    # SSL support
+    SSLProxyEngine on
+    # Prevent thundering herds.
+    CacheLock on
+    CacheLockPath "/tmp/mod_cache-lock"
+    CacheLockMaxAge 5
+    # 5GiB
+    CacheMaxFileSize 5368709120
+    # Ignore expire headers as the urls use sha256 hashes.
+    CacheIgnoreQueryString On
+    # NOTE(pabelanger): In the case of docker, if neither an expiry date nor
+    # last-modified date are provided default expire to 1 day. This is up from
+    # 1 hour.
+    CacheDefaultExpire 86400
+    CacheStoreExpired On
+
+    # dseasb33srnrn.cloudfront.net
+    CacheEnable disk "/cloudfront"
+    ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/"
+
+    # production.cloudflare.docker.com
+    CacheEnable disk "/cloudflare"
+    ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/"
+
+    # NOTE(corvus): Ensure this stanza is last since it's the most
+    # greedy match.
+    CacheEnable disk  "/"
+    ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0
+    ProxyPassReverse "/" "https://registry-1.docker.io/"
+</VirtualHost>

playbooks/service-mirror.yaml

@@ -0,0 +1,11 @@
+- hosts: "mirror_opendev:!disabled"
+  name: "Configure per region opendev mirrors"
+  roles:
+    - role: kerberos-client
+      kerberos_realm: 'OPENSTACK.ORG'
+      kerberos_admin_server: 'kdc.openstack.org'
+      kerberos_kdcs:
+        - kdc03.openstack.org
+        - kdc04.openstack.org
+    - role: openafs-client
+    - role: mirror

playbooks/zuul/run-base.yaml

@@ -81,6 +81,7 @@
         - host_vars/bridge.openstack.org.yaml
         - host_vars/letsencrypt01.opendev.org.yaml
         - host_vars/letsencrypt02.opendev.org.yaml
+        - host_vars/mirror01.region.provider.opendev.org.yaml
     - name: Display group membership
       command: ansible localhost -m debug -a 'var=groups'
     - name: Run base.yaml

playbooks/zuul/templates/gate-groups.yaml.j2

@@ -8,3 +8,4 @@ groups:
   letsencrypt:
     - letsencrypt01.opendev.org
     - letsencrypt02.opendev.org
+    - mirror01.region.provider.opendev.org

playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2

@@ -0,0 +1,4 @@
+letsencrypt_certs:
+  mirror01-region-provider-opendev-org-main:
+    - mirror01.region.provider.opendev.org
+    - mirror.region.provider.opendev.org

run_all.sh

@@ -100,6 +100,10 @@ start_timer
 timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml
 send_timer nodepool
 
+start_timer
+timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml
+send_timer nodepool
+
 start_timer
 timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml
 send_timer registry

testinfra/test_mirror.py

@@ -0,0 +1,32 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['mirror01.region.provider.opendev.org']
+
+
+def test_apache(host):
+    apache = host.service('apache2')
+    assert apache.is_running
+
+def test_mirror_indexes(host):
+    cmd = host.run("wget --no-check-certificate -qO- https://localhost/")
+    assert '<a href="debian/">' in cmd.stdout
+
+    cmd = host.run("wget -qO- http://localhost/")
+    assert '<a href="debian/">' in cmd.stdout
+
+# NOTE(ianw): further testing idea for anyone interested; get the
+# actual IP address of the mirror node and connect via that, and then
+# also poke at the other proxy ports