From 670107045af6ae6ed6c9c81ba0ea648d26270cba Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 10 May 2019 17:17:49 +1000 Subject: [PATCH] Create opendev mirrors This impelements mirrors to live in the opendev.org namespace. The implementation is Ansible native for deployment on a Bionic node. The hostname prefix remains the same (mirrorXX.region.provider.) but the groups.yaml splits the opendev.org mirrors into a separate group. The matches in the puppet group are also updated so to not run puppet on the hosts. The kerberos and openafs client parts do not need any updating and works on the Bionic host. The hosts are setup to provision certificates for themselves from letsencrypt. Note we've added a new handler for mirror nodes to use that restarts apache on certificate issue/renewal. The new "mirror" role is a port of the existing puppet mirror.pp. It installs apache, sets up some modules, makes some symlinks, sets up a cleanup cron job and installs the apache vhost configuration. The vhost configuration is also ported from the extant puppet. It is simplified somewhat; but the biggest change is that we have extracted the main port 80 configuration into a macro which is applied to both port 80 and 443; i.e. the host will have SSL support. The other ports are left alone for now, but can be updated in due course. Thus we should be able to CNAME the existing mirrors to new nodes, and any existing http access can continue. We can update our mirror setup scripts to point to https resources as appropriate. Change-Id: Iec576d631dd5b02f6b9fb445ee600be060f9cf1e --- .zuul.yaml | 30 ++ inventory/groups.yaml | 9 +- playbooks/group_vars/mirror_opendev.yaml | 6 + .../handlers/main.yaml | 3 + .../handlers/restart_apache.yaml | 8 + playbooks/roles/mirror/README.rst | 6 + playbooks/roles/mirror/defaults/main.yaml | 3 + playbooks/roles/mirror/files/robots.txt | 2 + playbooks/roles/mirror/handlers/main.yaml | 4 + playbooks/roles/mirror/tasks/main.yaml | 151 +++++++ .../roles/mirror/templates/mirror.vhost.j2 | 404 ++++++++++++++++++ playbooks/service-mirror.yaml | 11 + playbooks/zuul/run-base.yaml | 1 + playbooks/zuul/templates/gate-groups.yaml.j2 | 1 + ...rror01.region.provider.opendev.org.yaml.j2 | 4 + run_all.sh | 4 + testinfra/test_mirror.py | 32 ++ 17 files changed, 676 insertions(+), 3 deletions(-) create mode 100644 playbooks/group_vars/mirror_opendev.yaml create mode 100644 playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml create mode 100644 playbooks/roles/mirror/README.rst create mode 100644 playbooks/roles/mirror/defaults/main.yaml create mode 100644 playbooks/roles/mirror/files/robots.txt create mode 100644 playbooks/roles/mirror/handlers/main.yaml create mode 100644 playbooks/roles/mirror/tasks/main.yaml create mode 100644 playbooks/roles/mirror/templates/mirror.vhost.j2 create mode 100644 playbooks/service-mirror.yaml create mode 100644 playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2 create mode 100644 testinfra/test_mirror.py diff --git a/.zuul.yaml b/.zuul.yaml index dcb7de8a1d..921a89857a 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -481,6 +481,35 @@ - testinfra/test_adns.py - testinfra/test_ns.py +- job: + name: system-config-run-mirror + parent: system-config-run + description: | + Run the playbook for a mirror node + nodeset: + nodes: + - name: bridge.openstack.org + label: ubuntu-bionic + - name: mirror01.region.provider.opendev.org + label: ubuntu-bionic + vars: + run_playbooks: + - playbooks/service-letsencrypt.yaml + - playbooks/service-mirror.yaml + files: + - .zuul.yaml + - roles/ + - playbooks/roles/mirror/ + - playbooks/roles/letsencrypt.* + - playbooks/service-letsencrypt.yaml + - playbooks/service-mirror.yaml + - testinfra/test_mirror.py + + host-vars: + mirror.region.provider.opendev.org: + host_copy_output: + '/var/log/apache2/': logs + - job: name: system-config-run-docker-registry parent: system-config-run @@ -615,6 +644,7 @@ - system-config-run-dns - system-config-run-eavesdrop - system-config-run-nodepool + - system-config-run-mirror - system-config-run-docker-registry - system-config-run-gitea: dependencies: diff --git a/inventory/groups.yaml b/inventory/groups.yaml index 3944e5f4ac..fa18f4c3bb 100644 --- a/inventory/groups.yaml +++ b/inventory/groups.yaml @@ -57,6 +57,7 @@ groups: - opendev-k8s*.opendev.org letsencrypt: - graphite01.opendev.org + - mirror[0-9]*.opendev.org logstash: - logstash[0-9]*.open*.org logstash-worker: @@ -65,7 +66,9 @@ groups: - lists*.katacontainers.io - lists*.open*.org mirror: - - mirror[0-9]*.open*.org + - mirror[0-9]*.openstack.org + mirror_opendev: + - mirror[0-9]*.opendev.org nodepool: - nb[0-9]*.open*.org - nl[0-9]*.open*.org @@ -110,7 +113,7 @@ groups: - logstash-worker[0-9]*.open*.org - logstash[0-9]*.open*.org - mirror-update[0-9]*.open*.org - - mirror[0-9]*.open*.org + - mirror[0-9]*.openstack.org - nb[0-9]*.open*.org - nl[0-9]*.open*.org - openstackid-dev*.openstack.org @@ -161,7 +164,7 @@ groups: - logstash-worker[0-9]*.open*.org - logstash[0-9]*.open*.org - mirror-update[0-9]*.open*.org - - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.open.*\.org + - ^mirror[0-9].*\..*\.(?!linaro|linaro-london|arm64ci).*\.openstack\.org - ^nb(?!03)[0-9]*\.open.*\.org - nl[0-9]*.open*.org - openstackid[0-9]*.openstack.org diff --git a/playbooks/group_vars/mirror_opendev.yaml b/playbooks/group_vars/mirror_opendev.yaml new file mode 100644 index 0000000000..3f4cb5ba92 --- /dev/null +++ b/playbooks/group_vars/mirror_opendev.yaml @@ -0,0 +1,6 @@ +iptables_extra_public_tcp_ports: + - 80 + - 443 + - 8080 + - 8081 + - 8082 diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml index 24fb8a4eab..86c0760a3a 100644 --- a/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml +++ b/playbooks/roles/letsencrypt-create-certs/handlers/main.yaml @@ -30,3 +30,6 @@ import_tasks: touch_file.yaml vars: touch_file: '/tmp/letsencrypt02-main-service.stamp' + +- name: letsencrypt updated mirror01-region-provider-opendev-org-main + import_tasks: restart_apache.yaml diff --git a/playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml b/playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml new file mode 100644 index 0000000000..6b643ad236 --- /dev/null +++ b/playbooks/roles/letsencrypt-create-certs/handlers/restart_apache.yaml @@ -0,0 +1,8 @@ +- name: Populate service facts + service_facts: + +- name: Restart apache + service: + name: apache2 + state: restarted + when: "'apache2' in ansible_facts.services" \ No newline at end of file diff --git a/playbooks/roles/mirror/README.rst b/playbooks/roles/mirror/README.rst new file mode 100644 index 0000000000..412267c32f --- /dev/null +++ b/playbooks/roles/mirror/README.rst @@ -0,0 +1,6 @@ +Configure an opendev mirror + +This role installs and configures a mirror node + +**Role Variables** + diff --git a/playbooks/roles/mirror/defaults/main.yaml b/playbooks/roles/mirror/defaults/main.yaml new file mode 100644 index 0000000000..8268337479 --- /dev/null +++ b/playbooks/roles/mirror/defaults/main.yaml @@ -0,0 +1,3 @@ +mirror_root: '/afs/openstack.org/mirror' +www_base: '/var/www' +www_root: '{{ www_base }}/mirror' \ No newline at end of file diff --git a/playbooks/roles/mirror/files/robots.txt b/playbooks/roles/mirror/files/robots.txt new file mode 100644 index 0000000000..1f53798bb4 --- /dev/null +++ b/playbooks/roles/mirror/files/robots.txt @@ -0,0 +1,2 @@ +User-agent: * +Disallow: / diff --git a/playbooks/roles/mirror/handlers/main.yaml b/playbooks/roles/mirror/handlers/main.yaml new file mode 100644 index 0000000000..6edd3d3105 --- /dev/null +++ b/playbooks/roles/mirror/handlers/main.yaml @@ -0,0 +1,4 @@ +- name: restart apache2 + service: + name: apache2 + state: restarted \ No newline at end of file diff --git a/playbooks/roles/mirror/tasks/main.yaml b/playbooks/roles/mirror/tasks/main.yaml new file mode 100644 index 0000000000..ef167f323d --- /dev/null +++ b/playbooks/roles/mirror/tasks/main.yaml @@ -0,0 +1,151 @@ +- name: Check AFS mounted + stat: + path: "/afs/openstack.org/mirror" + register: afs_mirror +- name: Sanity check AFS + assert: + that: + - afs_mirror.stat.exists + +- name: Install apache2 + apt: + name: + - apache2 + - apache2-utils + state: present + +- name: Rewrite module + apache2_module: + state: present + name: rewrite + +- name: Substitute module + apache2_module: + state: present + name: substitute + +- name: Cache module + apache2_module: + state: present + name: cache + +- name: Cache disk module + apache2_module: + state: present + name: cache_disk + +- name: Proxy module + apache2_module: + state: present + name: proxy + +- name: HTTP Proxy module + apache2_module: + state: present + name: proxy_http + +- name: Apache macro module + apache2_module: + state: present + name: macro + +- name: Apache 2 ssl module + apache2_module: + state: present + name: ssl + +- name: Apache webroot + file: + path: '{{ www_base }}' + state: directory + owner: root + group: root + +- name: Apache www root + file: + path: '{{ www_root }}' + state: directory + owner: root + group: root + +- name: AFS content symlinks + file: + src: '{{ mirror_root }}/{{ item }}' + dest: '{{ www_root }}/{{ item }}' + state: link + owner: root + group: root + with_items: + - centos + - ceph-deb-hammer + - ceph-deb-jewel + - ceph-deb-luminous + - ceph-deb-mimic + - deb-docker + - debian + - debian-security + - debian-openstack + - epel + - fedora + - opensuse + - ubuntu-ports + - ubuntu-cloud-archive + - wheel + - yum-puppetlabs + +- name: Install robots.txt + copy: + src: robots.txt + dest: '{{ www_root }}' + owner: root + group: root + mode: 0444 + +- name: Apache proxy cache + file: + path: /var/cache/apache2/proxy + owner: www-data + group: www-data + mode: 0755 + state: directory + +- name: Set mirror servername and alias + set_fact: + apache_server_name: '{{ inventory_hostname }}' + # Strip the numeric host value from mirror01.region.provider.o.o + # for the serveralias + apache_server_alias: '{{ inventory_hostname | regex_replace("^mirror\d\d\.", "mirror.") }}' + +- name: Create mirror virtual host + template: + src: mirror.vhost.j2 + dest: /etc/apache2/sites-available/mirror.conf + +- name: Make sure default site disabled + command: a2dissite 000-default.conf + args: + removes: /etc/apache2/sites-enabled/000-default.conf + +- name: Enable mirror virtual host + command: a2ensite mirror + args: + creates: /etc/apache2/sites-enabled/mirror.conf + notify: + - restart apache2 + +- name: Debug config + slurp: + src: /etc/apache2/sites-available/mirror.conf + register: http_config +- name: Show config + debug: + msg: '{{ http_config["content"] | b64decode }}' + +# Clean apache cache once an hour, keep size down to 70GiB. +- name: Proxy cleanup cron job + cron: + name: Apache cache cleanup + state: present + job: /usr/bin/flock -n /var/run/htcacheclean.lock /usr/bin/htcacheclean -n -p /var/cache/apache2/proxy -t -l 70200M > /dev/null + minute: '0' + hour: '*' diff --git a/playbooks/roles/mirror/templates/mirror.vhost.j2 b/playbooks/roles/mirror/templates/mirror.vhost.j2 new file mode 100644 index 0000000000..68ce814309 --- /dev/null +++ b/playbooks/roles/mirror/templates/mirror.vhost.j2 @@ -0,0 +1,404 @@ +NameVirtualHost *:80 +NameVirtualHost *:443 + +# Dedicated port for proxy caching, as not to affect afs mirrors. +Listen 8080 +NameVirtualHost *:8080 + +Listen 8081 +NameVirtualHost *:8081 + +Listen 8082 +NameVirtualHost *:8082 + +LogFormat "%h %l %u %t \"%r\" %>s %b %{cache-status}e \"%{Referer}i\" \"%{User-agent}i\"" combined-cache + + + + DocumentRoot /var/www/mirror + + Options Indexes FollowSymLinks MultiViews + AllowOverride None + Order allow,deny + allow from all + Satisfy any + = 2.4> + Require all granted + + + + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + CacheStoreExpired On + # Pip sets Cache-Control: max-age=0 on requests for pypi index pages. + # This means we don't use the cache for those requests. This setting + # should force the proxy to ignore cache-control on the request side + # but we should still cache things based on the cache-control responses + # from the backed servers. + CacheIgnoreCacheControl On + + # Added Aug 2017 in an attempt to avoid occasional 502 errors (around + # 0.05% of requests) of the type: + # + # End of file found: ... AH01102: error reading status line from remote server ... + # + # Per [1]: + # + # This avoids the "proxy: error reading status line from remote + # server" error message caused by the race condition that the backend + # server closed the pooled connection after the connection check by the + # proxy and before data sent by the proxy reached the backend. + # + # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html + SetEnv proxy-initial-not-pooled 1 + + RewriteEngine On + # pypi + CacheEnable disk "/pypi" + ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypi/" "https://pypi.org/ + + # files.pythonhosted.org + CacheEnable disk "/pypifiles" + ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/" + + # Rewrite the locations of the actual files + + SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE + Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni" + + + # Wheel URL's are: + # /wheel/{distro}-{distro-version}/a/a/a-etc.whl + # /wheel/{distro}-{distro-version}/a/abcd/abcd-etc.whl + # /wheel/{distro}-{distro-version}/a/abcde/abcde-etc.whl + RewriteCond %{REQUEST_URI} ^/wheel/([^/]+)/([^/])([^/]*) + RewriteCond %{DOCUMENT_ROOT}/wheel/$1/$2/$2$3 -d + RewriteRule ^/wheel/([^/]+)/([^/])([^/]*)(/.*)?$ /wheel/$1/$2/$2$3$4 [L] + + # Special cases for openstack.nose_plugin & backports.* + RewriteCond %{REQUEST_URI} ^/wheel/ + RewriteRule ^(.*)/openstack-nose-plugin(.*)$ $1/openstack.nose_plugin$2 + RewriteCond %{REQUEST_URI} ^/wheel/ + RewriteRule ^(.*)/backports-(.*)$ $1/backports.$2 + + # Try again but replacing -'s with .'s + RewriteCond %{REQUEST_URI} ^/wheel/ + RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-f + RewriteCond %{DOCUMENT_ROOT}/%{REQUEST_URI} !-d + RewriteRule (.*)-(.*) $1.$2 [N] + + ErrorLog /var/log/apache2/proxy_$port_error.log + LogLevel warn + CustomLog /var/log/apache2/proxy_$port_access.log combined-cache + ServerSignature Off + + + + + ServerName {{ apache_server_name }} + ServerAlias {{ apache_server_alias }} + + Use BaseProxy 80 + + + + ServerName {{ apache_server_name }} + ServerAlias {{ apache_server_alias }} + + SSLCertificateFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.cer + SSLCertificateKeyFile /etc/letsencrypt-certs/{{ apache_server_name }}/{{ apache_server_name }}.key + SSLCertificateChainFile /etc/letsencrypt-certs/{{ apache_server_name }}/ca.cer + SSLProtocol All -SSLv2 -SSLv3 + # Note: this list should ensure ciphers that provide forward secrecy + SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP + SSLHonorCipherOrder on + + Use BaseProxy 443 + + + + ServerName {{ apache_server_name }}:8080 + ServerAlias {{ apache_server_alias }}:8080 + + # Disable directory listing by default. + + Order Deny,Allow + Deny from all + Options None + AllowOverride None + + + ErrorLog /var/log/apache2/proxy_8080_error.log + LogLevel warn + CustomLog /var/log/apache2/proxy_8080_access.log combined-cache + ServerSignature Off + + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + CacheStoreExpired On + + # Added Aug 2017 in an attempt to avoid occasional 502 errors (around + # 0.05% of requests) of the type: + # + # End of file found: ... AH01102: error reading status line from remote server ... + # + # Per [1]: + # + # This avoids the "proxy: error reading status line from remote + # server" error message caused by the race condition that the backend + # server closed the pooled connection after the connection check by the + # proxy and before data sent by the proxy reached the backend. + # + # [1] https://httpd.apache.org/docs/2.4/mod/mod_proxy_http.html + SetEnv proxy-initial-not-pooled 1 + + # Per site caching reverse proxy rules + # Only cache specific backends, rely on afs cache otherwise. + + # buildlogs.centos.org (302 redirects to buildlogs.cdn.centos.org) + CacheEnable disk "/buildlogs.centos" + ProxyPass "/buildlogs.centos/" "https://buildlogs.centos.org/" ttl=120 disablereuse=On retry=0 + ProxyPassReverse "/buildlogs.centos/" "https://buildlogs.centos.org/" + + # buildlogs.cdn.centos.org + CacheEnable disk "/buildlogs.cdn.centos" + ProxyPass "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" ttl=120 disablereuse=On retry=0 + ProxyPassReverse "/buildlogs.cdn.centos/" "https://buildlogs.cdn.centos.org/" + + # rdo + CacheEnable disk "/rdo" + ProxyPass "/rdo/" "https://trunk.rdoproject.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/rdo/" "https://trunk.rdoproject.org/" + + # cbs.centos.org + CacheEnable disk "/cbs.centos" + ProxyPass "/cbs.centos/" "https://cbs.centos.org/repos/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/cbs.centos/" "https://cbs.centos.org/repos/" + + # tarballs + CacheEnable disk "/tarballs" + ProxyPass "/tarballs/" "https://tarballs.openstack.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/tarballs/" "https://tarballs.openstack.org/" + + # pypi + CacheEnable disk "/pypi" + ProxyPass "/pypi/" "https://pypi.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypi/" "https://pypi.org/ + + # files.pythonhosted.org + CacheEnable disk "/pypifiles" + ProxyPass "/pypifiles/" "https://files.pythonhosted.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/pypifiles/" "https://files.pythonhosted.org/" + + # Rewrite the locations of the actual files + + SetOutputFilter INFLATE;SUBSTITUTE;DEFLATE + Substitute "s|https://files.pythonhosted.org/|/pypifiles/|ni" + + + # images.linuxcontainers.org + CacheEnable disk "/images.linuxcontainers" + ProxyPass "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/images.linuxcontainers/" "http://us.images.linuxcontainers.org/" + + # registry.npmjs.org + CacheEnable disk "/registry.npmjs" + ProxyPass "/registry.npmjs/" "https://registry.npmjs.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/registry.npmjs/" "https://registry.npmjs.org/" + + # api.rubygems.org + CacheEnable disk "/api.rubygems" + ProxyPass "/api.rubygems/" "https://api.rubygems.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/api.rubygems/" "https://api.rubygems.org/" + + # rubygems.org + CacheEnable disk "/rubygems" + ProxyPass "/rubygems/" "https://rubygems.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/rubygems/" "https://rubygems.org/" + + # opendaylight + CacheEnable disk "/opendaylight" + ProxyPass "/opendaylight/" "https://nexus.opendaylight.org/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/opendaylight/" "https://nexus.opendaylight.org/" + + # elastico + CacheEnable disk "/elastic" + ProxyPass "/elastic/" "https://packages.elastic.co/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/elastic/" "https://packages.elastic.co/" + + # grafana + CacheEnable disk "/grafana" + ProxyPass "/grafana" "https://packagecloud.io/grafana/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/grafana/" "https://packagecloud.io/grafana/" + + # OracleLinux + CacheEnable disk "/oraclelinux" + ProxyPass "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/oraclelinux/" "http://yum.oracle.com/repo/OracleLinux/" + + # Percona + CacheEnable disk "/percona" + ProxyPass "/percona/" "https://repo.percona.com/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/percona/" "https://repo.percona.com/" + + # MariaDB + CacheEnable disk "/MariaDB" + ProxyPass "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/MariaDB/" "https://downloads.mariadb.com/MariaDB/" + + # Docker + CacheEnable disk "/docker" + ProxyPass "/docker/" "https://download.docker.com/linux/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/docker/" "https://download.docker.com/linux/" + + # Alpine + CacheEnable disk "/alpine" + ProxyPass "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/alpine/" "http://dl-cdn.alpinelinux.org/alpine/" + + # LXC (copr) + CacheEnable disk "/copr-lxc2" + ProxyPass "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/copr-lxc2/" "https://copr-be.cloud.fedoraproject.org/results/thm/lxc2.0/" + + + +# Docker registry v1 proxy. + + ServerName {{ apache_server_name }}:8081 + ServerAlias {{ apache_server_alias }}:8081 + + # Disable directory listing by default. + + Order Deny,Allow + Deny from all + Options None + AllowOverride None + + + ErrorLog /var/log/apache2/proxy_8081_error.log + LogLevel warn + CustomLog /var/log/apache2/proxy_8081_access.log combined-cache + ServerSignature Off + + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + # Ignore expire headers as the urls use sha256 hashes. + CacheIgnoreQueryString On + # NOTE(pabelanger): In the case of docker, if neither an expiry date nor + # last-modified date are provided default expire to 1 day. This is up from + # 1 hour. + CacheDefaultExpire 86400 + CacheStoreExpired On + + # registry-1.docker.io + CacheEnable disk "/registry-1.docker" + ProxyPass "/registry-1.docker/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/registry-1.docker/" "https://registry-1.docker.io/" + + # dseasb33srnrn.cloudfront.net + CacheEnable disk "/cloudfront" + ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" + + # production.cloudflare.docker.com + CacheEnable disk "/cloudflare" + ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" + + + +# Docker registry v2 proxy. + + ServerName {{ apache_server_name }}:8082 + ServerAlias {{ apache_server_alias }}:8082 + + # Disable directory listing by default. + + Order Deny,Allow + Deny from all + Options None + AllowOverride None + + + ErrorLog /var/log/apache2/proxy_8082_error.log + LogLevel warn + CustomLog /var/log/apache2/proxy_8082_access.log combined-cache + ServerSignature Off + + # Caching reverse proxy for things that don't make sense in AFS + # + # General cache rules + CacheRoot "/var/cache/apache2/proxy" + CacheDirLevels 5 + CacheDirLength 2 + # SSL support + SSLProxyEngine on + # Prevent thundering herds. + CacheLock on + CacheLockPath "/tmp/mod_cache-lock" + CacheLockMaxAge 5 + # 5GiB + CacheMaxFileSize 5368709120 + # Ignore expire headers as the urls use sha256 hashes. + CacheIgnoreQueryString On + # NOTE(pabelanger): In the case of docker, if neither an expiry date nor + # last-modified date are provided default expire to 1 day. This is up from + # 1 hour. + CacheDefaultExpire 86400 + CacheStoreExpired On + + # dseasb33srnrn.cloudfront.net + CacheEnable disk "/cloudfront" + ProxyPass "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/cloudfront/" "https://dseasb33srnrn.cloudfront.net/" + + # production.cloudflare.docker.com + CacheEnable disk "/cloudflare" + ProxyPass "/cloudflare/" "https://production.cloudflare.docker.com/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/cloudflare/" "https://production.cloudflare.docker.com/" + + # NOTE(corvus): Ensure this stanza is last since it's the most + # greedy match. + CacheEnable disk "/" + ProxyPass "/" "https://registry-1.docker.io/" ttl=120 keepalive=On retry=0 + ProxyPassReverse "/" "https://registry-1.docker.io/" + diff --git a/playbooks/service-mirror.yaml b/playbooks/service-mirror.yaml new file mode 100644 index 0000000000..ba6f0c59d6 --- /dev/null +++ b/playbooks/service-mirror.yaml @@ -0,0 +1,11 @@ +- hosts: "mirror_opendev:!disabled" + name: "Configure per region opendev mirrors" + roles: + - role: kerberos-client + kerberos_realm: 'OPENSTACK.ORG' + kerberos_admin_server: 'kdc.openstack.org' + kerberos_kdcs: + - kdc03.openstack.org + - kdc04.openstack.org + - role: openafs-client + - role: mirror \ No newline at end of file diff --git a/playbooks/zuul/run-base.yaml b/playbooks/zuul/run-base.yaml index 649dcb1e2f..5786816f70 100644 --- a/playbooks/zuul/run-base.yaml +++ b/playbooks/zuul/run-base.yaml @@ -81,6 +81,7 @@ - host_vars/bridge.openstack.org.yaml - host_vars/letsencrypt01.opendev.org.yaml - host_vars/letsencrypt02.opendev.org.yaml + - host_vars/mirror01.region.provider.opendev.org.yaml - name: Display group membership command: ansible localhost -m debug -a 'var=groups' - name: Run base.yaml diff --git a/playbooks/zuul/templates/gate-groups.yaml.j2 b/playbooks/zuul/templates/gate-groups.yaml.j2 index 206b4e6585..fbecbbc3e8 100644 --- a/playbooks/zuul/templates/gate-groups.yaml.j2 +++ b/playbooks/zuul/templates/gate-groups.yaml.j2 @@ -8,3 +8,4 @@ groups: letsencrypt: - letsencrypt01.opendev.org - letsencrypt02.opendev.org + - mirror01.region.provider.opendev.org diff --git a/playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2 b/playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2 new file mode 100644 index 0000000000..083952b24c --- /dev/null +++ b/playbooks/zuul/templates/host_vars/mirror01.region.provider.opendev.org.yaml.j2 @@ -0,0 +1,4 @@ +letsencrypt_certs: + mirror01-region-provider-opendev-org-main: + - mirror01.region.provider.opendev.org + - mirror.region.provider.opendev.org diff --git a/run_all.sh b/run_all.sh index 8647625b01..33f7927661 100755 --- a/run_all.sh +++ b/run_all.sh @@ -100,6 +100,10 @@ start_timer timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-nodepool.yaml send_timer nodepool +start_timer +timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-mirror.yaml +send_timer nodepool + start_timer timeout -k 2m 30m ansible-playbook -f 50 ${ANSIBLE_PLAYBOOKS}/service-registry.yaml send_timer registry diff --git a/testinfra/test_mirror.py b/testinfra/test_mirror.py new file mode 100644 index 0000000000..404733f1fa --- /dev/null +++ b/testinfra/test_mirror.py @@ -0,0 +1,32 @@ +# Copyright 2019 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +testinfra_hosts = ['mirror01.region.provider.opendev.org'] + + +def test_apache(host): + apache = host.service('apache2') + assert apache.is_running + +def test_mirror_indexes(host): + cmd = host.run("wget --no-check-certificate -qO- https://localhost/") + assert '' in cmd.stdout + + cmd = host.run("wget -qO- http://localhost/") + assert '' in cmd.stdout + +# NOTE(ianw): further testing idea for anyone interested; get the +# actual IP address of the mirror node and connect via that, and then +# also poke at the other proxy ports