Add roles for a basic static server

Basic implementation of the opendev static server, described in

 https://docs.opendev.org/opendev/infra-specs/latest/specs/retire-static.html

Change-Id: Ie1b92f06b71aa6069fe831b26ba1cc272ce4562c
Story: #2006598
Task:  #37757

.zuul.yaml

@@ -1069,6 +1069,35 @@
       - playbooks/roles/gerrit/
       - testinfra/test_gerrit.py
 
+- job:
+    name: system-config-run-static
+    parent: system-config-run
+    description: |
+      Run the playbook for a static node
+    nodeset:
+      nodes:
+        - name: bridge.openstack.org
+          label: ubuntu-bionic
+        - name: static01.opendev.org
+          label: ubuntu-bionic
+    vars:
+      run_playbooks:
+        - playbooks/service-letsencrypt.yaml
+        - playbooks/service-static.yaml
+    files:
+      - playbooks/bridge.yaml
+      - playbooks/roles/static/
+      - playbooks/roles/letsencrypt.*
+      - playbooks/service-letsencrypt.yaml
+      - playbooks/service-static.yaml
+      - testinfra/test_static.py
+    host-vars:
+      static01.opendev.org:
+        host_copy_output:
+          '/var/log/acme.sh/': logs
+          '/etc/apache2/': logs
+          '/var/log/apache2/': logs
+
 - job:
     name: infra-prod-playbook
     description: |
@@ -1119,6 +1148,7 @@
         - system-config-run-nodepool
         - system-config-run-mirror-x86
         - system-config-run-mirror-update
+        - system-config-run-static
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:
@@ -1188,6 +1218,7 @@
         - system-config-run-nodepool
         - system-config-run-mirror-x86
         - system-config-run-mirror-update
+        - system-config-run-static
         - system-config-run-docker-registry
         - system-config-run-gitea:
             dependencies:

inventory/groups.yaml

@@ -10,6 +10,7 @@ groups:
     - ze[0-9]*.open*.org
     - afsdb*.open*.org
     - afs[0-9]*.open*.org
+    - static[0-9]*.opendev.org
   afsadmin: mirror-update[0-9]*.openstack.org
   afsdb: afsdb[0-9]*.open*.org
   ask: ask*.open*.org
@@ -69,6 +70,7 @@ groups:
     - mirror[0-9]*.opendev.org
     - files[0-9]*.open*.org
     - static.openstack.org
+    - static[0-9]*.opendev.org
     - gitea[0-9]*.opendev.org
     - zuul[0-9]*.open*.org
   logstash:
@@ -137,7 +139,7 @@ groups:
     - planet[0-9]*.open*.org
     - refstack*.open*.org
     - review[0-9]*.open*.org
-    - static*.open*.org
+    - static*.openstack.org
     - status*.open*.org
     - storyboard-dev[0-9]*.opendev.org
     - storyboard[0-9]*.opendev.org
@@ -185,7 +187,7 @@ groups:
     - planet[0-9]*.open*.org
     - refstack*.open*.org
     - review[0-9]*.open*.org
-    - static*.open*.org
+    - static*.openstack.org
     - status*.open*.org
     - storyboard[0-9]*.opendev.org
     - storyboard-dev[0-9]*.opendev.org
@@ -208,7 +210,11 @@ groups:
   review:
     - review[0-9]*.open*.org
   static:
-    - static*.open*.org
+    - static*.openstack.org
+  # NOTE(ianw): 2019-12 : rename below when static.openstack.org is
+  # gone
+  static_opendev:
+    - static[0-9]*.opendev.org
   status:
     - status*.open*.org
   storyboard:

playbooks/host_vars/static01.opendev.org.yaml

@@ -0,0 +1,7 @@
+ansible_python_interpreter: python3
+letsencrypt_certs:
+  static01-governance-openstack-org:
+    - governance.openstack.org
+  static01-security-openstack-org:
+    - security.openstack.org
+

playbooks/roles/letsencrypt-create-certs/handlers/main.yaml

@@ -34,6 +34,13 @@
 - name: letsencrypt updated insecure-ci-registry01-main
   include_tasks: roles/letsencrypt-create-certs/handlers/restart_zuul_registry.yaml
 
+# Static
+- name: letsencrypt updated static01-governance-openstack-org
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
+- name: letsencrypt updated static01-security-openstack-org
+  include_tasks: roles/letsencrypt-create-certs/handlers/restart_apache.yaml
+
 # Mirrors
 
 - name: letsencrypt updated mirror01-dfw-rax-main

playbooks/roles/static/README.rst

@@ -0,0 +1,6 @@
+Configure an static webserver
+
+This role installs and configures a static webserver to serve content
+published in AFS
+
+**Role Variables**

playbooks/roles/static/files/50-governance.openstack.org.conf

@@ -0,0 +1,95 @@
+Define AFS_ROOT /afs/openstack.org/project/governance.openstack.org
+
+<VirtualHost *:80>
+  ServerName governance.openstack.org
+  RewriteEngine On
+  RewriteRule ^/(.*) https://governance.openstack.org/$1 [last,redirect=permanent]
+  LogLevel warn
+  ErrorLog /var/log/apache2/governance.openstack.org_error.log
+  CustomLog /var/log/apache2/governance.openstack.org_access.log combined
+  ServerSignature Off
+</VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost *:443>
+
+  ServerName governance.openstack.org
+
+  DocumentRoot ${AFS_ROOT}
+
+  SSLCertificateFile      /etc/letsencrypt-certs/governance.openstack.org/governance.openstack.org.cer
+  SSLCertificateKeyFile   /etc/letsencrypt-certs/governance.openstack.org/governance.openstack.org.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/governance.openstack.org/ca.cer
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  # Alias other folders
+  Alias "/election/" "${AFS_ROOT}/election/"
+  Alias "/sigs/" "${AFS_ROOT}/sigs/"
+  Alias "/tc/" "${AFS_ROOT}/tc/"
+  Alias "/uc/" "${AFS_ROOT}/uc/"
+  # keep last
+  Alias "/" "${AFS_ROOT}/governance/"
+
+  # Set up redirects
+  Redirect "/badges/" "/tc/badges/"
+  Redirect "/goals/" "/tc/goals/"
+  Redirect "/reference/" "/tc/reference/"
+  Redirect "/resolutions/" "/tc/resolutions/"
+
+  <Directory ${AFS_ROOT}/election>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  <Directory ${AFS_ROOT}/governance>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  <Directory ${AFS_ROOT}/sigs>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  <Directory ${AFS_ROOT}/tc>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  <Directory ${AFS_ROOT}/uc>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  <Directory ${AFS_ROOT}/badges>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverride None
+    Satisfy Any
+    Require all granted
+    <IfModule mod_headers.c>
+        Header set Cache-Control "private, no-cache, no-store, proxy-revalidate, no-transform"
+        Header set Pragma "no-cache"
+    </IfModule>
+    ErrorDocument 404 /badges/project-unofficial.svg
+  </Directory>
+
+  LogLevel warn
+  ErrorLog /var/log/apache2/governance.openstack.org_error.log
+  CustomLog /var/log/apache2/governance.openstack.org_access.log combined
+  ServerSignature Off
+
+</VirtualHost>
+</IfModule>

playbooks/roles/static/files/50-security.openstack.org.conf

@@ -0,0 +1,41 @@
+Define AFS_ROOT /afs/openstack.org/project/security.openstack.org
+
+<VirtualHost *:80>
+  ServerName security.openstack.org
+  RewriteEngine On
+  RewriteRule ^/(.*) https://security.openstack.org/$1 [last,redirect=permanent]
+  LogLevel warn
+  ErrorLog /var/log/apache2/security.openstack.org_error.log
+  CustomLog /var/log/apache2/security.openstack.org_access.log combined
+  ServerSignature Off
+</VirtualHost>
+
+<IfModule mod_ssl.c>
+<VirtualHost *:443>
+
+  ServerName security.openstack.org
+
+  DocumentRoot ${AFS_ROOT}
+
+  SSLCertificateFile      /etc/letsencrypt-certs/security.openstack.org/security.openstack.org.cer
+  SSLCertificateKeyFile   /etc/letsencrypt-certs/security.openstack.org/security.openstack.org.key
+  SSLCertificateChainFile /etc/letsencrypt-certs/security.openstack.org/ca.cer
+  SSLProtocol All -SSLv2 -SSLv3
+  # Note: this list should ensure ciphers that provide forward secrecy
+  SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:!AES256:!aNULL:!eNULL:!MD5:!DSS:!PSK:!SRP
+  SSLHonorCipherOrder on
+
+  <Directory ${AFS_ROOT}>
+    Options Indexes FollowSymLinks MultiViews
+    AllowOverrideList Redirect RedirectMatch
+    Satisfy Any
+    Require all granted
+  </Directory>
+
+  LogLevel warn
+  ErrorLog /var/log/apache2/security.openstack.org_error.log
+  CustomLog /var/log/apache2/security.openstack.org_access.log combined
+  ServerSignature Off
+
+</VirtualHost>
+</IfModule>

playbooks/roles/static/handlers/main.yaml

@@ -0,0 +1,4 @@
+- name: Reload apache2
+  service:
+    name: apache2
+    state: reloaded

playbooks/roles/static/tasks/main.yaml

@@ -0,0 +1,88 @@
+- name: Check AFS mounted
+  stat:
+    path: "/afs/openstack.org/project"
+  register: afs_root
+
+- name: Sanity check AFS
+  assert:
+    that:
+      - afs_root.stat.exists
+
+- name: Install apache2
+  apt:
+    name:
+      - apache2
+      - apache2-utils
+    state: present
+
+- name: Rewrite module
+  apache2_module:
+    state: present
+    name: rewrite
+
+- name: Substitute module
+  apache2_module:
+    state: present
+    name: substitute
+
+- name: Cache module
+  apache2_module:
+    state: present
+    name: cache
+
+- name: Cache disk module
+  apache2_module:
+    state: present
+    name: cache_disk
+
+- name: Apache macro module
+  apache2_module:
+    state: present
+    name: macro
+
+- name: Apache 2 ssl module
+  apache2_module:
+    state: present
+    name: ssl
+
+- name: Apache 2 headers module
+  apache2_module:
+    state: present
+    name: headers
+
+- name: Make sure default site disabled
+  command: a2dissite 000-default.conf
+  args:
+    removes: /etc/apache2/sites-enabled/000-default.conf
+
+# governance.openstack.org
+- name: Install governance.openstack.org
+  copy:
+    src: 50-governance.openstack.org.conf
+    dest: /etc/apache2/sites-available/
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Enable governance.openstack.org
+  command: a2ensite 50-governance.openstack.org
+  args:
+    creates: /etc/apache2/sites-enabled/50-governance.openstack.org
+  notify:
+    - Reload apache2
+
+# security.openstack.org
+- name: Install security.openstack.org
+  copy:
+    src: 50-security.openstack.org.conf
+    dest: /etc/apache2/sites-available/
+    owner: root
+    group: root
+    mode: 0644
+
+- name: Enable security.openstack.org
+  command: a2ensite 50-security.openstack.org
+  args:
+    creates: /etc/apache2/sites-enabled/50-security.openstack.org
+  notify:
+    - Reload apache2

playbooks/service-static.yaml

@@ -0,0 +1,12 @@
+- hosts: "static_opendev:!disabled"
+  name: "Static webserver"
+  roles:
+    - role: kerberos-client
+      kerberos_realm: 'OPENSTACK.ORG'
+      kerberos_admin_server: 'kdc.openstack.org'
+      kerberos_kdcs:
+        - kdc03.openstack.org
+        - kdc04.openstack.org
+    - role: openafs-client
+      openafs_client_cache_size: "{{ afs_client_cache_size | default(50000000) }}" # 50GiB
+    - role: static

testinfra/test_static.py

@@ -0,0 +1,33 @@
+# Copyright 2019 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+
+testinfra_hosts = ['static01.opendev.org']
+
+
+def test_apache(host):
+    apache = host.service('apache2')
+    assert apache.is_running
+
+def test_governance_openstack_org(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve governance.openstack.org:443:127.0.0.1 '
+                   'https://governance.openstack.org/')
+    assert 'OpenStack Governance' in cmd.stdout
+
+def test_security_openstack_org(host):
+    cmd = host.run('curl --insecure '
+                   '--resolve security.openstack.org:443:127.0.0.1 '
+                   'https://security.openstack.org/')
+    assert 'OpenStack Security Project' in cmd.stdout