Merge "Generate ssl check list directly from letsencrypt variables"

inventory/groups.yaml

@@ -28,6 +28,8 @@ groups:
 #  backup-server:
 #    - backup[0-9]*.opendev.org
   cacti: cacti[0-9]*.open*.org
+  certcheck:
+    - cacti[0-9]*.open*.org
   cloud-launcher:
     - bridge.openstack.org
   codesearch:

manifests/site.pp

@@ -28,7 +28,9 @@ node /^health\d*\.openstack\.org$/ {
 # Node-OS: xenial
 node /^cacti\d+\.open.*\.org$/ {
   $group = "cacti"
-  include openstack_project::ssl_cert_check
+  # NOTE(ianw) 2020-05 : disabled pending removal, migrated to
+  # ansible.
+  #  include openstack_project::ssl_cert_check
   class { 'openstack_project::cacti':
     cacti_hosts => hiera_array('cacti_hosts'),
     vhost_name  => 'cacti.openstack.org',

playbooks/group_vars/certcheck.yaml

@@ -0,0 +1,17 @@
+letsencrypt_certcheck_additional_domains:
+  - ask.openstack.org 443
+  - ethercalc.openstack.org 443
+  - etherpad.openstack.org 443
+  - firehose.openstack.org 8883
+  - git.openstack.org 443
+  - openstackid-dev.openstack.org 443
+  - openstackid.org 443
+  - refstack.openstack.org 443
+  - review.openstack.org 443
+  - storyboard.openstack.org 443
+  - survey.openstack.org 443
+  - static.openstack.org 443
+  - translate.openstack.org 443
+  - wiki.openstack.org 443
+  - www.openstack.org 443
+  - zuul.openstack.org 443

playbooks/host_vars/gitea01.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea01-main:
-    - gitea01.opendev.org
+    - gitea01.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea02.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea02-main:
-    - gitea02.opendev.org
+    - gitea02.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea03.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea03-main:
-    - gitea03.opendev.org
+    - gitea03.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea04.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea04-main:
-    - gitea04.opendev.org
+    - gitea04.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea05.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea05-main:
-    - gitea05.opendev.org
+    - gitea05.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea06.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea06-main:
-    - gitea06.opendev.org
+    - gitea06.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea07.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea07-main:
-    - gitea07.opendev.org
+    - gitea07.opendev.org:3000
     - opendev.org

playbooks/host_vars/gitea08.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   gitea08-main:
-    - gitea08.opendev.org
+    - gitea08.opendev.org:3000
     - opendev.org

playbooks/host_vars/insecure-ci-registry01.opendev.org.yaml

@@ -1,4 +1,4 @@
 letsencrypt_certs:
   insecure-ci-registry01-main:
-    - insecure-ci-registry01.opendev.org
+    - insecure-ci-registry01.opendev.org:5000
     - insecure-ci-registry.opendev.org

playbooks/roles/install-certcheck/README.rst

@@ -0,0 +1,24 @@
+Install ssl-cert-check
+
+Installs the ssl-cert-check tool and a cron job to check the freshness
+of the SSL certificates for the configured domains daily.
+
+**Role Variables**
+
+.. zuul:rolevar:: ssl_cert_check_domain_list
+   :default: /var/lib/certcheck/domainlist
+
+   The list of domains to check
+
+.. zuul:rolevar:: ssl_cert_check_days
+   :default: 30
+
+   Warn about certificates who have less than this number of days to
+   expiry.
+
+.. zuul:rolevar:: ssl_cert_check_email
+   :default: root
+
+   The email to send reports to
+
+

playbooks/roles/install-certcheck/defaults/main.yaml

@@ -0,0 +1,3 @@
+cert_check_domain_list: /var/lib/certcheck/domainlist
+cert_check_days: 30
+cert_check_email: root

playbooks/roles/install-certcheck/tasks/main.yaml

@@ -0,0 +1,32 @@
+- name: Ensure dependencies
+  package:
+    name:
+      - openssl
+      - bsd-mailx
+
+- name: Ensure certcheck user
+  user:
+    name: certcheck
+    comment: User for SSL validation
+
+- name: Ensure certcheck config directory
+  file:
+    state: directory
+    path: '{{ cert_check_domain_list | dirname }}'
+    owner: certcheck
+    group: certcheck
+    mode: 0755
+
+- name: Pull latest ssl-cert-check from git
+  git:
+    repo: 'https://github.com/Matty9191/ssl-cert-check'
+    dest: /opt/ssl-cert-check
+
+- name: Install cron job
+  cron:
+    user: certcheck
+    name: 'Run certcheck'
+    state: present
+    job: "/opt/ssl-cert-check/ssl-cert-check -a -q -f {{ cert_check_domain_list }} -x {{ cert_check_days }} -e {{ cert_check_email }}"
+    hour: 12
+    minute: 04

playbooks/roles/letsencrypt-config-certcheck/README.rst

@@ -0,0 +1,27 @@
+Generate SSL check list
+
+This role automatically generates a list of domains for the
+certificate validation checks.  This ensures our certificates are
+valid and are being renewed as expected.
+
+This role must run after ``letsencrypt-request-certs`` role, as that
+builds the ``letsencrypt_certcheck_domains`` variable for each
+host and certificate.  It must also run on a host that has already run
+the ``install-certcheck`` role.
+
+**Role Variables**
+
+.. zuul:rolevar:: letsencrypt_certcheck_domain_list
+   :default: /var/lib/certcheck/ssldomains
+
+   The ssl-cert-check domain configuration file to write.  See also
+   the ``install-certcheck`` role.
+
+.. zuul:rolevar:: letsencrypt_certcheck_additional_domains
+   :default: []
+
+   A list of additional domains to check for hosts not using the
+   ``letsencrypt-*`` roles.  Each entry should be in the format
+   ``hostname port``.
+
+

playbooks/roles/letsencrypt-config-certcheck/defaults/main.yaml

@@ -0,0 +1 @@
+letsencrypt_certcheck_domain_list: /var/lib/certcheck/ssldomains

playbooks/roles/letsencrypt-config-certcheck/tasks/main.yaml

@@ -0,0 +1,17 @@
+- name: Make domain list
+  set_fact:
+    letsencrypt_certcheck_domains: []
+
+- name: Build SSL domain list
+  set_fact:
+    letsencrypt_certcheck_domains: '{{ letsencrypt_certcheck_domains }} + {{ hostvars[item]["letsencrypt_certcheck_domains" ] }}'
+  with_inventory_hostnames:
+    - letsencrypt:!disabled
+
+- name: Write configuration file
+  template:
+    dest: '{{ letsencrypt_certcheck_domain_list }}'
+    src: ssldomains.j2
+    owner: certcheck
+    group: certcheck
+    mode: 0644

playbooks/roles/letsencrypt-config-certcheck/templates/ssldomains.j2

@@ -0,0 +1,6 @@
+{% for domain in letsencrypt_certcheck_domains %}
+{{ domain }}
+{% endfor %}
+{% for domain in letsencrypt_certcheck_additional_domains %}
+{{ domain }}
+{% endfor %}

playbooks/roles/letsencrypt-create-certs/tasks/acme.yaml

@@ -1,6 +1,6 @@
 - name: 'Build arguments for letsencrypt acme.sh driver for: {{ item.key }}'
   set_fact:
-    acme_args: '"{% for domain in item.value %}-d {{ domain }} {% endfor %}"'
+    acme_args: '"{% for domain in item.value %}-d {{ domain.split(":")[0] }} {% endfor %}"'
 
 - name: 'Run acme.sh driver for {{ item.key }} certificate issue'
   shell:
@@ -12,4 +12,4 @@
     LETSENCRYPT_STAGING: '{{ "1" if letsencrypt_use_staging else "0" }}'
   notify: 'letsencrypt updated {{ item.key }}'
 
-# Keys generated!
+# Keys generated!

playbooks/roles/letsencrypt-request-certs/README.rst

@@ -62,3 +62,12 @@ provision process.
      _acme-challenge.hostname01.opendev.org.  IN   CNAME  acme.opendev.org.
      _acme-challenge.hostname.opendev.org.    IN   CNAME  acme.opendev.org.
      _acme-challenge.foo.opendev.org.         IN   CNAME  acme.opendev.org.
+
+   The hostname in the first entry for each certificate will be
+   registered with the ``letsencrypt-config-certcheck`` for periodic
+   freshness tests; from the example above, ``hostname01.opendev.org``
+   and ``foo.opendev.org`` would be checked.  By default this will
+   check on port 443; if the certificate is actually active on another
+   port you can specify it after a colon;
+   e.g. ``foo.opendev.org:5000`` would indicate this host listens with
+   this certificate on port 5000.

playbooks/roles/letsencrypt-request-certs/tasks/acme.yaml

@@ -2,7 +2,7 @@
   set_fact:
     # NOTE(ianw): note the domains are passed in one string (between
     # ") as it makes argument parsing a little easier in the driver.sh
-    acme_args: '"{% for domain in cert.value %}-d {{ domain }} {% endfor %}"'
+    acme_args: '"{% for domain in cert.value %}-d {{ domain.split(":")[0] }} {% endfor %}"'
 
 - name: Run acme.sh driver for certificate issue
   shell:

playbooks/roles/letsencrypt-request-certs/tasks/main.yaml

@@ -1,10 +1,6 @@
 - set_fact:
     acme_txt_required: []
 
-- name: Show cert list
-  debug:
-    var: letsencrypt_certs
-
 # Handle multiple certs for a single host; like
 #
 # letsencrypt_certs:
@@ -22,5 +18,25 @@
   loop_control:
     loop_var: cert
 
-- debug:
-    var: acme_txt_required
+- name: Create ssl check domain list
+  # For each generated certificate get the first entry as the domain
+  # to run the certificate validation tests against.  If it specifies
+  # a port explicitly, use that, otherwise assume 443.
+  #
+  # Later in ssl-check role, the final certificate validation list is
+  # generated by walking the letsencrypt_certcheck_domains variable
+  # for each host in the letsencrypt group.
+  set_fact:
+    letsencrypt_certcheck_domains: >-
+      {%- set d = [] -%}
+      {%- for cert in letsencrypt_certs.keys() -%}
+       {%- for host in letsencrypt_certs[cert] -%}
+        {%- if loop.first -%}
+         {%- if not ":" in host -%}
+           {%- set host = host+":443" -%}
+         {%- endif -%}
+         {%- set d = d.append(host.replace(":"," ")) -%}
+        {% endif %}
+       {% endfor %}
+      {% endfor %}
+      {{- d -}}

playbooks/service-letsencrypt.yaml

@@ -1,6 +1,9 @@
 # This needs to happen in order.  letsencrypt hosts export their TXT
 # authentication records which is installed onto adns1, and then the
 # hosts verify to issue/renew keys
+- hosts: "certcheck:!disabled"
+  roles:
+    - install-certcheck
 - hosts: "letsencrypt:!disabled"
   name: "Base: deploy and renew certificates"
   roles:
@@ -14,3 +17,6 @@
   name: "Create certs"
   roles:
     - letsencrypt-create-certs
+- hosts: "certcheck:!disabled"
+  roles:
+    - letsencrypt-config-certcheck

playbooks/zuul/templates/gate-groups.yaml.j2

@@ -5,6 +5,9 @@ groups:
   docker:
     - bionic-docker
 
+  certcheck:
+    - bridge.openstack.org
+
   letsencrypt:
     - letsencrypt01.opendev.org
     - letsencrypt02.opendev.org

playbooks/zuul/templates/host_vars/letsencrypt01.opendev.org.yaml.j2

@@ -1,7 +1,7 @@
 letsencrypt_certs:
   letsencrypt01-main-service:
-    - letsencrypt01.opendev.org
+    - letsencrypt01.opendev.org:5000
     - letsencrypt.opendev.org
     - alias.opendev.org
   letsencrypt01-other-service:
-    - someotherservice.opendev.org
+    - someotherservice.opendev.org

testinfra/test_letsencrypt.py

@@ -15,6 +15,7 @@
 import pytest
 
 testinfra_hosts = ['adns-letsencrypt.opendev.org',
+                   'bridge.openstack.org',
                    'letsencrypt01.opendev.org',
                    'letsencrypt02.opendev.org']
 
@@ -138,3 +139,25 @@ def test_acme_sh_config(host):
     config = host.file('/root/.acme.sh/account.conf')
     assert config.exists
     assert config.contains("^ACCOUNT_EMAIL='le-test@opendev.org'")
+
+def test_certcheck_config(host, zuul_data):
+    if host.backend.get_hostname() != 'bridge.openstack.org':
+        pytest.skip()
+
+    if zuul_data['extra']['zuul']['job'] != 'system-config-run-letsencrypt':
+        pytest.skip()
+
+    domainlist = host.file('/var/lib/certcheck/ssldomains')
+
+    # TODO(ianw): figure out a flag or something from the
+    # system-config-run-letsencrypt test so that we can assert this
+    # file exists only in that case.
+    if not domainlist.exists:
+        pytest.skip()
+
+    assert domainlist.exists
+    assert domainlist.user == 'certcheck'
+    # from variables
+    assert domainlist.contains('^letsencrypt01.opendev.org 5000')
+    # from extra list; may need to change if list is modified
+    assert domainlist.contains('^wiki.openstack.org 443')

zuul.d/system-config-run.yaml

@@ -191,6 +191,9 @@
         - playbooks/service-nameserver.yaml
         - playbooks/service-letsencrypt.yaml
     host-vars:
+      bridge.openstack.org:
+        host_copy_output:
+          '/var/lib/certcheck': logs
       letsencrypt01.opendev.org:
         host_copy_output:
           '/var/log/acme.sh': logs