static: add a periodic 404 checker

This is an alternative to Iccf24a72cf82592bae8c699f9f857aa54fc74f10
which removes the 404 scraping tool.  It creates a zuul user and
enables login via the system-config per-project ssh key, and then runs
the 404 scraping script against it periodically.

Change-Id: I30467d791a7877b5469b173926216615eb57d035

.zuul.yaml

@@ -1145,6 +1145,12 @@
     vars:
       playbook_name: zuul_reconfigure.yaml
 
+- job:
+    name: system-config-static-404-checker
+    description: |
+      Run 404 scraping script on static.opendev.org
+    run: playbooks/periodic/404.yaml
+
 - project:
     templates:
       - system-config-zuul-role-integration
@@ -1312,3 +1318,6 @@
         - system-config-promote-image-haproxy-statsd
         - system-config-promote-image-python-base
         - system-config-promote-image-python-builder
+    periodic:
+      jobs:
+        - system-config-static-404-checker

playbooks/periodic/404.yaml

@@ -0,0 +1,37 @@
+- hosts: localhost
+  tasks:
+    - name: Add static.opendev.org to inventory
+      add_host:
+        name: static.opendev.org
+        ansible_connection: ssh
+        ansible_host: static.opendev.org
+        ansible_port: 22
+        ansible_user: zuul
+
+# NOTE(ianw): 2020-02-25 just for initial testing run this for one log
+# in a dumb way.  We can scrape a few more sites.  Overall, we expect
+# this to be replaced with a better analysis tool, see
+#   https://review.opendev.org/709236
+- hosts: static.opendev.org
+  tasks:
+    - name: Run 404 scraping script
+      become: yes
+      shell: |
+        SOURCE_FILE=/var/log/docs.openstack.org_access.log
+        INTERMEDIATE_FILE=$(mktemp)
+
+        # Get just the lines with 404s in them
+        grep ' 404 ' $SOURCE_FILE | sed -n -e 's/.*"GET \(\/.*\) HTTP\/1\.." 404 .*/\1/p' > $INTERMEDIATE_FILE
+
+        if [ -f "$SOURCE_FILE.1" ] ; then
+        # We get roughly the last days worth of logs by looking at the last two
+        # log files.
+        grep ' 404 ' $SOURCE_FILE.1 | sed -n -e 's/.*"GET \(\/.*\) HTTP\/1\.." 404 .*/\1/p' >> $INTERMEDIATE_FILE
+        fi
+
+        # Process those 404s to count them and return sorted by count
+        sort $INTERMEDIATE_FILE | uniq -c | sort -rn | grep '\(html\|\/$\)'
+
+        rm ${INTERMEDIATE_FILE}
+      args:
+        executable: /bin/bash

playbooks/roles/static/files/zuul.sudo

@@ -0,0 +1 @@
+zuul ALL=(ALL) NOPASSWD: ALL

playbooks/roles/static/tasks/main.yaml

@@ -71,3 +71,6 @@
     - 50-tarballs.opendev.org
     - 50-tarballs.openstack.org
     - 50-zuul-ci.org
+
+- name: Install zuul user
+  include_tasks: zuul.yaml

playbooks/roles/static/tasks/zuul.yaml

@@ -0,0 +1,20 @@
+- name: Create zuul user
+  user:
+    name: zuul
+    comment: User for running remote zuul jobs
+    shell: /bin/bash
+
+- name: Install sudo permissions for zuul
+  copy:
+    src: zuul.sudo
+    dest: '/etc/sudoers.d/zuul'
+    owner: root
+    group: root
+    mode: 0440
+
+- name: Install system-config per-project key for zuul
+  authorized_key:
+    user: zuul
+    state: present
+    key: |
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDcXd/QJDEprSLh6N6bULnhchf9M+uzYBEJ2b51Au67FON+5M6VEj5Ut+DlkEPhabOP+tSv9Cn1HpmpBjdEOXdmBj6JS7G/gBb4w28oZDyNjrPT2ebpRw/XnVEkGfikR2J+j3o7CV+ybhLDalXm2TUDReVXnONUq3YzZbjRzoYs0xxrxyss47vZP0xFpsAt9jCMAJW2k6H589VUY38k9LFyhZUZ72FB6eJ68B9GN0TimBYm2DqvupBGQrRhkP8OZ0WoBV8PulKXaHVFdmfBNHB7E7FLlZKuiM6nkV4bOWMGOB/TF++wXBK86t9po3pWCM7+kr72xGRTE+6LuZ2z1K+h'

testinfra/test_static.py

@@ -21,6 +21,14 @@ def test_apache(host):
     apache = host.service('apache2')
     assert apache.is_running
 
+def test_zuul_user(host):
+    user = host.user('zuul')
+    assert user.exists
+
+    authorized_keys = host.file('%s/.ssh/authorized_keys' %
+                                user.home)
+    assert authorized_keys.exists
+
 def test_static_opendev_org(host):
     cmd = host.run('curl --insecure '
                    '--resolve static.opendev.org:443:127.0.0.1 '