From 55da1e3d06ab034854a875abaaf6a31cb8203fdf Mon Sep 17 00:00:00 2001
From: Ian Wienand <iwienand@redhat.com>
Date: Fri, 21 Feb 2020 14:00:47 +1100
Subject: [PATCH] Revert "Generate list of 404s for docs.o.o"

This reverts commit c25e91f49632d8e187f35807f250567446bd5102.

This script parses the Apache logs and writes out a local count of the
404 data to files.openstack.org, and then exports it via
files.openstack.org.

As part of the spec [1] we're trying to remove publishing from local
volumes, in general.

Since this is not widely used, there is only one link to it, it's not
discoverable from the landing page of files.openstack.org (which just
shows the afs directory listing), it has a very long latency making it
not that useful for debugging and grepping the logs there have been no
accesses in the past 2 weeks (as far back as logs go) I propose we
remove it.

If we want to retain this, we should publish the output alongside the
docs AFS volume.  That could certainly be done by distributing the
docs keytab to the host and having it write out in a similar cron job.
Another option could be to setup a keypair for remote login and keep
that as a secret in Zuul, and do the same from a periodic job
(complicated by apache logs being root only, so needs some sudo magic
or similar).  Or, we could figure out an altogether better, privacy
respecting client analytics solution.

[1] https://docs.opendev.org/opendev/infra-specs/latest/specs/retire-static.html

Depends-On: https://review.opendev.org/709036
Change-Id: Iccf24a72cf82592bae8c699f9f857aa54fc74f10
---
 modules/openstack_project/files/files/404s.sh | 21 -----------------
 modules/openstack_project/manifests/files.pp  | 23 -------------------
 .../templates/files.vhost.erb                 |  6 -----
 3 files changed, 50 deletions(-)
 delete mode 100644 modules/openstack_project/files/files/404s.sh

diff --git a/modules/openstack_project/files/files/404s.sh b/modules/openstack_project/files/files/404s.sh
deleted file mode 100644
index 83abadd3ee..0000000000
--- a/modules/openstack_project/files/files/404s.sh
+++ /dev/null
@@ -1,21 +0,0 @@
-#!/bin/bash
-
-SOURCE_FILE=$1
-OUTPUT_DIR=$2
-INTERMEDIATE_FILE=$(mktemp)
-
-# Get just the lines with 404s in them
-grep ' 404 ' $SOURCE_FILE | sed -n -e 's/.*"GET \(\/.*\) HTTP\/1\.." 404 .*/\1/p' > $INTERMEDIATE_FILE
-
-if [ -f "$SOURCE_FILE.1" ] ; then
-    # We get roughly the last day's worth of logs by looking at the last two
-    # log files.
-    grep ' 404 ' $SOURCE_FILE.1 | sed -n -e 's/.*"GET \(\/.*\) HTTP\/1\.." 404 .*/\1/p' >> $INTERMEDIATE_FILE
-fi
-
-# Process those 404s to count them and return sorted by count
-sort $INTERMEDIATE_FILE | uniq -c | sort -rn | grep '\(html\|\/$\)' > $OUTPUT_DIR/$(date +%F).txt
-
-# cleanup
-rm $INTERMEDIATE_FILE
-find $OUTPUT_DIR -type f -name '*.txt' -mtime +30 -delete
diff --git a/modules/openstack_project/manifests/files.pp b/modules/openstack_project/manifests/files.pp
index ae4ac9a1b3..d511ddd58f 100644
--- a/modules/openstack_project/manifests/files.pp
+++ b/modules/openstack_project/manifests/files.pp
@@ -124,28 +124,6 @@ class openstack_project::files (
     notify => Service['httpd'],
   }
 
-  file {'/usr/local/bin/404s.sh':
-    ensure => present,
-    owner  => 'root',
-    group  => 'root',
-    mode   => '0755',
-    source => 'puppet:///modules/openstack_project/files/404s.sh',
-  }
-  file {'/var/www/docs-404s':
-    ensure => directory,
-    owner   => 'root',
-    group   => 'root',
-    mode    => '0755',
-  }
-  cron {'generate_docs_404s':
-    # This seems to be about half an hour after apache rotates logs.
-    hour        => '7',
-    minute      => '0',
-    environment => 'PATH=/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/bin',
-    command     => '404s.sh /var/log/apache2/docs.openstack.org_access.log /var/www/docs-404s/',
-    require     => File['/usr/local/bin/404s.sh'],
-  }
-
   ###########################################################
   # docs.openstack.org
 
@@ -181,7 +159,6 @@ class openstack_project::files (
     before  => File['/etc/ssl/certs/docs.openstack.org.pem'],
   }
 
-
   ###########################################################
   # developer.openstack.org
 
diff --git a/modules/openstack_project/templates/files.vhost.erb b/modules/openstack_project/templates/files.vhost.erb
index 4022097e29..2c870a1b47 100644
--- a/modules/openstack_project/templates/files.vhost.erb
+++ b/modules/openstack_project/templates/files.vhost.erb
@@ -23,12 +23,6 @@ NameVirtualHost <%= @vhost_name %>:<%= @port %>
       Require all granted
     </Directory>
 
-    Alias /docs-404s /var/www/docs-404s
-    <Directory "/var/www/docs-404s">
-      Options +Indexes
-      Require all granted
-    </Directory>
-
     ErrorLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_error.log
     LogLevel warn
     CustomLog /var/log/<%= scope.lookupvar("httpd::params::apache_name") %>/<%= @name %>_access.log combined