From 25aa3690252c605d237c187524d0adf73a8424e5 Mon Sep 17 00:00:00 2001
From: Apurva Gokani <apurvagokani@gmail.com>
Date: Wed, 6 Jan 2021 17:57:10 -0600
Subject: [PATCH] postgres archive cleanup script

This change adds  cleanup mechanism to archive by following steps:
1) add archive_cleanup.sh under /tmp directory
2) through the start.sh this script will be triggered
3) It runs every hour, checking utilization of archive dir
4) If it is above threshold it deletes half of old files

Change-Id: I918284b0aa5a698a6028b9807fcbf6559ef0ff45
---
 postgresql/Chart.yaml                         |  2 +-
 .../bin/_postgresql_archive_cleanup.sh.tpl    | 46 +++++++++++++++++++
 postgresql/templates/bin/_start.sh.tpl        |  2 +
 postgresql/templates/configmap-bin.yaml       |  1 +
 postgresql/templates/statefulset.yaml         |  8 ++++
 postgresql/values.yaml                        |  1 +
 6 files changed, 59 insertions(+), 1 deletion(-)
 create mode 100644 postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl

diff --git a/postgresql/Chart.yaml b/postgresql/Chart.yaml
index 7ebe66d8a..c11a455ae 100644
--- a/postgresql/Chart.yaml
+++ b/postgresql/Chart.yaml
@@ -15,7 +15,7 @@ apiVersion: v1
 appVersion: v9.6
 description: OpenStack-Helm PostgreSQL
 name: postgresql
-version: 0.1.6
+version: 0.1.7
 home: https://www.postgresql.org
 sources:
   - https://github.com/postgres/postgres
diff --git a/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl b/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl
new file mode 100644
index 000000000..d8ed7bb1b
--- /dev/null
+++ b/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl
@@ -0,0 +1,46 @@
+#!/bin/bash
+
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+set +ex
+
+# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage
+clean_up () {
+  echo "Cleanup required as Utilization is above threshold"
+  # Get file count and delete half of the archive while maintaining the order of the files
+  FILE_COUNT=$(ls -1 ${ARCHIVE_PATH} | sort | wc -l)
+  COUNT=0
+  echo $((FILE_COUNT/2))
+  for file in $(ls -1 ${ARCHIVE_PATH} | sort); do
+    if [[ $COUNT -lt $((FILE_COUNT/2)) ]]; then
+      echo "removing following file $file"
+      rm -rf ${ARCHIVE_PATH}/$file
+    else
+      break
+    fi
+    COUNT=$((COUNT+1))
+  done
+}
+#infinite loop to check the utilization of archive
+while true
+do
+  # checking the utilization of archive directory
+  UTILIZATION=$(df -h ${ARCHIVE_PATH} | awk ' NR==2 {print $5} ' | awk '{ print substr( $0, 1, length($0)-1 ) }')
+  if [[ $UTILIZATION -gt ${ARCHIVE_LIMIT} ]];
+  then
+    clean_up
+  fi
+  sleep 3600
+done
+
+
diff --git a/postgresql/templates/bin/_start.sh.tpl b/postgresql/templates/bin/_start.sh.tpl
index b671761c1..14d56a273 100644
--- a/postgresql/templates/bin/_start.sh.tpl
+++ b/postgresql/templates/bin/_start.sh.tpl
@@ -35,4 +35,6 @@ fi
 
 set -x
 
+bash /tmp/archive_cleanup.sh &
+
 exec /docker-entrypoint.sh postgres -c config_file=/tmp/postgresql.conf
diff --git a/postgresql/templates/configmap-bin.yaml b/postgresql/templates/configmap-bin.yaml
index 2c0e502dd..b5b8ec151 100644
--- a/postgresql/templates/configmap-bin.yaml
+++ b/postgresql/templates/configmap-bin.yaml
@@ -28,6 +28,7 @@ data:
 {{- end }}
   start.sh: {{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
   readiness.sh: {{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
+  archive_cleanup.sh: {{ tuple "bin/_postgresql_archive_cleanup.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
   db_test.sh: {{ tuple "bin/_db_test.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
 {{- if .Values.conf.backup.enabled }}
   backup_postgresql.sh: {{ tuple "bin/_backup_postgresql.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
diff --git a/postgresql/templates/statefulset.yaml b/postgresql/templates/statefulset.yaml
index 221f8c64e..082725197 100644
--- a/postgresql/templates/statefulset.yaml
+++ b/postgresql/templates/statefulset.yaml
@@ -169,6 +169,10 @@ spec:
           env:
             - name: PGDATA
               value: "{{ .Values.storage.mount.path }}/pgdata"
+            - name: ARCHIVE_LIMIT
+              value: "{{ .Values.storage.archive.archive_limit }}"
+            - name: ARCHIVE_PATH
+              value: "{{ .Values.storage.archive.mount_path }}"
             - name: KUBERNETES_NAMESPACE
               valueFrom:
                 fieldRef:
@@ -226,6 +230,10 @@ spec:
             - name: postgresql-archive
               mountPath: {{ .Values.storage.archive.mount_path }}
               subPath: {{ .Values.storage.mount.subpath }}
+            - name: postgresql-bin
+              mountPath: /tmp/archive_cleanup.sh
+              subPath: archive_cleanup.sh
+              readOnly: true
 {{- end }}
       volumes:
         - name: pod-tmp
diff --git a/postgresql/values.yaml b/postgresql/values.yaml
index ee50bb734..bd949c483 100644
--- a/postgresql/values.yaml
+++ b/postgresql/values.yaml
@@ -190,6 +190,7 @@ storage:
     subpath: .
   archive:
     mount_path: /var/lib/archive
+    archive_limit: 60
 
 labels:
   server: