From 6c6acbfc802e9f0fe5a03b6df206c58ad6c39cb1 Mon Sep 17 00:00:00 2001 From: Anthony Lin Date: Wed, 21 Feb 2018 08:35:19 +0000 Subject: [PATCH] Add Log Rotate Side Car Container We need a side car container to perform log rotation on the log files. Logs shall be retained for 30 days. This is the default setting and can be changed by updating values.yaml Also cleaned up README.md Change-Id: I39a7797e96abd349160d753f8917f7f78f7d8797 --- .../templates/statefulset-airflow-worker.yaml | 28 ++++++++ charts/shipyard/values.yaml | 69 +++++++++++-------- images/airflow/Dockerfile | 24 ++++--- images/airflow/README.md | 7 -- images/airflow/script/airflow_logrotate.sh | 29 ++++++++ images/airflow/script/entrypoint.sh | 12 +++- 6 files changed, 120 insertions(+), 49 deletions(-) delete mode 100644 images/airflow/README.md create mode 100644 images/airflow/script/airflow_logrotate.sh diff --git a/charts/shipyard/templates/statefulset-airflow-worker.yaml b/charts/shipyard/templates/statefulset-airflow-worker.yaml index e9abf094..e78dcbd0 100644 --- a/charts/shipyard/templates/statefulset-airflow-worker.yaml +++ b/charts/shipyard/templates/statefulset-airflow-worker.yaml @@ -146,6 +146,34 @@ spec: {{- end }} - name: airflow-logs mountPath: {{ .Values.conf.airflow.core.base_log_folder }} +{{- if .Values.prod_environment }} + - name: airflow-logrotate + image: {{ .Values.images.tags.airflow }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.airflow.logrotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: DAYS_BEFORE_LOG_DELETION + value: {{ .Values.logrotate.days_before_deletion | quote }} + - name: LOGROTATE_PATH + value: {{ .Values.conf.airflow.core.base_log_folder }} + - name: LOGROTATE_ENABLED + value: "enabled" + readinessProbe: + exec: + command: + - sh + - -c + - |- + set -ex + ps -ef | grep 'sleep 3600' | grep -v 'grep' + initialDelaySeconds: 15 + periodSeconds: 15 + securityContext: + privileged: true + volumeMounts: + - name: airflow-logs + mountPath: {{ .Values.conf.airflow.core.base_log_folder }} +{{- end }} volumes: - name: airflow-etc configMap: diff --git a/charts/shipyard/values.yaml b/charts/shipyard/values.yaml index 30043dc9..a52c2067 100644 --- a/charts/shipyard/values.yaml +++ b/charts/shipyard/values.yaml @@ -135,6 +135,9 @@ volume_worker: class_name: general size: 5Gi +logrotate: + days_before_deletion: 30 + # typically overriden by environmental # values, but should include all endpoints # required by this chart @@ -547,21 +550,7 @@ pod: timeout: 30 resources: enabled: false - shipyard_api: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" airflow: - worker: - limits: - memory: "128Mi" - cpu: "100m" - requests: - memory: "128Mi" - cpu: "100m" flower: limits: memory: "128Mi" @@ -569,7 +558,7 @@ pod: requests: memory: "128Mi" cpu: "100m" - web: + logrotate: limits: memory: "128Mi" cpu: "100m" @@ -583,15 +572,29 @@ pod: requests: memory: "128Mi" cpu: "100m" - jobs: - shipyard_db_sync: + web: limits: memory: "128Mi" - cpu: "500m" + cpu: "100m" requests: memory: "128Mi" - cpu: "500m" - shipyard_db_init: + cpu: "100m" + worker: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "128Mi" + cpu: "100m" + shipyard_api: + limits: + memory: "128Mi" + cpu: "100m" + requests: + memory: "128Mi" + cpu: "100m" + jobs: + airflow_db_init: limits: memory: "128Mi" cpu: "500m" @@ -605,14 +608,7 @@ pod: requests: memory: "128Mi" cpu: "500m" - airflow_db_init: - limits: - memory: "128Mi" - cpu: "500m" - requests: - memory: "128Mi" - cpu: "500m" - ks_user: + ks_endpoints: limits: memory: "128Mi" cpu: "100m" @@ -626,14 +622,27 @@ pod: requests: memory: "128Mi" cpu: "100m" - ks_endpoints: + ks_user: limits: memory: "128Mi" cpu: "100m" requests: memory: "128Mi" cpu: "100m" - + shipyard_db_init: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + shipyard_db_sync: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" manifests: configmap_shipyard_bin: true configmap_shipyard_etc: true diff --git a/images/airflow/Dockerfile b/images/airflow/Dockerfile index 61b7f678..7c93b277 100644 --- a/images/airflow/Dockerfile +++ b/images/airflow/Dockerfile @@ -1,4 +1,4 @@ -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -16,8 +16,19 @@ FROM ubuntu:16.04 # Do not prompt user for choices on installation/configuration of packages +# Set port 8080 for Airflow Web +# Set port 5555 for Airflow Flower +# Set port 8793 for Airflow Worker ENV DEBIAN_FRONTEND noninteractive ENV container docker +ENV WEB_PORT 8080 +ENV FLOWER_PORT 5555 +ENV WORKER_PORT 8793 + +# Expose port for applications +EXPOSE $WEB_PORT +EXPOSE $FLOWER_PORT +EXPOSE $WORKER_PORT # Airflow Home Directory ARG AIRFLOW_HOME=/usr/local/airflow @@ -57,8 +68,6 @@ RUN set -ex && \ /usr/share/doc-base # Copy dependency requirements -# Install DryDock libraries -# Install Armada libraries COPY ./requirements.txt /tmp/ RUN pip3 install -r /tmp/requirements.txt @@ -79,19 +88,16 @@ RUN curl -L -o /usr/local/bin/kubectl \ https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \ && chmod +x /usr/local/bin/kubectl -# Copy entrypoint.sh and airflow_start_service.sh +# Copy entrypoint.sh, airflow_start_service.sh and +# airflow_logrotate.sh COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh COPY script/airflow_start_service.sh ${AIRFLOW_HOME}/airflow_start_service.sh +COPY script/airflow_logrotate.sh ${AIRFLOW_HOME}/airflow_logrotate.sh # Change permissions RUN chown -R airflow: ${AIRFLOW_HOME} \ && chmod +x ${AIRFLOW_HOME}/entrypoint.sh -# Expose port 8080 for Airflow Web -# Expose port 5555 for Airflow Flower -# Expose port 8793 for Airflow Worker -EXPOSE 8080 5555 8793 - # Set work directory USER airflow WORKDIR ${AIRFLOW_HOME} diff --git a/images/airflow/README.md b/images/airflow/README.md deleted file mode 100644 index d33d88d4..00000000 --- a/images/airflow/README.md +++ /dev/null @@ -1,7 +0,0 @@ -## Docker Build ## - -We can build the airflow image by executing the following command: - -``` -docker build -t attcomdev/airflow-helm:v0.1.0 . -``` diff --git a/images/airflow/script/airflow_logrotate.sh b/images/airflow/script/airflow_logrotate.sh new file mode 100644 index 00000000..eda69d8d --- /dev/null +++ b/images/airflow/script/airflow_logrotate.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +while true; do + + # Delete logs that are more than 30 days old in the directories + # under the Airflow log path + # Delete empty directories under the Airflow log path + find ${LOGROTATE_PATH} \( -type f -name '*.log' -mtime +${DAYS_BEFORE_LOG_DELETION} -o -type d -empty \) -print -delete + + # Sleep for 1 hr between each wait loop + sleep 3600 + +done diff --git a/images/airflow/script/entrypoint.sh b/images/airflow/script/entrypoint.sh index e8aa28f1..415c33f0 100644 --- a/images/airflow/script/entrypoint.sh +++ b/images/airflow/script/entrypoint.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# Copyright 2018 AT&T Intellectual Property. All other rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,7 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Execute Airflow Start Service Script -CMD="bash /usr/local/airflow/airflow_start_service.sh" +if [[ ${LOGROTATE_ENABLED} == "enabled" ]]; then + # Execute log rotate script for log rotate side car + # container + CMD="bash /usr/local/airflow/airflow_logrotate.sh" +else + # Execute Airflow Start Service Script + CMD="bash /usr/local/airflow/airflow_start_service.sh" +fi exec $CMD "$@"