Add Log Rotate Side Car Container
We need a side car container to perform log rotation on the log files. Logs shall be retained for 30 days. This is the default setting and can be changed by updating values.yaml Also cleaned up README.md Change-Id: I39a7797e96abd349160d753f8917f7f78f7d8797
This commit is contained in:
parent
80210df387
commit
6c6acbfc80
|
@ -146,6 +146,34 @@ spec:
|
|||
{{- end }}
|
||||
- name: airflow-logs
|
||||
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
|
||||
{{- if .Values.prod_environment }}
|
||||
- name: airflow-logrotate
|
||||
image: {{ .Values.images.tags.airflow }}
|
||||
imagePullPolicy: {{ .Values.images.pull_policy }}
|
||||
{{ tuple $envAll $envAll.Values.pod.resources.airflow.logrotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
|
||||
env:
|
||||
- name: DAYS_BEFORE_LOG_DELETION
|
||||
value: {{ .Values.logrotate.days_before_deletion | quote }}
|
||||
- name: LOGROTATE_PATH
|
||||
value: {{ .Values.conf.airflow.core.base_log_folder }}
|
||||
- name: LOGROTATE_ENABLED
|
||||
value: "enabled"
|
||||
readinessProbe:
|
||||
exec:
|
||||
command:
|
||||
- sh
|
||||
- -c
|
||||
- |-
|
||||
set -ex
|
||||
ps -ef | grep 'sleep 3600' | grep -v 'grep'
|
||||
initialDelaySeconds: 15
|
||||
periodSeconds: 15
|
||||
securityContext:
|
||||
privileged: true
|
||||
volumeMounts:
|
||||
- name: airflow-logs
|
||||
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
|
||||
{{- end }}
|
||||
volumes:
|
||||
- name: airflow-etc
|
||||
configMap:
|
||||
|
|
|
@ -135,6 +135,9 @@ volume_worker:
|
|||
class_name: general
|
||||
size: 5Gi
|
||||
|
||||
logrotate:
|
||||
days_before_deletion: 30
|
||||
|
||||
# typically overriden by environmental
|
||||
# values, but should include all endpoints
|
||||
# required by this chart
|
||||
|
@ -547,21 +550,7 @@ pod:
|
|||
timeout: 30
|
||||
resources:
|
||||
enabled: false
|
||||
shipyard_api:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
airflow:
|
||||
worker:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
flower:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
|
@ -569,7 +558,7 @@ pod:
|
|||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
web:
|
||||
logrotate:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
|
@ -583,15 +572,29 @@ pod:
|
|||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
jobs:
|
||||
shipyard_db_sync:
|
||||
web:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
shipyard_db_init:
|
||||
cpu: "100m"
|
||||
worker:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
shipyard_api:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
jobs:
|
||||
airflow_db_init:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
|
@ -605,14 +608,7 @@ pod:
|
|||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
airflow_db_init:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
ks_user:
|
||||
ks_endpoints:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
|
@ -626,14 +622,27 @@ pod:
|
|||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
ks_endpoints:
|
||||
ks_user:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "100m"
|
||||
|
||||
shipyard_db_init:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
shipyard_db_sync:
|
||||
limits:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
requests:
|
||||
memory: "128Mi"
|
||||
cpu: "500m"
|
||||
manifests:
|
||||
configmap_shipyard_bin: true
|
||||
configmap_shipyard_etc: true
|
||||
|
|
|
@ -1,4 +1,4 @@
|
|||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -16,8 +16,19 @@
|
|||
FROM ubuntu:16.04
|
||||
|
||||
# Do not prompt user for choices on installation/configuration of packages
|
||||
# Set port 8080 for Airflow Web
|
||||
# Set port 5555 for Airflow Flower
|
||||
# Set port 8793 for Airflow Worker
|
||||
ENV DEBIAN_FRONTEND noninteractive
|
||||
ENV container docker
|
||||
ENV WEB_PORT 8080
|
||||
ENV FLOWER_PORT 5555
|
||||
ENV WORKER_PORT 8793
|
||||
|
||||
# Expose port for applications
|
||||
EXPOSE $WEB_PORT
|
||||
EXPOSE $FLOWER_PORT
|
||||
EXPOSE $WORKER_PORT
|
||||
|
||||
# Airflow Home Directory
|
||||
ARG AIRFLOW_HOME=/usr/local/airflow
|
||||
|
@ -57,8 +68,6 @@ RUN set -ex && \
|
|||
/usr/share/doc-base
|
||||
|
||||
# Copy dependency requirements
|
||||
# Install DryDock libraries
|
||||
# Install Armada libraries
|
||||
COPY ./requirements.txt /tmp/
|
||||
RUN pip3 install -r /tmp/requirements.txt
|
||||
|
||||
|
@ -79,19 +88,16 @@ RUN curl -L -o /usr/local/bin/kubectl \
|
|||
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
|
||||
&& chmod +x /usr/local/bin/kubectl
|
||||
|
||||
# Copy entrypoint.sh and airflow_start_service.sh
|
||||
# Copy entrypoint.sh, airflow_start_service.sh and
|
||||
# airflow_logrotate.sh
|
||||
COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh
|
||||
COPY script/airflow_start_service.sh ${AIRFLOW_HOME}/airflow_start_service.sh
|
||||
COPY script/airflow_logrotate.sh ${AIRFLOW_HOME}/airflow_logrotate.sh
|
||||
|
||||
# Change permissions
|
||||
RUN chown -R airflow: ${AIRFLOW_HOME} \
|
||||
&& chmod +x ${AIRFLOW_HOME}/entrypoint.sh
|
||||
|
||||
# Expose port 8080 for Airflow Web
|
||||
# Expose port 5555 for Airflow Flower
|
||||
# Expose port 8793 for Airflow Worker
|
||||
EXPOSE 8080 5555 8793
|
||||
|
||||
# Set work directory
|
||||
USER airflow
|
||||
WORKDIR ${AIRFLOW_HOME}
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
## Docker Build ##
|
||||
|
||||
We can build the airflow image by executing the following command:
|
||||
|
||||
```
|
||||
docker build -t attcomdev/airflow-helm:v0.1.0 .
|
||||
```
|
|
@ -0,0 +1,29 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
set -x
|
||||
|
||||
while true; do
|
||||
|
||||
# Delete logs that are more than 30 days old in the directories
|
||||
# under the Airflow log path
|
||||
# Delete empty directories under the Airflow log path
|
||||
find ${LOGROTATE_PATH} \( -type f -name '*.log' -mtime +${DAYS_BEFORE_LOG_DELETION} -o -type d -empty \) -print -delete
|
||||
|
||||
# Sleep for 1 hr between each wait loop
|
||||
sleep 3600
|
||||
|
||||
done
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
|
||||
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
@ -14,7 +14,13 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Execute Airflow Start Service Script
|
||||
CMD="bash /usr/local/airflow/airflow_start_service.sh"
|
||||
if [[ ${LOGROTATE_ENABLED} == "enabled" ]]; then
|
||||
# Execute log rotate script for log rotate side car
|
||||
# container
|
||||
CMD="bash /usr/local/airflow/airflow_logrotate.sh"
|
||||
else
|
||||
# Execute Airflow Start Service Script
|
||||
CMD="bash /usr/local/airflow/airflow_start_service.sh"
|
||||
fi
|
||||
|
||||
exec $CMD "$@"
|
||||
|
|
Loading…
Reference in New Issue