Add Log Rotate Side Car Container

We need a side car container to perform log rotation
on the log files. Logs shall be retained for 30 days.
This is the default setting and can be changed by updating
values.yaml

Also cleaned up README.md

Change-Id: I39a7797e96abd349160d753f8917f7f78f7d8797
changes/87/569187/1
Anthony Lin 5 years ago
parent 80210df387
commit 6c6acbfc80

@ -146,6 +146,34 @@ spec:
{{- end }}
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- if .Values.prod_environment }}
- name: airflow-logrotate
image: {{ .Values.images.tags.airflow }}
imagePullPolicy: {{ .Values.images.pull_policy }}
{{ tuple $envAll $envAll.Values.pod.resources.airflow.logrotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}
env:
- name: DAYS_BEFORE_LOG_DELETION
value: {{ .Values.logrotate.days_before_deletion | quote }}
- name: LOGROTATE_PATH
value: {{ .Values.conf.airflow.core.base_log_folder }}
- name: LOGROTATE_ENABLED
value: "enabled"
readinessProbe:
exec:
command:
- sh
- -c
- |-
set -ex
ps -ef | grep 'sleep 3600' | grep -v 'grep'
initialDelaySeconds: 15
periodSeconds: 15
securityContext:
privileged: true
volumeMounts:
- name: airflow-logs
mountPath: {{ .Values.conf.airflow.core.base_log_folder }}
{{- end }}
volumes:
- name: airflow-etc
configMap:

@ -135,6 +135,9 @@ volume_worker:
class_name: general
size: 5Gi
logrotate:
days_before_deletion: 30
# typically overriden by environmental
# values, but should include all endpoints
# required by this chart
@ -547,51 +550,51 @@ pod:
timeout: 30
resources:
enabled: false
shipyard_api:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
airflow:
worker:
flower:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
flower:
logrotate:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
web:
scheduler:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
scheduler:
web:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
jobs:
shipyard_db_sync:
worker:
limits:
memory: "128Mi"
cpu: "500m"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "500m"
shipyard_db_init:
cpu: "100m"
shipyard_api:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
jobs:
airflow_db_init:
limits:
memory: "128Mi"
cpu: "500m"
@ -605,35 +608,41 @@ pod:
requests:
memory: "128Mi"
cpu: "500m"
airflow_db_init:
ks_endpoints:
limits:
memory: "128Mi"
cpu: "500m"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "500m"
ks_user:
cpu: "100m"
ks_service:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_service:
ks_user:
limits:
memory: "128Mi"
cpu: "100m"
requests:
memory: "128Mi"
cpu: "100m"
ks_endpoints:
shipyard_db_init:
limits:
memory: "128Mi"
cpu: "100m"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "100m"
cpu: "500m"
shipyard_db_sync:
limits:
memory: "128Mi"
cpu: "500m"
requests:
memory: "128Mi"
cpu: "500m"
manifests:
configmap_shipyard_bin: true
configmap_shipyard_etc: true

@ -1,4 +1,4 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -16,8 +16,19 @@
FROM ubuntu:16.04
# Do not prompt user for choices on installation/configuration of packages
# Set port 8080 for Airflow Web
# Set port 5555 for Airflow Flower
# Set port 8793 for Airflow Worker
ENV DEBIAN_FRONTEND noninteractive
ENV container docker
ENV WEB_PORT 8080
ENV FLOWER_PORT 5555
ENV WORKER_PORT 8793
# Expose port for applications
EXPOSE $WEB_PORT
EXPOSE $FLOWER_PORT
EXPOSE $WORKER_PORT
# Airflow Home Directory
ARG AIRFLOW_HOME=/usr/local/airflow
@ -57,8 +68,6 @@ RUN set -ex && \
/usr/share/doc-base
# Copy dependency requirements
# Install DryDock libraries
# Install Armada libraries
COPY ./requirements.txt /tmp/
RUN pip3 install -r /tmp/requirements.txt
@ -79,19 +88,16 @@ RUN curl -L -o /usr/local/bin/kubectl \
https://storage.googleapis.com/kubernetes-release/release/v${KUBECTL_VERSION}/bin/linux/amd64/kubectl \
&& chmod +x /usr/local/bin/kubectl
# Copy entrypoint.sh and airflow_start_service.sh
# Copy entrypoint.sh, airflow_start_service.sh and
# airflow_logrotate.sh
COPY script/entrypoint.sh ${AIRFLOW_HOME}/entrypoint.sh
COPY script/airflow_start_service.sh ${AIRFLOW_HOME}/airflow_start_service.sh
COPY script/airflow_logrotate.sh ${AIRFLOW_HOME}/airflow_logrotate.sh
# Change permissions
RUN chown -R airflow: ${AIRFLOW_HOME} \
&& chmod +x ${AIRFLOW_HOME}/entrypoint.sh
# Expose port 8080 for Airflow Web
# Expose port 5555 for Airflow Flower
# Expose port 8793 for Airflow Worker
EXPOSE 8080 5555 8793
# Set work directory
USER airflow
WORKDIR ${AIRFLOW_HOME}

@ -1,7 +0,0 @@
## Docker Build ##
We can build the airflow image by executing the following command:
```
docker build -t attcomdev/airflow-helm:v0.1.0 .
```

@ -0,0 +1,29 @@
#!/bin/bash
#
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -x
while true; do
# Delete logs that are more than 30 days old in the directories
# under the Airflow log path
# Delete empty directories under the Airflow log path
find ${LOGROTATE_PATH} \( -type f -name '*.log' -mtime +${DAYS_BEFORE_LOG_DELETION} -o -type d -empty \) -print -delete
# Sleep for 1 hr between each wait loop
sleep 3600
done

@ -1,6 +1,6 @@
#!/bin/bash
#
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -14,7 +14,13 @@
# See the License for the specific language governing permissions and
# limitations under the License.
# Execute Airflow Start Service Script
CMD="bash /usr/local/airflow/airflow_start_service.sh"
if [[ ${LOGROTATE_ENABLED} == "enabled" ]]; then
# Execute log rotate script for log rotate side car
# container
CMD="bash /usr/local/airflow/airflow_logrotate.sh"
else
# Execute Airflow Start Service Script
CMD="bash /usr/local/airflow/airflow_start_service.sh"
fi
exec $CMD "$@"

Loading…
Cancel
Save