Ceph-OSD: Add log tailer to ceph-osd

This change adds in a log tailer script in a pod that tails the ceph
logs, periodicly truncating them.

Change-Id: Idbe03d4123f86b1b88e277fea6d13f58104f94b0
Signed-off-by: Pete Birley <pete@port.direct>
This commit is contained in:
Pete Birley 2020-03-26 14:15:51 -05:00 committed by Pete Birley
parent aa01846f60
commit 5701d26a13
9 changed files with 90 additions and 1 deletions

View File

@ -76,6 +76,9 @@ fi
mkdir -p /etc/forego/"${CLUSTER}"
echo "" > /etc/forego/"${CLUSTER}"/Procfile
# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
echo "${OSD_ID}" > /tmp/osd-id
for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do
OSD_PATH="$OSD_PATH_BASE-$OSD_ID/"
OSD_KEYRING="${OSD_PATH%/}/keyring"

View File

@ -0,0 +1,39 @@
#!/bin/bash
set -ex
osd_id_file="/tmp/osd-id"
function wait_for_file() {
local file="$1"; shift
local wait_seconds="${1:-30}"; shift
until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do
sleep 1
done
((++wait_seconds))
}
wait_for_file "${osd_id_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}"
log_file="/var/log/ceph/${DAEMON_NAME}.$(cat "${osd_id_file}").log"
wait_for_file "${log_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}"
function tail_file () {
while true; do
tail --retry -f "${log_file}"
done
}
function truncate_log () {
while true; do
sleep ${TRUNCATE_PERIOD}
if [[ -f ${log_file} ]] ; then
truncate -s "${TRUNCATE_SIZE}" "${log_file}"
fi
done
}
tail_file &
truncate_log &
wait -n

View File

@ -107,6 +107,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
chown -R ceph. ${OSD_PATH};
fi
# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
echo "${OSD_ID}" > /tmp/osd-id
if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then
chown -R ceph. /var/lib/ceph/journal
ceph-osd \

View File

@ -64,6 +64,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
chown -R ceph. ${OSD_PATH};
fi
# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
echo "${OSD_ID}" > /tmp/osd-id
exec /usr/bin/ceph-osd \
--cluster ${CLUSTER} \
${CEPH_OSD_OPTIONS} \

View File

@ -128,6 +128,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
chown -R ceph. ${OSD_PATH};
fi
# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
echo "${OSD_ID}" > /tmp/osd-id
if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then
chown -R ceph. /var/lib/ceph/journal
ceph-osd \

View File

@ -102,6 +102,9 @@ if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then
chown -R ceph. ${OSD_PATH};
fi
# NOTE(gagehugo): Writing the OSD_ID to tmp for logging
echo "${OSD_ID}" > /tmp/osd-id
exec /usr/bin/ceph-osd \
--cluster ${CLUSTER} \
${CEPH_OSD_OPTIONS} \

View File

@ -32,6 +32,8 @@ data:
{{- end }}
osd-start.sh: |
{{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
log-tail.sh: |
{{ tuple "bin/osd/_log-tail.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
osd-directory-ceph-disk.sh: |
{{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
osd-block-ceph-disk.sh: |

View File

@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/}}
{{- define "ceph.osd.daemonset" }}
{{- $daemonset := index . 0 }}
{{- $configMapName := index . 1 }}
@ -236,6 +235,30 @@ spec:
mountPath: /var/log/ceph
readOnly: false
containers:
- name: log-runner
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ dict "envAll" $envAll "application" "osd" "container" "log_runner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }}
env:
- name: DAEMON_NAME
value: "ceph-osd"
- name: TRUNCATE_SIZE
value: {{ .Values.logging.truncate.size | quote }}
- name: TRUNCATE_PERIOD
value: {{ .Values.logging.truncate.period | quote }}
- name: WAIT_FOR_OSD_ID_TIMEOUT
value: {{ .Values.logging.osd_id.timeout | quote }}
command:
- /tmp/log-tail.sh
volumeMounts:
- name: pod-tmp
mountPath: /tmp
- name: ceph-osd-bin
mountPath: /tmp/log-tail.sh
subPath: log-tail.sh
readOnly: true
- name: pod-var-log
mountPath: /var/log/ceph
readOnly: false
- name: ceph-osd-default
{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }}
{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }}

View File

@ -69,6 +69,9 @@ pod:
runAsUser: 0
privileged: true
readOnlyRootFilesystem: true
log_runner:
runAsUser: 0
readOnlyRootFilesystem: true
bootstrap:
pod:
runAsUser: 65534
@ -303,6 +306,13 @@ dependencies:
- endpoint: internal
service: ceph_mon
logging:
truncate:
size: 0
period: 3600
osd_id:
timeout: 300
bootstrap:
enabled: true
script: |