b366dc252a
The IPs of ceph monitors cannot (or should not) change. This refactor allows the ceph monitors to act as statefulsets. It also persists their on disk data to nodeDirs, to allow cluster wide restarts (in parallel), where previously this would lose data. This is accompanied by some docker images changes that ensure that auth for the OSDs is restored/reinserted as auth data is somehow not persisted to disk at this time.
106 lines
3.3 KiB
YAML
106 lines
3.3 KiB
YAML
---
|
|
apiVersion: apps/v1beta1
|
|
kind: StatefulSet
|
|
metadata:
|
|
labels:
|
|
app: ceph
|
|
daemon: mon
|
|
name: ceph-mon
|
|
spec:
|
|
serviceName: {{ .Values.service.mon.name | quote }}
|
|
replicas: {{ .Values.replicas.mon }}
|
|
template:
|
|
metadata:
|
|
name: ceph-mon
|
|
labels:
|
|
app: ceph
|
|
daemon: mon
|
|
annotations:
|
|
# alanmeadows: this soft requirement allows single
|
|
# host deployments to spawn several ceph-mon
|
|
# containers
|
|
scheduler.alpha.kubernetes.io/affinity: >
|
|
{
|
|
"podAntiAffinity": {
|
|
"preferredDuringSchedulingIgnoredDuringExecution": [{
|
|
"labelSelector": {
|
|
"matchExpressions": [{
|
|
"key": "daemon",
|
|
"operator": "In",
|
|
"values":["mon"]
|
|
}]
|
|
},
|
|
"topologyKey": "kubernetes.io/hostname",
|
|
"weight": 10
|
|
}]
|
|
}
|
|
}
|
|
spec:
|
|
nodeSelector:
|
|
{{ .Values.labels.node_selector_key }}: {{ .Values.labels.node_selector_value }}
|
|
serviceAccount: default
|
|
volumes:
|
|
- name: ceph-conf
|
|
secret:
|
|
secretName: ceph-conf-combined
|
|
- name: ceph-bootstrap-osd-keyring
|
|
secret:
|
|
secretName: ceph-bootstrap-osd-keyring
|
|
- name: ceph-bootstrap-mds-keyring
|
|
secret:
|
|
secretName: ceph-bootstrap-mds-keyring
|
|
- name: ceph-bootstrap-rgw-keyring
|
|
secret:
|
|
secretName: ceph-bootstrap-rgw-keyring
|
|
- name: ceph-monfs
|
|
hostPath:
|
|
path: {{ .Values.storage.mon_directory }}
|
|
containers:
|
|
- name: ceph-mon
|
|
image: {{ .Values.images.daemon }}
|
|
imagePullPolicy: Always
|
|
lifecycle:
|
|
preStop:
|
|
exec:
|
|
# remove the mon on Pod stop.
|
|
command:
|
|
- "/remove-mon.sh"
|
|
ports:
|
|
- containerPort: 6789
|
|
env:
|
|
- name: CEPH_DAEMON
|
|
value: MON
|
|
- name: KV_TYPE
|
|
value: k8s
|
|
- name: NETWORK_AUTO_DETECT
|
|
value: "1"
|
|
- name: CLUSTER
|
|
value: ceph
|
|
volumeMounts:
|
|
- name: ceph-conf
|
|
mountPath: /etc/ceph
|
|
- name: ceph-bootstrap-osd-keyring
|
|
mountPath: /var/lib/ceph/bootstrap-osd
|
|
- name: ceph-bootstrap-mds-keyring
|
|
mountPath: /var/lib/ceph/bootstrap-mds
|
|
- name: ceph-bootstrap-rgw-keyring
|
|
mountPath: /var/lib/ceph/bootstrap-rgw
|
|
- name: ceph-monfs
|
|
mountPath: /var/lib/ceph/mon
|
|
livenessProbe:
|
|
tcpSocket:
|
|
port: 6789
|
|
initialDelaySeconds: 60
|
|
timeoutSeconds: 5
|
|
readinessProbe:
|
|
tcpSocket:
|
|
port: 6789
|
|
timeoutSeconds: 5
|
|
resources:
|
|
requests:
|
|
memory: {{ .Values.resources.mon.requests.memory | quote }}
|
|
cpu: {{ .Values.resources.mon.requests.cpu | quote }}
|
|
limits:
|
|
memory: {{ .Values.resources.mon.limits.memory | quote }}
|
|
cpu: {{ .Values.resources.mon.limits.cpu | quote }}
|