Browse Source

Merge "Implements: etcd backup"

changes/71/634071/12
Zuul 2 months ago
parent
commit
ea6b129606

+ 61
- 0
charts/etcd/templates/bin/_etcdbackup.tpl View File

@@ -0,0 +1,61 @@
1
+#!/bin/sh
2
+{{/*
3
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
4
+
5
+Licensed under the Apache License, Version 2.0 (the "License");
6
+you may not use this file except in compliance with the License.
7
+You may obtain a copy of the License at
8
+
9
+   http://www.apache.org/licenses/LICENSE-2.0
10
+
11
+Unless required by applicable law or agreed to in writing, software
12
+distributed under the License is distributed on an "AS IS" BASIS,
13
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14
+See the License for the specific language governing permissions and
15
+limitations under the License.
16
+*/}}
17
+set -ex
18
+BACKUP_DIR="/var/lib/etcd/backup"
19
+BACKUP_LOG={{ .Values.backup.backup_log_file | quote }}
20
+NUM_TO_KEEP={{ .Values.backup.no_backup_keep | quote }}
21
+SKIP_BACKUP=0
22
+
23
+etcdbackup() {
24
+  etcdctl snapshot save $BACKUP_DIR/etcd-backup-$(date +"%m-%d-%Y-%H-%M-%S").db >> $BACKUP_LOG
25
+  BACKUP_RETURN_CODE=$?
26
+  if [[ $BACKUP_RETURN_CODE != 0 ]]; then
27
+    echo "There was an error backing up the databases. Return code was $BACKUP_RETURN_CODE."
28
+    exit $BACKUP_RETURN_CODE
29
+  fi
30
+  LATEST_BACKUP=`ls -t $BACKUP_DIR | head -1`
31
+  echo "Archiving $LATEST_BACKUP..."
32
+  cd $BACKUP_DIR
33
+  tar -czf $BACKUP_DIR/$LATEST_BACKUP.tar.gz $LATEST_BACKUP
34
+  rm -rf $LATEST_BACKUP
35
+  echo "Clearing earliest backups..."
36
+  NUM_LOCAL_BACKUPS=`ls -ld $BACKUP_DIR | wc -l`
37
+  while [ $NUM_LOCAL_BACKUPS -gt $NUM_TO_KEEP ]
38
+  do
39
+    EARLIEST_BACKUP=`ls -tr $BACKUP_DIR | head -1`
40
+    echo "Deleting $EARLIEST_BACKUP..."
41
+    rm -rf "$BACKUP_DIR/$EARLIEST_BACKUP"
42
+    NUM_LOCAL_BACKUPS=`ls -ld $BACKUP_DIR | wc -l`
43
+  done
44
+}
45
+
46
+if ! [ -x "$(which etcdctl)" ]; then
47
+  echo "ERROR: etcdctl not available, Please use the correct image."
48
+  SKIP_BACKUP=1
49
+fi
50
+
51
+if [ ! -d "$BACKUP_DIR" ]; then
52
+  echo "ERROR: $BACKUP_DIR doesn't exist, Backup will not continue"
53
+  SKIP_BACKUP=1
54
+fi
55
+
56
+if [ $SKIP_BACKUP == '0' ]; then
57
+  etcdbackup
58
+else
59
+  echo "Error: etcd backup failed."
60
+  exit 1
61
+fi

+ 2
- 0
charts/etcd/templates/configmap-bin.yaml View File

@@ -29,4 +29,6 @@ data:
29 29
 {{ tuple "bin/_pre_stop.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
30 30
   readiness: |+
31 31
 {{ tuple "bin/_readiness.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32
+  etcdbackup: |+
33
+{{ tuple "bin/_etcdbackup.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
32 34
 {{- end }}

+ 124
- 0
charts/etcd/templates/cron-job-etcd-backup.yaml View File

@@ -0,0 +1,124 @@
1
+{{/*
2
+Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+{{- if .Values.manifests.cron_etcd_backup }}
17
+{{- $envAll := . }}
18
+{{- $serviceAccountName := "etcd-backup" }}
19
+{{- $applicationName := "etcd-backup" }}
20
+---
21
+apiVersion: rbac.authorization.k8s.io/v1beta1
22
+kind: Role
23
+metadata:
24
+  name: {{ $serviceAccountName }}
25
+rules:
26
+  - apiGroups:
27
+      - ""
28
+    resources:
29
+      - secrets
30
+    verbs:
31
+      - get
32
+      - list
33
+---
34
+apiVersion: v1
35
+kind: ServiceAccount
36
+metadata:
37
+  labels:
38
+    component: etcd-backup
39
+  name: {{ $serviceAccountName }}
40
+  namespace: {{ $envAll.Release.Namespace }}
41
+---
42
+apiVersion: rbac.authorization.k8s.io/v1beta1
43
+kind: RoleBinding
44
+metadata:
45
+  name: {{ $serviceAccountName }}
46
+roleRef:
47
+  apiGroup: rbac.authorization.k8s.io
48
+  kind: Role
49
+  name: {{ $serviceAccountName }}
50
+subjects:
51
+  - kind: ServiceAccount
52
+    name: {{ $serviceAccountName }}
53
+    namespace: {{ $envAll.Release.Namespace }}
54
+---
55
+apiVersion: batch/v1beta1
56
+kind: CronJob
57
+metadata:
58
+  name: etcd-backup
59
+spec:
60
+  schedule: {{ .Values.jobs.etcd_backup.cron | quote }}
61
+  successfulJobsHistoryLimit: {{ .Values.jobs.etcd_backup.history.success }}
62
+  failedJobsHistoryLimit: {{ .Values.jobs.etcd_backup.history.failed }}
63
+  concurrencyPolicy: Forbid
64
+  jobTemplate:
65
+    metadata:
66
+      labels:
67
+{{ tuple $envAll $applicationName "etcd-anchor" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
68
+    spec:
69
+      template:
70
+        spec:
71
+          serviceAccountName: {{ $serviceAccountName }}
72
+          restartPolicy: OnFailure
73
+          nodeSelector:
74
+            {{ .Values.labels.anchor.node_selector_key }}: {{ .Values.labels.anchor.node_selector_value }}
75
+          containers:
76
+            - name: etcd-backup
77
+              image: {{ .Values.images.tags.etcdctl }}
78
+              imagePullPolicy: {{ .Values.images.pull_policy }}
79
+{{ tuple $envAll $envAll.Values.pod.resources.jobs.etcd_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }}
80
+              env:
81
+                - name: ETCDCTL_API
82
+                  value: '3'
83
+                - name: ETCDCTL_DIAL_TIMEOUT
84
+                  value: {{ .Values.backup.etcdctl_dial_timeout }}
85
+                - name: ETCDCTL_ENDPOINTS
86
+                  value: https://{{ .Values.anchor.etcdctl_endpoint }}:{{ .Values.network.service_client.port }}
87
+                - name: ETCDCTL_CACERT
88
+                  value: /etc/etcd/tls/certs/client-ca.pem
89
+                - name: ETCDCTL_CERT
90
+                  value: /etc/etcd/tls/certs/anchor-etcd-client.pem
91
+                - name: ETCDCTL_KEY
92
+                  value: /etc/etcd/tls/keys/anchor-etcd-client-key.pem
93
+                - name: CLIENT_ENDPOINT
94
+                  value: https://$(POD_IP):{{ .Values.network.service_client.target_port }}
95
+                - name: PEER_ENDPOINT
96
+                  value: https://$(POD_IP):{{ .Values.network.service_peer.target_port }}
97
+              command:
98
+                - /tmp/bin/etcdbackup
99
+              volumeMounts:
100
+                - name: {{ .Values.service.name }}-bin
101
+                  mountPath: /tmp/bin
102
+                - name: {{ .Values.service.name }}-certs
103
+                  mountPath: /etc/etcd/tls/certs
104
+                - name: {{ .Values.service.name }}-keys
105
+                  mountPath: /etc/etcd/tls/keys
106
+                - name: etcd-backup
107
+                  mountPath: /var/lib/etcd/backup
108
+          volumes:
109
+          - name: {{ .Values.service.name }}-bin
110
+            configMap:
111
+              name: {{ .Values.service.name }}-bin
112
+              defaultMode: 0555
113
+          - name: {{ .Values.service.name }}-certs
114
+            configMap:
115
+              name: {{ .Values.service.name }}-certs
116
+              defaultMode: 0444
117
+          - name: etcd-backup
118
+            hostPath:
119
+              path: {{ .Values.backup.host_backup_path }}
120
+          - name: {{ .Values.service.name }}-keys
121
+            secret:
122
+              secretName: {{ .Values.service.name }}-keys
123
+              defaultMode: 0444
124
+{{- end }}

+ 27
- 0
charts/etcd/values.yaml View File

@@ -39,6 +39,12 @@ etcd:
39 39
   host_data_path: /var/lib/etcd/example
40 40
   cleanup_data: true
41 41
 
42
+backup:
43
+  host_backup_path: /var/lib/etcd/backup
44
+  backup_log_file: /var/log/etcd-backup.log
45
+  no_backup_keep: 10
46
+  etcdctl_dial_timeout: 15s
47
+
42 48
 network:
43 49
   service_client:
44 50
     name: service_client
@@ -83,6 +89,11 @@ nodes:
83 89
         cert: placeholder
84 90
         key: placeholder
85 91
 
92
+dependencies:
93
+   static:
94
+     etcd_backup:
95
+       jobs:
96
+         - etcd_backup_job
86 97
 pod:
87 98
   mounts:
88 99
     daemonset_anchor:
@@ -123,6 +134,21 @@ pod:
123 134
       requests:
124 135
         memory: "128Mi"
125 136
         cpu: "100m"
137
+    jobs:
138
+      etcdbackup:
139
+        limits:
140
+          memory: "128Mi"
141
+          cpu: "100m"
142
+        requests:
143
+          memory: "128Mi"
144
+          cpu: "100m"
145
+
146
+jobs:
147
+  etcd_backup:
148
+    cron: "0 */12 * * *"
149
+    history:
150
+      success: 3
151
+      failed: 1
126 152
 
127 153
 manifests:
128 154
   configmap_bin: true
@@ -132,3 +158,4 @@ manifests:
132 158
   secret: true
133 159
   service: true
134 160
   test_etcd_health: true
161
+  cron_etcd_backup: true

Loading…
Cancel
Save