Browse Source

Add ability to control pam_limits via new module 'limits'

1) 'Values' configures limit settings to be persisted.
2) Previous DivingBell controlled limits those were set
but now are gone are cleared.
3) Previous values of newly set limits are backed up
to /var/divingbell/limits
4) New limit is applied via adding a separate conf file
to /etc/security/limits.d
5) The Doc is updated with appropriate details.
6) Dev env with Vagrant
7) Increase number of expected DaemonSets in 020-test
8) Demo: https://asciinema.org/a/209619

Change-Id: I5efb39c498c2b666b4ba97271b59757f4a0c1ca7
skovaleff 6 months ago
parent
commit
7ed8c29f99

+ 29
- 0
Vagrantfile View File

@@ -0,0 +1,29 @@
1
+# -*- mode: ruby -*-
2
+# vi: set ft=ruby :
3
+
4
+Vagrant.configure("2") do |config|
5
+    config.vm.box = "generic/ubuntu1604"
6
+
7
+    [:virtualbox, :parallels, :libvirt, :hyperv].each do |provider|
8
+        config.vm.provider provider do |vplh, override|
9
+            vplh.cpus = 4
10
+            vplh.memory = 4096
11
+        end
12
+    end
13
+
14
+    config.vm.synced_folder "./", "/root/deploy/airship-divingbell"
15
+
16
+    config.vm.define "dbtest" do |node|
17
+    node.vm.hostname = "dbtest"
18
+    node.vm.provision :shell, inline: <<-SHELL
19
+      #mkdir /root/deploy
20
+      #git clone git://git.openstack.org/openstack/airship-divingbell /root/deploy/airship-divingbell
21
+      git clone https://git.openstack.org/openstack/openstack-helm-infra /root/deploy/openstack-helm-infra
22
+      cd /root/deploy/openstack-helm-infra
23
+      ./tools/gate/devel/start.sh full
24
+      cd /root/deploy/airship-divingbell/
25
+      ./tools/gate/scripts/010-build-charts.sh
26
+      ./tools/gate/scripts/020-test-divingbell.sh
27
+    SHELL
28
+    end
29
+end

+ 109
- 0
divingbell/templates/bin/_limits.sh.tpl View File

@@ -0,0 +1,109 @@
1
+#!/bin/bash
2
+
3
+{{/*
4
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#     http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+*/}}
18
+
19
+set -e
20
+
21
+cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/limits_host.sh
22
+{{ include "divingbell.shcommon" . }}
23
+
24
+fname_prefix='60-divingbell-'
25
+persist_path='/etc/security/limits.d'
26
+
27
+if [ ! -d "${persist_path}" ]; then
28
+  mkdir -p "${persist_path}"
29
+fi
30
+
31
+write_test "${persist_path}"
32
+
33
+add_limits_param(){
34
+  local limit="${1}"
35
+  die_if_null "${limit}" ", limit not supplied to function"
36
+  local domain="${2}"
37
+  die_if_null "${domain}" ", domain not supplied to function"
38
+  local type="${3}"
39
+  die_if_null "${type}" ", type not supplied to function"
40
+  local item="${4}"
41
+  die_if_null "${item}" ", item not supplied to function"
42
+  local value="${5}"
43
+  die_if_null "${value}" ", value not supplied to function"
44
+
45
+  file_content="${domain} ${type} ${item} ${value}"
46
+  file_name="${fname_prefix}${limit}.conf"
47
+  file_path="${persist_path}/${file_name}"
48
+
49
+  # Persist the new setting
50
+  if [ -f "${file_path}" ] &&
51
+     [ "$(cat ${file_path})" != "${file_content}" ] ||
52
+     [ ! -f "${file_path}" ]
53
+  then
54
+    echo "${file_content}" > "${file_path}"
55
+    log.INFO "Limits setting applied: ${file_content}"
56
+  else
57
+    log.INFO "No changes made to limits param: ${limit}"
58
+  fi
59
+
60
+  curr_settings="${curr_settings}${file_name}"$'\n'
61
+}
62
+
63
+{{- range $index, $limit := .Values.conf.limits }}
64
+add_limits_param {{ $index | squote }} {{ $limit.domain | squote }} {{ $limit.type | squote }}\
65
+                 {{ $limit.item | squote }} {{ $limit.value | squote }}
66
+{{- end }}
67
+
68
+# Revert any previously applied limits settings which are now absent
69
+prev_files="$(find "${persist_path}" -type f)"
70
+if [ -n "${prev_files}" ]; then
71
+  basename -a ${prev_files} | sort > /tmp/prev_settings
72
+  echo "${curr_settings}" | sort > /tmp/curr_settings
73
+  revert_list="$(comm -23 /tmp/prev_settings /tmp/curr_settings)"
74
+  IFS=$'\n'
75
+  for orig_limits_setting in ${revert_list}; do
76
+    rm "${persist_path}/${orig_limits_setting}"
77
+    log.INFO "Reverted limits setting: ${persist_path}/${orig_limits_setting}"
78
+  done
79
+fi
80
+
81
+# Print limit settings
82
+# su is a simple and fast way to see applied changes
83
+# bash, bash -c, sudo, setsid didn't work out for me.
84
+su -c "prlimit --noheadings --output RESOURCE,SOFT,HARD"
85
+# The setting is persisted for a new process.
86
+# It's deliberate design decision to let current process be intact.
87
+# For this test it's just test bash process.
88
+# For production case it's limits_host.sh run by DivingBell pod which is in sleep mode.
89
+
90
+if [ -n "${curr_settings}" ]; then
91
+  log.INFO 'All limits configuration successfully validated on this node.'
92
+else
93
+  log.WARN 'No limits overrides defined for this node.'
94
+fi
95
+
96
+exit 0
97
+EOF
98
+
99
+chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/limits_host.sh
100
+chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/limits_host.sh
101
+
102
+sleep 1
103
+echo 'INFO Putting the daemon to sleep.'
104
+
105
+while [ 1 ]; do
106
+  sleep 300
107
+done
108
+
109
+exit 0

+ 1
- 1
divingbell/templates/bin/_sysctl.sh.tpl View File

@@ -120,7 +120,7 @@ fi
120 120
 if [ -n "${curr_settings}" ]; then
121 121
   log.INFO 'All sysctl configuration successfully validated on this node.'
122 122
 else
123
-  log.WARN 'No syctl overrides defined for this node.'
123
+  log.WARN 'No sysctl overrides defined for this node.'
124 124
 fi
125 125
 
126 126
 exit 0

+ 30
- 0
divingbell/templates/configmap-limits.yaml View File

@@ -0,0 +1,30 @@
1
+{{/*
2
+Copyright 2018 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.configmap.limits" }}
18
+{{- $configMapName := index . 0 }}
19
+{{- $envAll := index . 1 }}
20
+{{- with $envAll }}
21
+---
22
+apiVersion: v1
23
+kind: ConfigMap
24
+metadata:
25
+  name: {{ $configMapName }}
26
+data:
27
+  limits: |+
28
+{{ tuple "bin/_limits.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }}
29
+{{- end }}
30
+{{- end }}

+ 71
- 0
divingbell/templates/daemonset-limits.yaml View File

@@ -0,0 +1,71 @@
1
+{{/*
2
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.daemonset.limits" }}
18
+  {{- $daemonset := index . 0 }}
19
+  {{- $configMapName := index . 1 }}
20
+  {{- $envAll := index . 2 }}
21
+  {{- with $envAll }}
22
+---
23
+apiVersion: extensions/v1beta1
24
+kind: DaemonSet
25
+metadata:
26
+  name: {{ $daemonset }}
27
+  annotations:
28
+    {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }}
29
+spec:
30
+{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
31
+  template:
32
+    metadata:
33
+      labels:
34
+{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
35
+    spec:
36
+      hostNetwork: true
37
+      hostPID: true
38
+      hostIPC: true
39
+      containers:
40
+      - name: {{ $daemonset }}
41
+        image: {{ .Values.images.divingbell }}
42
+        imagePullPolicy: {{ .Values.images.pull_policy }}
43
+{{ tuple $envAll $envAll.Values.pod.resources.limits | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
44
+        command:
45
+        - /tmp/{{ $daemonset }}.sh
46
+        volumeMounts:
47
+        - name: rootfs-{{ $daemonset }}
48
+          mountPath: {{ .Values.conf.chroot_mnt_path }}
49
+        - name: {{ $configMapName }}
50
+          mountPath: /tmp/{{ $daemonset }}.sh
51
+          subPath: {{ $daemonset }}
52
+          readOnly: true
53
+        securityContext:
54
+          privileged: true
55
+      volumes:
56
+      - name: rootfs-{{ $daemonset }}
57
+        hostPath:
58
+          path: /
59
+      - name: {{ $configMapName }}
60
+        configMap:
61
+          name: {{ $configMapName }}
62
+          defaultMode: 0555
63
+  {{- end }}
64
+{{- end }}
65
+{{- if .Values.manifests.daemonset_limits }}
66
+{{- $daemonset := "limits" }}
67
+{{- $configMapName := "divingbell-limits" }}
68
+{{- $daemonset_yaml := list $daemonset $configMapName . | include "divingbell.daemonset.limits" | toString | fromYaml }}
69
+{{- $configmap_include := "divingbell.configmap.limits" }}
70
+{{- list $daemonset $daemonset_yaml $configmap_include $configMapName . | include "helm-toolkit.utils.daemonset_overrides" }}
71
+{{- end }}

+ 27
- 0
divingbell/values.yaml View File

@@ -25,6 +25,21 @@ conf:
25 25
   chroot_mnt_path: '/mnt'
26 26
   log_colors: False
27 27
 
28
+##  data.values.conf.sysctl
29
+#  sysctl:
30
+#    fs.suid_dumpable: '0'
31
+##  data.values.conf.limits
32
+#  limits:
33
+#    nofile:
34
+#      domain: 'root'
35
+#      type: 'soft'
36
+#      item: 'nofile'
37
+#      value: '101'
38
+#    core_dump:
39
+#      domain: '0:'
40
+#      type: 'hard'
41
+#      item: 'core'
42
+#      value: 0
28 43
 pod:
29 44
   lifecycle:
30 45
     upgrades:
@@ -46,6 +61,10 @@ pod:
46 61
           enabled: true
47 62
           min_ready_seconds: 0
48 63
           max_unavailable: 100%
64
+        limits:
65
+          enabled: true
66
+          min_ready_seconds: 0
67
+          max_unavailable: 100%
49 68
   resources:
50 69
     enabled: false
51 70
     ethtool:
@@ -76,9 +95,17 @@ pod:
76 95
       requests:
77 96
         memory: "128Mi"
78 97
         cpu: "100m"
98
+    limits:
99
+      limits:
100
+        memory: "128Mi"
101
+        cpu: "100m"
102
+      requests:
103
+        memory: "128Mi"
104
+        cpu: "100m"
79 105
 
80 106
 manifests:
81 107
   daemonset_ethtool: true
82 108
   daemonset_mounts: true
83 109
   daemonset_uamlite: true
84 110
   daemonset_sysctl: true
111
+  daemonset_limits: true

+ 35
- 1
doc/source/index.rst View File

@@ -49,7 +49,8 @@ In order to keep configuration as isolated as possible from other systems that
49 49
 manage common files like /etc/fstab and /etc/sysctl.conf, Divingbell daemonsets
50 50
 manage all of their configuration in separate files (e.g. by writing unique
51 51
 files to /etc/sysctl.d or defining unique Systemd units) to avoid potential
52
-conflicts.
52
+conflicts. Another example is limit management, Divingbell daemonset writes
53
+separate files to /etc/security/limits.d.
53 54
 
54 55
 To maximize robustness and utility, the daemonsets in this chart are made to be
55 56
 idempotent. In addition, they are designed to implicitly restore the original
@@ -78,6 +79,27 @@ Used to manage host level sysctl tunables. Ex::
78 79
         net/ipv4/ip_forward: 1
79 80
         net/ipv6/conf/all/forwarding: 1
80 81
 
82
+limits
83
+^^^^^^
84
+
85
+Used to manage host level limits. Ex::
86
+
87
+  conf:
88
+    limits:
89
+      nofile:
90
+        domain: 'root'
91
+        type: 'soft'
92
+        item: 'nofile'
93
+        value: '101'
94
+      core_dump:
95
+        domain: '0:'
96
+        type: 'hard'
97
+        item: 'core'
98
+        value: 0
99
+
100
+Previous values of newly set limits are backed up to /var/divingbell/limits
101
+
102
+
81 103
 mounts
82 104
 ^^^^^^
83 105
 
@@ -256,6 +278,18 @@ Caveats:
256 278
    "another_label" would take precedence and be applied to nodes that
257 279
    contained both of the defined labels.
258 280
 
281
+Dev Environment with Vagrant
282
+----------------------------
283
+The point of Dev env to prepare working environment for development.
284
+
285
+Vagrantfile allows to run on working copy with modifications
286
+e.g. to 020-test script. The approach is to setup Gate test
287
+but do not delete the pods and other stuff. You have:
288
+
289
+1. test run of previous tests and their results
290
+2. your changes from working tree are applied smoothly
291
+3. your not committed test runs in prepared env
292
+
259 293
 Recorded Demo
260 294
 -------------
261 295
 

+ 38
- 4
tools/gate/scripts/020-test-divingbell.sh View File

@@ -57,7 +57,7 @@ for line in ${nic_info}; do
57 57
   fi
58 58
   if [ "${physical_nic}" = 'true' ] && [[ ${line} = *'logical name'* ]]; then
59 59
     DEVICE="$(echo "${line}" | cut -d':' -f2 | tr -d '[:space:]')"
60
-    echo "Found deivce: '${DEVICE}' to use for ethtool testing"
60
+    echo "Found device: '${DEVICE}' to use for ethtool testing"
61 61
     break
62 62
   fi
63 63
 done
@@ -99,6 +99,7 @@ _teardown_systemd(){
99 99
 clean_persistent_files(){
100 100
   sudo rm -r /var/${NAME} >& /dev/null || true
101 101
   sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true
102
+  sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true
102 103
   _teardown_systemd ${MOUNTS_PATH1} mount
103 104
   _teardown_systemd ${MOUNTS_PATH2} mount
104 105
   _teardown_systemd ${MOUNTS_PATH3} mount
@@ -312,6 +313,38 @@ test_sysctl(){
312 313
   echo '[SUCCESS] sysctl test5 passed successfully' >> "${TEST_RESULTS}"
313 314
 }
314 315
 
316
+_test_limits_value(){
317
+  local limit=${1}
318
+  local domain=${2}
319
+  local type=${3}
320
+  local item=${4}
321
+  local value=${5}
322
+  test "$(cat /etc/security/limits.d/60-${NAME}-${limit}.conf)" = \
323
+    "$domain $type $item $value"
324
+}
325
+
326
+test_limits(){
327
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}.yaml
328
+  echo "conf:
329
+  limits:
330
+    limit1:
331
+      domain: root
332
+      type: hard
333
+      item: core
334
+      value: 0
335
+    limit2:
336
+      domain: '0:'
337
+      type: soft
338
+      item: nofile
339
+      value: 101" > "${overrides_yaml}"
340
+  echo $(cat ${overrides_yaml})
341
+  install_base "--values=${overrides_yaml}"
342
+  get_container_status limits
343
+  _test_limits_value limit1 root hard core 0
344
+  _test_limits_value limit2 '0:' soft nofile 101
345
+  echo "[SUCCESS] test range loop for limits passed successfully" >> "${TEST_RESULTS}"
346
+}
347
+
315 348
 _test_if_mounted_positive(){
316 349
   mountpoint "${1}" || (echo "Expect ${1} to be mounted, but was not"; exit 1)
317 350
   df -h | grep "${1}" | grep "${2}" ||
@@ -815,9 +848,9 @@ test_overrides(){
815 848
 
816 849
   # Compare against expected number of generated daemonsets
817 850
   daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
818
-  if [ "${daemonset_count}" != "12" ]; then
851
+  if [ "${daemonset_count}" != "13" ]; then
819 852
     echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
820
-    echo "Expected 12 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
853
+    echo "Expected 13 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
821 854
     exit 1
822 855
   else
823 856
     echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@@ -995,13 +1028,14 @@ init_default_state
995 1028
 # run tests
996 1029
 install_base
997 1030
 test_sysctl
1031
+test_limits
998 1032
 test_mounts
999 1033
 test_ethtool
1000 1034
 test_uamlite
1001 1035
 purge_containers
1002 1036
 test_overrides
1003 1037
 
1004
-# retore initial state
1038
+# restore initial state
1005 1039
 init_default_state
1006 1040
 
1007 1041
 echo "All tests pass for ${NAME}"

Loading…
Cancel
Save