Browse Source

Add new apparmor daemonset

Implemented daemonset that will manage host apparmor profiles.
Tests and documentation added.

demo: https://asciinema.org/a/uQjlWgC4bjI3WkfontmThf8t0

Co-Authored-By: Vladyslav Drok <vdrok@mirantis.com>
Change-Id: I13f7357c15b5c4386a61bba50f097eb434d7f211
Nikita Koshikov 7 months ago
parent
commit
606cf35bda

+ 137
- 0
divingbell/templates/bin/_apparmor.sh.tpl View File

@@ -0,0 +1,137 @@
1
+#!/bin/bash
2
+
3
+{{/*
4
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#     http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+*/}}
18
+
19
+set -e
20
+
21
+cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh
22
+{{ include "divingbell.shcommon" . }}
23
+
24
+load_flags="-r -W"
25
+{{- if hasKey .Values.conf "apparmor" }}
26
+{{- if hasKey .Values.conf.apparmor "complain_mode" }}
27
+{{- if .Values.conf.apparmor.complain_mode }}
28
+load_flags="$load_flags -C"
29
+{{- end }}
30
+{{- end }}
31
+{{- end }}
32
+load_cmd="apparmor_parser $load_flags"
33
+unload_cmd='apparmor_parser -R'
34
+defaults_path='/var/divingbell/apparmor'
35
+persist_path='/etc/apparmor.d'
36
+declare -A CURRENT_FILENAMES
37
+declare -A SAVED_STATE_FILENAMES
38
+
39
+if [ ! -d "${defaults_path}" ]; then
40
+  mkdir -p "${defaults_path}"
41
+fi
42
+
43
+write_test "${defaults_path}"
44
+write_test "${persist_path}"
45
+
46
+save_apparmor_profile(){
47
+  local filename="$1"
48
+  local data="$2"
49
+  CURRENT_FILENAMES["$filename"]=''
50
+
51
+  #Check if host already had the same filename
52
+  if [ ${SAVED_STATE_FILENAMES["$filename"]+_} ]; then
53
+    unset SAVED_STATE_FILENAMES["$filename"]
54
+  fi
55
+
56
+  echo -ne "${data}" > ${defaults_path}/${filename}
57
+  if [ ! -L ${persist_path}/${filename} ]; then
58
+    ln -s ${defaults_path}/${filename}  ${persist_path}/${filename}
59
+  fi
60
+}
61
+
62
+#######################################
63
+#Stage 1
64
+#Collect data
65
+#######################################
66
+
67
+#Search for any saved apparmor profiles
68
+pushd $defaults_path
69
+count=$(find . -type f | wc -l)
70
+
71
+#Check if directory is non-empty
72
+if [ $count -gt 0 ]; then
73
+  for f in $(find . -type f|xargs -n1 basename); do
74
+    SAVED_STATE_FILENAMES[$f]=''
75
+  done
76
+fi
77
+
78
+#######################################
79
+#Stage 2
80
+#Save new apparmor profiles
81
+#######################################
82
+
83
+{{- if hasKey .Values.conf "apparmor" }}
84
+{{- if hasKey .Values.conf.apparmor "profiles" }}
85
+{{- range $filename, $value := .Values.conf.apparmor.profiles }}
86
+save_apparmor_profile {{ $filename | squote }} {{ $value | squote }}
87
+{{- end }}
88
+{{- end }}
89
+{{- end }}
90
+
91
+
92
+#######################################
93
+#Stage 3
94
+#Clean stale apparmor profiles
95
+#######################################
96
+
97
+#If hash is not empty - there are old filenames that need to be handled
98
+if [ ${#SAVED_STATE_FILENAMES[@]} -gt 0 ]; then
99
+  for filename in ${!SAVED_STATE_FILENAMES[@]}; do
100
+    #Unload any previously applied apparmor profiles which are now absent
101
+    $unload_cmd ${defaults_path}/${filename} || die "Problem unloading profile ${defaults_path}/${filename}"
102
+    if [ -L ${persist_path}/${filename} ]; then
103
+      unlink ${persist_path}/${filename}
104
+    fi
105
+    rm -f ${defaults_path}/${filename}
106
+    # log/append the stale profiles that require eventual reboot
107
+    echo "apparmor: stale profile ${defaults_path}/${filename}" >> /var/run/reboot-required.pkgs
108
+    unset SAVED_STATE_FILENAMES["$filename"]
109
+  done
110
+  # mark node as needing eventual reboot
111
+  echo '*** System restart required ***' > /var/run/reboot-required
112
+fi
113
+
114
+#######################################
115
+#Stage 4
116
+#Install/update new apparmor profiles
117
+#Save new apparmor profiles
118
+#######################################
119
+
120
+for filename in ${!CURRENT_FILENAMES[@]}; do
121
+  $load_cmd ${persist_path}/${filename} || die "Problem loading ${persist_path}/${filename}"
122
+done
123
+
124
+exit 0
125
+EOF
126
+
127
+chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apparmor_host.sh
128
+chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apparmor_host.sh
129
+
130
+sleep 1
131
+echo 'INFO Putting the daemon to sleep.'
132
+
133
+while [ 1 ]; do
134
+  sleep 300
135
+done
136
+
137
+exit 0

+ 69
- 0
divingbell/templates/daemonset-apparmor.yaml View File

@@ -0,0 +1,69 @@
1
+{{/*
2
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.daemonset.apparmor" }}
18
+  {{- $daemonset := index . 0 }}
19
+  {{- $secretName := index . 1 }}
20
+  {{- $envAll := index . 2 }}
21
+  {{- with $envAll }}
22
+---
23
+apiVersion: extensions/v1beta1
24
+kind: DaemonSet
25
+metadata:
26
+  name: {{ $daemonset }}
27
+spec:
28
+{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
29
+  template:
30
+    metadata:
31
+      labels:
32
+{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
33
+    spec:
34
+      hostNetwork: true
35
+      hostPID: true
36
+      hostIPC: true
37
+      containers:
38
+      - name: {{ $daemonset }}
39
+        image: {{ .Values.images.divingbell }}
40
+        imagePullPolicy: {{ .Values.images.pull_policy }}
41
+{{ tuple $envAll $envAll.Values.pod.resources.apparmor | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
42
+        command:
43
+        - /tmp/{{ $daemonset }}.sh
44
+        volumeMounts:
45
+        - name: rootfs-{{ $daemonset }}
46
+          mountPath: {{ .Values.conf.chroot_mnt_path }}
47
+        - name: {{ $secretName }}
48
+          mountPath: /tmp/{{ $daemonset }}.sh
49
+          subPath: {{ $daemonset }}
50
+          readOnly: true
51
+        securityContext:
52
+          privileged: true
53
+      volumes:
54
+      - name: rootfs-{{ $daemonset }}
55
+        hostPath:
56
+          path: /
57
+      - name: {{ $secretName }}
58
+        secret:
59
+          secretName: {{ $secretName }}
60
+          defaultMode: 0555
61
+  {{- end }}
62
+{{- end }}
63
+{{- if .Values.manifests.daemonset_apparmor }}
64
+{{- $daemonset := "apparmor" }}
65
+{{- $secretName := "divingbell-apparmor" }}
66
+{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apparmor" | toString | fromYaml }}
67
+{{- $secret_include := "divingbell.secret.apparmor" }}
68
+{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }}
69
+{{- end }}

+ 29
- 0
divingbell/templates/secret-apparmor.yaml View File

@@ -0,0 +1,29 @@
1
+{{/*
2
+Copyright 2017 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.secret.apparmor" }}
18
+{{- $secretName := index . 0 }}
19
+{{- $envAll := index . 1 }}
20
+{{- with $envAll }}
21
+---
22
+apiVersion: v1
23
+kind: Secret
24
+metadata:
25
+  name: {{ $secretName }}
26
+data:
27
+  apparmor: {{ tuple "bin/_apparmor.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc }}
28
+{{- end }}
29
+{{- end }}

+ 8
- 0
divingbell/values.yaml View File

@@ -127,6 +127,13 @@ pod:
127 127
           max_unavailable: 100%
128 128
   resources:
129 129
     enabled: false
130
+    apparmor:
131
+      limits:
132
+        memory: "128Mi"
133
+        cpu: "100m"
134
+      requests:
135
+        memory: "128Mi"
136
+        cpu: "100m"
130 137
     ethtool:
131 138
       limits:
132 139
         memory: "128Mi"
@@ -193,3 +200,4 @@ manifests:
193 200
   daemonset_apt: true
194 201
   daemonset_perm: true
195 202
   daemonset_exec: true
203
+  daemonset_apparmor: true

+ 76
- 0
doc/source/index.rst View File

@@ -250,9 +250,85 @@ The following set of options are not yet implemeneted::
250 250
     failing script should be retried. Failed exec count does not persist
251 251
     through pod/node restart. Default value is ``infinite``.
252 252
 
253
+apparmor
254
+^^^^^^^^
255
+
256
+Used to manage host level apparmor profiles/rules, Ex::
257
+
258
+    conf:
259
+      apparmor:
260
+        complain_mode: "true"
261
+        profiles:
262
+          profile-1: |
263
+            #include <tunables/global>
264
+              /usr/sbin/profile-1 {
265
+                #include <abstractions/apache2-common>
266
+                #include <abstractions/base>
267
+                #include <abstractions/nis>
268
+
269
+                capability dac_override,
270
+                capability dac_read_search,
271
+                capability net_bind_service,
272
+                capability setgid,
273
+                capability setuid,
274
+
275
+                /data/www/safe/* r,
276
+                deny /data/www/unsafe/* r,
277
+              }
278
+          profile-2: |
279
+            #include <tunables/global>
280
+              /usr/sbin/profile-2 {
281
+                #include <abstractions/apache2-common>
282
+                #include <abstractions/base>
283
+                #include <abstractions/nis>
284
+
285
+                capability dac_override,
286
+                capability dac_read_search,
287
+                capability net_bind_service,
288
+                capability setgid,
289
+                capability setuid,
290
+
291
+                /data/www/safe/* r,
292
+                deny /data/www/unsafe/* r,
293
+              }
294
+
253 295
 Operations
254 296
 ----------
255 297
 
298
+Setting apparmor profiles
299
+^^^^^^^^^^^^^^^^^^^^^^^^^
300
+
301
+The way apparmor loading/unloading implemented is through saving
302
+settings to a file and than running ``apparmor_parser`` command.
303
+The daemonset supports both enforcement and complain mode,
304
+enforcement being the default. To request complain mode for the
305
+profiles, add ``complain_mode: "true"`` nested under apparmor entry.
306
+
307
+It's easy to mess up host with rules, if profile names would
308
+distinguish from file content. Ex::
309
+
310
+    conf:
311
+      apparmor:
312
+        profiles:
313
+          profile-1: |
314
+            #include <tunables/global>
315
+              /usr/sbin/profile-1 {
316
+                #include <abstractions/base>
317
+                capability setgid,
318
+              }
319
+          profile-2: |
320
+            #include <tunables/global>
321
+              /usr/sbin/profile-1 {
322
+                #include <abstractions/base>
323
+                capability net_bind_service,
324
+              }
325
+
326
+Even when profiles are different (profile-1 vs profile-2) - filenames
327
+are the same (profile-1), that means that only one set of rules in
328
+memory would be active for particular profile (either setgid or
329
+net_bind_service), but not both. Such problems are hard to debug, so
330
+caution needed while setting configs up.
331
+
256 332
 Setting user passwords
257 333
 ^^^^^^^^^^^^^^^^^^^^^^
258 334
 

+ 168
- 2
tools/gate/scripts/020-test-divingbell.sh View File

@@ -56,7 +56,10 @@ APT_PACKAGE4=less
56 56
 APT_PACKAGE5=python-setuptools
57 57
 APT_PACKAGE6=telnetd
58 58
 EXEC_DIR=/var/${NAME}/exec
59
+# this used in test_overrides to check amount of daemonsets defined
60
+EXPECTED_NUMBER_OF_DAEMONSETS=17
59 61
 type lshw || apt -y install lshw
62
+type apparmor_parser || apt -y install apparmor
60 63
 nic_info="$(lshw -class network)"
61 64
 physical_nic=''
62 65
 IFS=$'\n'
@@ -109,6 +112,7 @@ clean_persistent_files(){
109 112
   sudo rm -r /var/${NAME} >& /dev/null || true
110 113
   sudo rm -r /etc/sysctl.d/60-${NAME}-* >& /dev/null || true
111 114
   sudo rm -r /etc/security/limits.d/60-${NAME}-* >& /dev/null || true
115
+  sudo rm -r /etc/apparmor.d/${NAME}-* >& /dev/null || true
112 116
   _teardown_systemd ${MOUNTS_PATH1} mount
113 117
   _teardown_systemd ${MOUNTS_PATH2} mount
114 118
   _teardown_systemd ${MOUNTS_PATH3} mount
@@ -1392,9 +1396,9 @@ test_overrides(){
1392 1396
 
1393 1397
   # Compare against expected number of generated daemonsets
1394 1398
   daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
1395
-  if [ "${daemonset_count}" != "16" ]; then
1399
+  if [ "${daemonset_count}" != "${EXPECTED_NUMBER_OF_DAEMONSETS}" ]; then
1396 1400
     echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
1397
-    echo "Expected 15 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
1401
+    echo "Expected ${EXPECTED_NUMBER_OF_DAEMONSETS} daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
1398 1402
     exit 1
1399 1403
   else
1400 1404
     echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@@ -1566,6 +1570,167 @@ test_overrides(){
1566 1570
 
1567 1571
 }
1568 1572
 
1573
+_test_apparmor_profile_added(){
1574
+  local profile_file=$1
1575
+  local profile_name=$2
1576
+  local defaults_path='/var/divingbell/apparmor'
1577
+  local persist_path='/etc/apparmor.d'
1578
+
1579
+  if [ ! -f "${defaults_path}/${profile_file}" ]; then
1580
+    return 1
1581
+  fi
1582
+  if [ ! -L "${persist_path}/${profile_file}" ]; then
1583
+    return 1
1584
+  fi
1585
+
1586
+  profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
1587
+
1588
+  if [ -z "$profile_loaded" ]; then
1589
+    return 1
1590
+  fi
1591
+  return 0
1592
+}
1593
+
1594
+_test_apparmor_profile_removed(){
1595
+  local profile_file=$1
1596
+  local profile_name=$2
1597
+  local defaults_path='/var/divingbell/apparmor'
1598
+  local persist_path='/etc/apparmor.d'
1599
+
1600
+  if [ -f "${defaults_path}/${profile_file}" ]; then
1601
+    return 1
1602
+  fi
1603
+  if [ -L "${persist_path}/${profile_file}" ]; then
1604
+    return 1
1605
+  fi
1606
+
1607
+  profile_loaded=$(grep $profile_name /sys/kernel/security/apparmor/profiles || : )
1608
+
1609
+  if [ ! -z "$profile_loaded" ]; then
1610
+    return 1
1611
+  fi
1612
+
1613
+  reboot_message_present=$(grep $profile_file /var/run/reboot-required.pkgs || : )
1614
+
1615
+  if [ -z "$reboot_message_present" ]; then
1616
+    return 1
1617
+  fi
1618
+
1619
+  return 0
1620
+}
1621
+
1622
+test_apparmor(){
1623
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-apparmor.yaml
1624
+
1625
+  #Test1 - check new profile added and loaded
1626
+  echo "conf:
1627
+  apparmor:
1628
+    profiles:
1629
+      divingbell-profile-1: |
1630
+        #include <tunables/global>
1631
+          /usr/sbin/profile-1 {
1632
+            #include <abstractions/apache2-common>
1633
+            #include <abstractions/base>
1634
+            #include <abstractions/nis>
1635
+
1636
+            capability dac_override,
1637
+            capability dac_read_search,
1638
+            capability net_bind_service,
1639
+            capability setgid,
1640
+            capability setuid,
1641
+
1642
+            /data/www/safe/* r,
1643
+            deny /data/www/unsafe/* r,
1644
+          }" > "${overrides_yaml}"
1645
+  install_base "--values=${overrides_yaml}"
1646
+  get_container_status apparmor
1647
+  _test_apparmor_profile_added divingbell-profile-1 profile-1
1648
+  echo '[SUCCESS] apparmor test1 passed successfully' >> "${TEST_RESULTS}"
1649
+
1650
+  #Test2 - check new profile added and loaded, profile-1 still exist
1651
+  echo "conf:
1652
+  apparmor:
1653
+    profiles:
1654
+      divingbell-profile-1: |
1655
+        #include <tunables/global>
1656
+          /usr/sbin/profile-1 {
1657
+            #include <abstractions/apache2-common>
1658
+            #include <abstractions/base>
1659
+            #include <abstractions/nis>
1660
+
1661
+            capability dac_override,
1662
+            capability dac_read_search,
1663
+            capability net_bind_service,
1664
+            capability setgid,
1665
+            capability setuid,
1666
+
1667
+            /data/www/safe/* r,
1668
+            deny /data/www/unsafe/* r,
1669
+          }
1670
+      divingbell-profile-2: |
1671
+        #include <tunables/global>
1672
+          /usr/sbin/profile-2 {
1673
+            #include <abstractions/apache2-common>
1674
+            #include <abstractions/base>
1675
+            #include <abstractions/nis>
1676
+
1677
+            capability dac_override,
1678
+            capability dac_read_search,
1679
+            capability net_bind_service,
1680
+            capability setgid,
1681
+            capability setuid,
1682
+
1683
+            /data/www/safe/* r,
1684
+            deny /data/www/unsafe/* r,
1685
+          }" > "${overrides_yaml}"
1686
+  install_base "--values=${overrides_yaml}"
1687
+  get_container_status apparmor
1688
+  _test_apparmor_profile_added divingbell-profile-1 profile-1
1689
+  _test_apparmor_profile_added divingbell-profile-2 profile-2
1690
+  echo '[SUCCESS] apparmor test2 passed successfully' >> "${TEST_RESULTS}"
1691
+
1692
+  #Test3 - check profile-2 removed, profile-1 still exist
1693
+  echo "conf:
1694
+  apparmor:
1695
+    complain_mode: true
1696
+    profiles:
1697
+      divingbell-profile-1: |
1698
+        #include <tunables/global>
1699
+          /usr/sbin/profile-1 {
1700
+            #include <abstractions/apache2-common>
1701
+            #include <abstractions/base>
1702
+            #include <abstractions/nis>
1703
+
1704
+            capability dac_override,
1705
+            capability dac_read_search,
1706
+            capability net_bind_service,
1707
+            capability setgid,
1708
+            capability setuid,
1709
+
1710
+            /data/www/safe/* r,
1711
+            deny /data/www/unsafe/* r,
1712
+          }" > "${overrides_yaml}"
1713
+  install_base "--values=${overrides_yaml}"
1714
+  get_container_status apparmor
1715
+  _test_apparmor_profile_added divingbell-profile-1 profile-1
1716
+  _test_apparmor_profile_removed divingbell-profile-2 profile-2
1717
+  echo '[SUCCESS] apparmor test3 passed successfully' >> "${TEST_RESULTS}"
1718
+
1719
+  #Test4 - check for bad profile input
1720
+  echo "conf:
1721
+  apparmor:
1722
+    profiles:
1723
+      divingbell-profile-3: |
1724
+        #include <tunables/global>
1725
+          /usr/sbin/profile-3 {
1726
+            bad data
1727
+          }" > "${overrides_yaml}"
1728
+  install_base "--values=${overrides_yaml}"
1729
+  get_container_status apparmor expect_failure
1730
+  _test_clog_msg 'AppArmor parser error for /etc/apparmor.d/divingbell-profile-3 in /etc/apparmor.d/divingbell-profile-3 at line 3: syntax error, unexpected TOK_ID, expecting TOK_MODE'
1731
+  echo '[SUCCESS] apparmor test4 passed successfully' >> "${TEST_RESULTS}"
1732
+}
1733
+
1569 1734
 # initialization
1570 1735
 init_default_state
1571 1736
 
@@ -1580,6 +1745,7 @@ if [[ -z $SKIP_BASE_TESTS ]]; then
1580 1745
   test_uamlite
1581 1746
   test_apt
1582 1747
   test_exec
1748
+  test_apparmor
1583 1749
 fi
1584 1750
 purge_containers
1585 1751
 test_overrides

Loading…
Cancel
Save