Browse Source

Merge "Add the ability to install packages via divingbell"

Zuul 5 months ago
parent
commit
96e2b073f3

+ 112
- 0
divingbell/templates/bin/_apt.sh.tpl View File

@@ -0,0 +1,112 @@
1
+#!/bin/bash
2
+
3
+{{/*
4
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
5
+#
6
+# Licensed under the Apache License, Version 2.0 (the "License");
7
+# you may not use this file except in compliance with the License.
8
+# You may obtain a copy of the License at
9
+#
10
+#     http://www.apache.org/licenses/LICENSE-2.0
11
+#
12
+# Unless required by applicable law or agreed to in writing, software
13
+# distributed under the License is distributed on an "AS IS" BASIS,
14
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15
+# See the License for the specific language governing permissions and
16
+# limitations under the License.
17
+*/}}
18
+
19
+set -e
20
+
21
+cat <<'EOF' > {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
22
+{{ include "divingbell.shcommon" . }}
23
+
24
+persist_path='/var/divingbell/apt'
25
+declare -A CURRENT_PACKAGES
26
+declare INSTALLED_THIS_TIME
27
+declare TO_DELETE
28
+declare TO_KEEP
29
+declare REQUESTED_PACKAGES
30
+
31
+if [ ! -d "${persist_path}" ]; then
32
+  mkdir -p "${persist_path}"
33
+fi
34
+
35
+write_test "${persist_path}"
36
+
37
+load_package_list_with_versions(){
38
+    set +x
39
+    for f in "$@"; do
40
+        IFS="=" read -r name version <<< $f;
41
+        IFS=":" read -r name arch <<< $name;
42
+        CURRENT_PACKAGES["$name"]="$version";
43
+    done
44
+    set -x
45
+}
46
+
47
+################################################
48
+#Stage 1
49
+#Collect data
50
+################################################
51
+
52
+# First 5 lines are field descriptions
53
+load_package_list_with_versions $(dpkg -l | awk 'NR>5 {print $2"="$3}')
54
+
55
+################################################
56
+#Stage 2
57
+#Install new packages
58
+################################################
59
+
60
+{{- if hasKey .Values.conf "apt" }}
61
+{{- if hasKey .Values.conf.apt "packages" }}
62
+{{- range .Values.conf.apt.packages }}
63
+if [[ "${CURRENT_PACKAGES[{{ .name | squote }}]+isset}" != "isset"{{- if .version }} || "${CURRENT_PACKAGES[{{ .name | squote }}]}" != {{ .version | squote }}{{- end }} ]]; then
64
+    apt-get install -y{{ if .repo }} -t {{ .repo | squote }}{{ end }} {{ .name | squote -}} {{- if .version }}={{ .version | squote }}{{ end }}
65
+    INSTALLED_THIS_TIME="$INSTALLED_THIS_TIME {{ .name }}"
66
+fi
67
+REQUESTED_PACKAGES="$REQUESTED_PACKAGES {{ .name }}"
68
+{{- end }}
69
+{{- end }}
70
+{{- end }}
71
+
72
+################################################
73
+#Stage 3
74
+#Remove packages not present in conf.apt anymore
75
+################################################
76
+
77
+echo $INSTALLED_THIS_TIME | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.new
78
+echo $REQUESTED_PACKAGES | sed 's/ /\n/g' | sed '/^[[:space:]]*$/d' | sort > ${persist_path}/packages.requested
79
+if [ -f ${persist_path}/packages ]; then
80
+    TO_DELETE=$(comm -23 ${persist_path}/packages ${persist_path}/packages.requested)
81
+    TO_KEEP=$(echo "$TO_DELETE" | comm -23 ${persist_path}/packages -)
82
+    if [ ! -z "$TO_DELETE" ]; then
83
+        for pkg in "$TO_DELETE"; do
84
+            apt-get purge -y $pkg
85
+        done
86
+        apt-get autoremove -y
87
+    fi
88
+    if [ ! -z "$TO_KEEP" ]; then
89
+        echo "$TO_KEEP" > ${persist_path}/packages
90
+    else
91
+        rm ${persist_path}/packages
92
+    fi
93
+fi
94
+if [ ! -z "$INSTALLED_THIS_TIME" ]; then
95
+    cat ${persist_path}/packages.new >> ${persist_path}/packages
96
+    sort ${persist_path}/packages -o ${persist_path}/packages
97
+fi
98
+
99
+exit 0
100
+EOF
101
+
102
+chmod 755 {{ .Values.conf.chroot_mnt_path | quote }}/tmp/apt.sh
103
+chroot {{ .Values.conf.chroot_mnt_path | quote }} /tmp/apt.sh
104
+
105
+sleep 1
106
+echo 'INFO Putting the daemon to sleep.'
107
+
108
+while [ 1 ]; do
109
+  sleep 300
110
+done
111
+
112
+exit 0

+ 69
- 0
divingbell/templates/daemonset-apt.yaml View File

@@ -0,0 +1,69 @@
1
+{{/*
2
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
3
+#
4
+# Licensed under the Apache License, Version 2.0 (the "License");
5
+# you may not use this file except in compliance with the License.
6
+# You may obtain a copy of the License at
7
+#
8
+#     http://www.apache.org/licenses/LICENSE-2.0
9
+#
10
+# Unless required by applicable law or agreed to in writing, software
11
+# distributed under the License is distributed on an "AS IS" BASIS,
12
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+# See the License for the specific language governing permissions and
14
+# limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.daemonset.apt" }}
18
+  {{- $daemonset := index . 0 }}
19
+  {{- $secretName := index . 1 }}
20
+  {{- $envAll := index . 2 }}
21
+  {{- with $envAll }}
22
+---
23
+apiVersion: extensions/v1beta1
24
+kind: DaemonSet
25
+metadata:
26
+  name: {{ $daemonset }}
27
+spec:
28
+{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }}
29
+  template:
30
+    metadata:
31
+      labels:
32
+{{ list $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }}
33
+    spec:
34
+      hostNetwork: true
35
+      hostPID: true
36
+      hostIPC: true
37
+      containers:
38
+      - name: {{ $daemonset }}
39
+        image: {{ .Values.images.divingbell }}
40
+        imagePullPolicy: {{ .Values.images.pull_policy }}
41
+{{ tuple $envAll $envAll.Values.pod.resources.apt | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }}
42
+        command:
43
+        - /tmp/{{ $daemonset }}.sh
44
+        volumeMounts:
45
+        - name: rootfs-{{ $daemonset }}
46
+          mountPath: {{ .Values.conf.chroot_mnt_path }}
47
+        - name: {{ $secretName }}
48
+          mountPath: /tmp/{{ $daemonset }}.sh
49
+          subPath: {{ $daemonset }}
50
+          readOnly: true
51
+        securityContext:
52
+          privileged: true
53
+      volumes:
54
+      - name: rootfs-{{ $daemonset }}
55
+        hostPath:
56
+          path: /
57
+      - name: {{ $secretName }}
58
+        secret:
59
+          secretName: {{ $secretName }}
60
+          defaultMode: 0555
61
+  {{- end }}
62
+{{- end }}
63
+{{- if .Values.manifests.daemonset_apt }}
64
+{{- $daemonset := "apt" }}
65
+{{- $secretName := "divingbell-apt" }}
66
+{{- $daemonset_yaml := list $daemonset $secretName . | include "divingbell.daemonset.apt" | toString | fromYaml }}
67
+{{- $secret_include := "divingbell.secret.apt" }}
68
+{{- list $daemonset $daemonset_yaml $secret_include $secretName . | include "helm-toolkit.utils.daemonset_overrides" }}
69
+{{- end }}

+ 30
- 0
divingbell/templates/secret-apt.yaml View File

@@ -0,0 +1,30 @@
1
+{{/*
2
+Copyright 2018 The Openstack-Helm Authors.
3
+
4
+Licensed under the Apache License, Version 2.0 (the "License");
5
+you may not use this file except in compliance with the License.
6
+You may obtain a copy of the License at
7
+
8
+   http://www.apache.org/licenses/LICENSE-2.0
9
+
10
+Unless required by applicable law or agreed to in writing, software
11
+distributed under the License is distributed on an "AS IS" BASIS,
12
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13
+See the License for the specific language governing permissions and
14
+limitations under the License.
15
+*/}}
16
+
17
+{{- define "divingbell.secret.apt" }}
18
+{{- $secretName := index . 0 }}
19
+{{- $envAll := index . 1 }}
20
+{{- with $envAll }}
21
+---
22
+apiVersion: v1
23
+kind: Secret
24
+metadata:
25
+  name: {{ $secretName }}
26
+data:
27
+  apt: |+
28
+{{ tuple "bin/_apt.sh.tpl" . | include "helm-toolkit.utils.template" | b64enc | indent 4 }}
29
+{{- end }}
30
+{{- end }}

+ 12
- 0
divingbell/values.yaml View File

@@ -61,6 +61,10 @@ pod:
61 61
           enabled: true
62 62
           min_ready_seconds: 0
63 63
           max_unavailable: 100%
64
+        apt:
65
+          enabled: true
66
+          min_ready_seconds: 0
67
+          max_unavailable: 100%
64 68
         limits:
65 69
           enabled: true
66 70
           min_ready_seconds: 0
@@ -102,6 +106,13 @@ pod:
102 106
       requests:
103 107
         memory: "128Mi"
104 108
         cpu: "100m"
109
+    apt:
110
+      limits:
111
+        memory: "128Mi"
112
+        cpu: "100m"
113
+      requests:
114
+        memory: "128Mi"
115
+        cpu: "100m"
105 116
 
106 117
 manifests:
107 118
   daemonset_ethtool: true
@@ -109,3 +120,4 @@ manifests:
109 120
   daemonset_uamlite: true
110 121
   daemonset_sysctl: true
111 122
   daemonset_limits: true
123
+  daemonset_apt: true

+ 17
- 5
doc/source/index.rst View File

@@ -124,11 +124,6 @@ Used to manage host level NIC tunables. Ex::
124 124
           tx-tcp-segmentation: off
125 125
           tx-checksum-ip-generic: on
126 126
 
127
-packages
128
-^^^^^^^^
129
-
130
-Not implemented
131
-
132 127
 uamlite
133 128
 ^^^^^^^
134 129
 
@@ -146,6 +141,23 @@ access. Ex::
146 141
           - ssh-rsa AAAAB3N... key1-comment
147 142
           - ssh-rsa AAAAVY6... key2-comment
148 143
 
144
+apt
145
+^^^
146
+
147
+``apt`` daemonset does package management. It is able to install a package of
148
+a specific version (or upgrade an existing one to requested version). Version
149
+is optional, and if not provided the latest available package is installed.
150
+It can also remove packages that were previously installed by divingbell (it is
151
+done by excluding the packages you want to remove from the configuration).
152
+Here is an example configuration for it::
153
+
154
+    conf:
155
+      apt:
156
+        packages:
157
+        - name: <PACKAGE1>
158
+          version: <VERSION1>
159
+        - name: <PACKAGE2>
160
+
149 161
 Operations
150 162
 ----------
151 163
 

+ 98
- 2
tools/gate/scripts/020-test-divingbell.sh View File

@@ -47,6 +47,13 @@ USERNAME3=userthree
47 47
 USERNAME3_SUDO=true
48 48
 USERNAME4=userfour
49 49
 USERNAME4_SUDO=false
50
+APT_PACKAGE1=python-pbr
51
+APT_VERSION1=1.8.0-4ubuntu1
52
+APT_PACKAGE2=python-yaml
53
+APT_PACKAGE3=python-simplejson
54
+APT_VERSION3=3.8.1-1ubuntu2
55
+APT_PACKAGE4=less
56
+APT_PACKAGE5=python-setuptools
50 57
 type lshw || apt -y install lshw
51 58
 nic_info="$(lshw -class network)"
52 59
 physical_nic=''
@@ -753,6 +760,94 @@ test_uamlite(){
753 760
   echo '[SUCCESS] uamlite test6 passed successfully' >> "${TEST_RESULTS}"
754 761
 }
755 762
 
763
+_test_apt_package_version(){
764
+  local pkg_name=$1
765
+  local pkg_ver=$2
766
+  if [ ${pkg_ver} = "none" ]; then
767
+    if [[ $(dpkg -l | grep ${pkg_name}) ]]; then
768
+      echo "[FAIL] Package ${pkg_name} should not be installed" >> "${TEST_RESULTS}"
769
+      return 1
770
+    fi
771
+  elif [ ${pkg_ver} = "any" ]; then
772
+    if [[ ! $(dpkg -l | grep ${pkg_name}) ]]; then
773
+      echo "[FAIL] Package ${pkg_name} should be installed" >> "${TEST_RESULTS}"
774
+      return 1
775
+    fi
776
+  else
777
+    if [ $(dpkg -l | awk "/[[:space:]]${pkg_name}[[:space:]]/"'{print $3}') != "${pkg_ver}" ]; then
778
+      echo "[FAIL] Package ${pkg_name} should be of version ${pkg_ver}" >> "${TEST_RESULTS}"
779
+      return 1
780
+    fi
781
+  fi
782
+}
783
+
784
+test_apt(){
785
+  # Test the valid set of packages
786
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set1.yaml
787
+  echo "conf:
788
+  apt:
789
+    packages:
790
+    - name: $APT_PACKAGE1
791
+      version: $APT_VERSION1
792
+    - name: $APT_PACKAGE2" > "${overrides_yaml}"
793
+  install_base "--values=${overrides_yaml}"
794
+  get_container_status apt
795
+  _test_apt_package_version $APT_PACKAGE1 $APT_VERSION1
796
+  _test_apt_package_version $APT_PACKAGE2 any
797
+  echo '[SUCCESS] apt test1 passed successfully' >> "${TEST_RESULTS}"
798
+
799
+  # Test removal of one package and install of one new package
800
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set2.yaml
801
+  echo "conf:
802
+  apt:
803
+    packages:
804
+    - name: $APT_PACKAGE2
805
+    - name: $APT_PACKAGE3
806
+      version: $APT_VERSION3" > "${overrides_yaml}"
807
+  install_base "--values=${overrides_yaml}"
808
+  get_container_status apt
809
+  _test_apt_package_version $APT_PACKAGE1 none
810
+  _test_apt_package_version $APT_PACKAGE2 any
811
+  _test_apt_package_version $APT_PACKAGE3 $APT_VERSION3
812
+  echo '[SUCCESS] apt test2 passed successfully' >> "${TEST_RESULTS}"
813
+
814
+  # Test removal of all installed packages and install of one that already exists
815
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set3.yaml
816
+  echo "conf:
817
+  apt:
818
+    packages:
819
+    - name: $APT_PACKAGE4" > "${overrides_yaml}"
820
+  install_base "--values=${overrides_yaml}"
821
+  get_container_status apt
822
+  _test_apt_package_version $APT_PACKAGE2 none
823
+  _test_apt_package_version $APT_PACKAGE3 none
824
+  echo '[SUCCESS] apt test3 passed successfully' >> "${TEST_RESULTS}"
825
+
826
+  # Test package not installed by divingbell not removed
827
+  local overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-set4.yaml
828
+  echo "conf:
829
+  apt:
830
+    packages:
831
+    - name: $APT_PACKAGE5" > "${overrides_yaml}"
832
+  install_base "--values=${overrides_yaml}"
833
+  get_container_status apt
834
+  _test_apt_package_version $APT_PACKAGE4 any  # Should still be present
835
+  _test_apt_package_version $APT_PACKAGE5 any
836
+  echo '[SUCCESS] apt test4 passed successfully' >> "${TEST_RESULTS}"
837
+
838
+  # Test invalid package name
839
+  overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-invalid1.yaml
840
+  echo "conf:
841
+  apt:
842
+    packages:
843
+    - name: some-random-name
844
+      version: whatever" > "${overrides_yaml}"
845
+  install_base "--values=${overrides_yaml}"
846
+  get_container_status apt expect_failure
847
+  _test_clog_msg 'E: Unable to locate package some-random-name'
848
+  echo '[SUCCESS] apt test5 passed successfully' >> "${TEST_RESULTS}"
849
+}
850
+
756 851
 # test daemonset value overrides for hosts and labels
757 852
 test_overrides(){
758 853
   overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-dryrun.yaml
@@ -848,9 +943,9 @@ test_overrides(){
848 943
 
849 944
   # Compare against expected number of generated daemonsets
850 945
   daemonset_count="$(echo "${tc_output}" | grep 'kind: DaemonSet' | wc -l)"
851
-  if [ "${daemonset_count}" != "13" ]; then
946
+  if [ "${daemonset_count}" != "14" ]; then
852 947
     echo '[FAILURE] overrides test 1 failed' >> "${TEST_RESULTS}"
853
-    echo "Expected 13 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
948
+    echo "Expected 14 daemonsets; got '${daemonset_count}'" >> "${TEST_RESULTS}"
854 949
     exit 1
855 950
   else
856 951
     echo '[SUCCESS] overrides test 1 passed successfully' >> "${TEST_RESULTS}"
@@ -1032,6 +1127,7 @@ test_limits
1032 1127
 test_mounts
1033 1128
 test_ethtool
1034 1129
 test_uamlite
1130
+test_apt
1035 1131
 purge_containers
1036 1132
 test_overrides
1037 1133
 

Loading…
Cancel
Save