Add label prioritization
Altered daemonset scheduling to determonistically ensure one and only one daemonset is schedule to each node. This is done via implicit label override prioritization. If nodes contain multiple labels, a given daemonset will always schedule to whichever label was the last to be defined in overrides yaml. Change-Id: Ib90f36f27e3bcd50d017262c07317aa3a64464bb
This commit is contained in:
parent
6d2f967ce4
commit
ae819b9a3b
@ -164,11 +164,10 @@ precedence and are used for that node. The label overrides are not used in this
|
|||||||
case. This is especially important to note if you are defining new host
|
case. This is especially important to note if you are defining new host
|
||||||
overrides for a node that is already consuming matching label overrides, as
|
overrides for a node that is already consuming matching label overrides, as
|
||||||
defining a host override would make those label overrides no longer apply.
|
defining a host override would make those label overrides no longer apply.
|
||||||
2. Daemonsets are generated regardless of the current state of the environment.
|
2. In the event of label conflicts, the last applicable label override defined
|
||||||
Ex: If your environment consists of a single node that matches a host override,
|
takes precedence. In this example, overrides defined for "another_label" would
|
||||||
the chart will still generate a default daemonset which would fail to schedule
|
take precedence and be applied to nodes that contained both of the defined
|
||||||
in this example. Likewise if the host or label in the override return no
|
labels.
|
||||||
candidates, these would also fail to schedule.
|
|
||||||
|
|
||||||
Recorded Demo
|
Recorded Demo
|
||||||
-------------
|
-------------
|
||||||
|
@ -63,6 +63,7 @@
|
|||||||
{{- end }}
|
{{- end }}
|
||||||
|
|
||||||
{{- if eq $type "labels" }}
|
{{- if eq $type "labels" }}
|
||||||
|
{{- set $.Values "__label_list" . }}
|
||||||
{{- range $label_data := . }}
|
{{- range $label_data := . }}
|
||||||
# dictionary that will contain all info needed to generate this
|
# dictionary that will contain all info needed to generate this
|
||||||
# iteration of the daemonset.
|
# iteration of the daemonset.
|
||||||
@ -86,8 +87,10 @@
|
|||||||
{{- $list_aggregate := list $label_dict }}
|
{{- $list_aggregate := list $label_dict }}
|
||||||
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
|
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
|
||||||
|
|
||||||
# Do not schedule to any other specified labels
|
# Do not schedule to other specified labels, with higher
|
||||||
{{- $other_labels := without $type_data $label_data }}
|
# precedence as the list position increases. Last defined label
|
||||||
|
# is highest priority.
|
||||||
|
{{- $other_labels := without $.Values.__label_list $label_data }}
|
||||||
{{- range $label_data2 := $other_labels }}
|
{{- range $label_data2 := $other_labels }}
|
||||||
{{- $label_dict := omit $label_data2.label "NULL" }}
|
{{- $label_dict := omit $label_data2.label "NULL" }}
|
||||||
|
|
||||||
@ -96,6 +99,7 @@
|
|||||||
{{- $list_aggregate := append $.Values.__current_label.matchExpressions $label_dict }}
|
{{- $list_aggregate := append $.Values.__current_label.matchExpressions $label_dict }}
|
||||||
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
|
{{- set $.Values.__current_label "matchExpressions" $list_aggregate }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
|
{{- set $.Values "__label_list" $other_labels }}
|
||||||
|
|
||||||
# Do not schedule to any other specified hosts
|
# Do not schedule to any other specified hosts
|
||||||
{{- range $type, $type_data := $val }}
|
{{- range $type, $type_data := $val }}
|
||||||
|
@ -159,7 +159,8 @@ get_container_status(){
|
|||||||
echo "${CLOGS}"
|
echo "${CLOGS}"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
elif [ "${status}" = 'INFO Putting the daemon to sleep.' ]; then
|
elif [ "${status}" = 'INFO Putting the daemon to sleep.' ] ||
|
||||||
|
[ "${status}" = 'DEBUG + exit 0' ]; then
|
||||||
if [ "${2}" = 'expect_failure' ]; then
|
if [ "${2}" = 'expect_failure' ]; then
|
||||||
echo 'Expected pod to die with error, but pod completed successfully'
|
echo 'Expected pod to die with error, but pod completed successfully'
|
||||||
echo 'pod logs:'
|
echo 'pod logs:'
|
||||||
@ -580,24 +581,15 @@ test_overrides(){
|
|||||||
# TODO: Implement more robust tests that do not depend on match expression
|
# TODO: Implement more robust tests that do not depend on match expression
|
||||||
# ordering.
|
# ordering.
|
||||||
|
|
||||||
# Verify generated affinity for one of the daemonset labels
|
# Verify generated affinity for test_label
|
||||||
echo "${tc_output}" | grep ' spec:
|
echo "${tc_output}" | grep ' spec:
|
||||||
affinity:
|
affinity:
|
||||||
nodeAffinity:
|
nodeAffinity:
|
||||||
requiredDuringSchedulingIgnoredDuringExecution:
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
nodeSelectorTerms:
|
nodeSelectorTerms:
|
||||||
- matchExpressions:
|
- matchExpressions:
|
||||||
- key: another_label
|
|
||||||
operator: In
|
|
||||||
values:
|
|
||||||
- "another_value"
|
|
||||||
- key: compute_type
|
|
||||||
operator: NotIn
|
|
||||||
values:
|
|
||||||
- "dpdk"
|
|
||||||
- "sriov"
|
|
||||||
- key: test_label
|
- key: test_label
|
||||||
operator: NotIn
|
operator: In
|
||||||
values:
|
values:
|
||||||
- "test_value"
|
- "test_value"
|
||||||
- key: kubernetes.io/hostname
|
- key: kubernetes.io/hostname
|
||||||
@ -616,6 +608,73 @@ test_overrides(){
|
|||||||
echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" ||
|
echo '[SUCCESS] overrides test 2 passed successfully' >> "${TEST_RESULTS}" ||
|
||||||
(echo '[FAILURE] overrides test 2 failed' && exit 1)
|
(echo '[FAILURE] overrides test 2 failed' && exit 1)
|
||||||
|
|
||||||
|
# Verify generated affinity for another_label
|
||||||
|
echo "${tc_output}" | grep ' spec:
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: another_label
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- "another_value"
|
||||||
|
- key: test_label
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "test_value"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "superhost"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "helm1"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "specialhost"
|
||||||
|
hostNetwork: true' &&
|
||||||
|
echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
|
||||||
|
(echo '[FAILURE] overrides test 3 failed' && exit 1)
|
||||||
|
|
||||||
|
# Verify generated affinity for compute_type
|
||||||
|
echo "${tc_output}" | grep ' spec:
|
||||||
|
affinity:
|
||||||
|
nodeAffinity:
|
||||||
|
requiredDuringSchedulingIgnoredDuringExecution:
|
||||||
|
nodeSelectorTerms:
|
||||||
|
- matchExpressions:
|
||||||
|
- key: compute_type
|
||||||
|
operator: In
|
||||||
|
values:
|
||||||
|
- "dpdk"
|
||||||
|
- "sriov"
|
||||||
|
- key: another_label
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "another_value"
|
||||||
|
- key: test_label
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "test_value"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "superhost"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "helm1"
|
||||||
|
- key: kubernetes.io/hostname
|
||||||
|
operator: NotIn
|
||||||
|
values:
|
||||||
|
- "specialhost"
|
||||||
|
hostNetwork: true' &&
|
||||||
|
echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
|
||||||
|
(echo '[FAILURE] overrides test 4 failed' && exit 1)
|
||||||
|
|
||||||
# Verify generated affinity for one of the daemonset hosts
|
# Verify generated affinity for one of the daemonset hosts
|
||||||
echo "${tc_output}" | grep ' spec:
|
echo "${tc_output}" | grep ' spec:
|
||||||
affinity:
|
affinity:
|
||||||
@ -629,8 +688,8 @@ test_overrides(){
|
|||||||
- "soup"
|
- "soup"
|
||||||
- "chips"
|
- "chips"
|
||||||
hostNetwork: true' &&
|
hostNetwork: true' &&
|
||||||
echo '[SUCCESS] overrides test 3 passed successfully' >> "${TEST_RESULTS}" ||
|
echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}" ||
|
||||||
(echo '[FAILURE] overrides test 3 failed' && exit 1)
|
(echo '[FAILURE] overrides test 5 failed' && exit 1)
|
||||||
|
|
||||||
# Verify generated affinity for one of the daemonset defaults
|
# Verify generated affinity for one of the daemonset defaults
|
||||||
echo "${tc_output}" | grep ' spec:
|
echo "${tc_output}" | grep ' spec:
|
||||||
@ -665,8 +724,8 @@ test_overrides(){
|
|||||||
values:
|
values:
|
||||||
- "test_value"
|
- "test_value"
|
||||||
hostNetwork: true' &&
|
hostNetwork: true' &&
|
||||||
echo '[SUCCESS] overrides test 4 passed successfully' >> "${TEST_RESULTS}" ||
|
echo '[SUCCESS] overrides test 6 passed successfully' >> "${TEST_RESULTS}" ||
|
||||||
(echo '[FAILURE] overrides test 4 failed' && exit 1)
|
(echo '[FAILURE] overrides test 6 failed' && exit 1)
|
||||||
|
|
||||||
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
|
overrides_yaml=${LOGS_SUBDIR}/${FUNCNAME}-functional.yaml
|
||||||
key1_override_val=0
|
key1_override_val=0
|
||||||
@ -686,7 +745,7 @@ test_overrides(){
|
|||||||
get_container_status sysctl
|
get_container_status sysctl
|
||||||
_test_sysctl_default $SYSCTL_KEY1 $key1_override_val
|
_test_sysctl_default $SYSCTL_KEY1 $key1_override_val
|
||||||
_test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
|
_test_sysctl_default $SYSCTL_KEY2 $key2_non_override_val
|
||||||
echo '[SUCCESS] overrides test 5 passed successfully' >> "${TEST_RESULTS}"
|
echo '[SUCCESS] overrides test 7 passed successfully' >> "${TEST_RESULTS}"
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user