From aa6b3bbeba027d5e385df48fa0d9c83e5b56f207 Mon Sep 17 00:00:00 2001 From: Spyros Trigazis Date: Tue, 22 Oct 2019 11:21:26 +0000 Subject: [PATCH] k8s_fedora: Add use_podman label Choose whether system containers etcd, kubernetes and the heat-agent will be installed with podman or atomic. This label is relevant for k8s_fedora drivers. k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be used pulling containers from docker.io/openstackmagnum. use_podman=true is accepted as well, which will pull containers by k8s.gcr.io. k8s_fedora_coreos_v1 defaults and accepts only use_podman=true. Fix upgrade for k8s_fedora_coreos_v1 and magnum-cordon systemd unit. Task: 37242 Story: 2005201 Change-Id: I0d5e4e059cd4f0458746df7c09d2fd47c389c6a0 Signed-off-by: Spyros Trigazis --- doc/source/user/index.rst | 17 ++++++ .../kubernetes/fragments/configure-etcd.sh | 42 ++++++++++++- .../fragments/configure-kubernetes-master.sh | 27 +++++++-- .../fragments/configure-kubernetes-minion.sh | 21 ++++++- .../fragments/start-container-agent.sh | 12 +++- .../fragments/upgrade-kubernetes.sh | 59 +++++++++++++++---- .../fragments/write-heat-params-master.sh | 1 + .../kubernetes/fragments/write-heat-params.sh | 1 + .../drivers/heat/k8s_fedora_template_def.py | 3 +- .../templates/kubecluster.yaml | 11 +++- .../templates/kubemaster.yaml | 7 +++ .../templates/kubeminion.yaml | 7 +++ .../templates/kubecluster.yaml | 25 ++++++++ .../templates/kubemaster.yaml | 18 ++++++ .../templates/kubeminion.yaml | 18 ++++++ .../unit/drivers/test_template_definition.py | 4 ++ .../notes/use_podman-39532143be2296c2.yaml | 17 ++++++ 17 files changed, 266 insertions(+), 24 deletions(-) create mode 100644 releasenotes/notes/use_podman-39532143be2296c2.yaml diff --git a/doc/source/user/index.rst b/doc/source/user/index.rst index f9ad4e1f0e..a6dbe1fc25 100644 --- a/doc/source/user/index.rst +++ b/doc/source/user/index.rst @@ -429,6 +429,9 @@ the table are linked to more details elsewhere in the user guide. | `npd_enabled`_ | - true | true | | | - false | | +---------------------------------------+--------------------+---------------+ +| `use_podman`_ | - true | see below | +| | - false | | ++---------------------------------------+--------------------+---------------+ .. _cluster: @@ -1372,7 +1375,21 @@ _`max_node_count` _`npd_enabled` Set Node Problem Detector service enabled or disabled. Default enabled. +_`use_podman` + Choose whether system containers etcd, kubernetes and the heat-agent will + be installed with podman or atomic. This label is relevant for + k8s_fedora drivers. + k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be + used pulling containers from docker.io/openstackmagnum. use_podman=true + is accepted as well, which will pull containers by k8s.gcr.io. + + k8s_fedora_coreos_v1 defaults and accepts only use_podman=true. + + Note that, to use kubernetes version greater or equal to v1.16.0 with the + k8s_fedora_atomic_v1 driver, you need to set use_podman=true. This is + necessary since v1.16 dropped the --containerized flag in kubelet. + https://github.com/kubernetes/kubernetes/pull/80043/files External load balancer for services ----------------------------------- diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh index 7ef2a8156a..0dffde8116 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh @@ -50,7 +50,8 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then fi -cat > /etc/systemd/system/etcd.service < /etc/systemd/system/etcd.service < /etc/etcd/etcd.conf <> /etc/etcd/etcd.conf <> /etc/etcd/etcd.conf +fi diff --git a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh index b7e9dd186d..afee7eb3ca 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh @@ -77,7 +77,8 @@ KUBE_PROXY_ARGS="" EOF -cat > /etc/systemd/system/kube-apiserver.service < /etc/systemd/system/kube-apiserver.service < /etc/systemd/system/kube-controller-manager.service < /etc/systemd/system/kube-controller-manager.service < /etc/systemd/system/kube-scheduler.service < /etc/systemd/system/kube-scheduler.service < /etc/systemd/system/kubelet.service < /etc/systemd/system/kubelet.service < /etc/systemd/system/kube-proxy.service < /etc/systemd/system/kube-proxy.service < /srv/magnum/kubernetes/install-kubernetes.sh < /etc/kubernetes/proxy < /etc/systemd/system/kubelet.service < /etc/systemd/system/kubelet.service < /etc/systemd/system/kube-proxy.service < /etc/systemd/system/kube-proxy.service < /srv/magnum/kubernetes/install-kubernetes.sh < /etc/systemd/system/heat-container-agent.service < /etc/systemd/system/heat-container-agent.service <> /etc/sysconfig/heat-params + ${ssh_cmd} podman rmi ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:${KUBE_TAG} + echo "KUBE_TAG=$new_kube_tag" >> /etc/sysconfig/heat-params - for service in ${SERVICE_LIST}; do - ${ssh_cmd} systemctl start ${service} - done + for service in ${SERVICE_LIST}; do + ${ssh_cmd} systemctl start ${service} + done + else + declare -A service_image_mapping + service_image_mapping=( ["kubelet"]="kubernetes-kubelet" ["kube-controller-manager"]="kubernetes-controller-manager" ["kube-scheduler"]="kubernetes-scheduler" ["kube-proxy"]="kubernetes-proxy" ["kube-apiserver"]="kubernetes-apiserver" ) + + SERVICE_LIST=$($ssh_cmd atomic containers list -f container=kube -q --no-trunc) + + for service in ${SERVICE_LIST}; do + ${ssh_cmd} systemctl stop ${service} + done + + for service in ${SERVICE_LIST}; do + ${ssh_cmd} atomic pull --storage ostree "docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag}" + done + + for service in ${SERVICE_LIST}; do + ${ssh_cmd} atomic containers update --rebase docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag} ${service} + done + + for service in ${SERVICE_LIST}; do + systemctl restart ${service} + done + + ${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${INSTANCE_NAME} + + for service in ${SERVICE_LIST}; do + ${ssh_cmd} atomic --assumeyes images "delete docker.io/openstackmagnum/${service_image_mapping[${service}]}:${KUBE_TAG}" + done + + ${ssh_cmd} atomic images prune + fi i=0 until kubectl uncordon ${INSTANCE_NAME} @@ -61,7 +96,7 @@ After=network.target kubelet.service [Service] Restart=Always RemainAfterExit=yes -ExecStart=${kubecontrol} uncordon ${HOSTNAME_OVERRIDE} +ExecStart=${kubecontrol} uncordon ${INSTANCE_NAME} [Install] WantedBy=multi-user.target diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh index 24a67e669c..b8bd4d8133 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh @@ -108,6 +108,7 @@ HEAT_PARAMS=/etc/sysconfig/heat-params NPD_ENABLED="$NPD_ENABLED" NODEGROUP_ROLE="$NODEGROUP_ROLE" NODEGROUP_NAME="$NODEGROUP_NAME" + USE_PODMAN="$USE_PODMAN" EOF } diff --git a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh index 273c2a52c7..b2b6a87147 100644 --- a/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh +++ b/magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh @@ -60,6 +60,7 @@ AUTO_HEALING_ENABLED="$AUTO_HEALING_ENABLED" AUTO_HEALING_CONTROLLER="$AUTO_HEALING_CONTROLLER" NODEGROUP_ROLE="$NODEGROUP_ROLE" NODEGROUP_NAME="$NODEGROUP_NAME" +USE_PODMAN="$USE_PODMAN" EOF } diff --git a/magnum/drivers/heat/k8s_fedora_template_def.py b/magnum/drivers/heat/k8s_fedora_template_def.py index 6f39497d12..9fd701cc8b 100644 --- a/magnum/drivers/heat/k8s_fedora_template_def.py +++ b/magnum/drivers/heat/k8s_fedora_template_def.py @@ -99,7 +99,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition): 'auto_healing_controller', 'magnum_auto_healer_tag', 'draino_tag', 'autoscaler_tag', 'min_node_count', 'max_node_count', 'npd_enabled', - 'ostree_remote', 'ostree_commit'] + 'ostree_remote', 'ostree_commit', + 'use_podman'] labels = self._get_relevant_labels(cluster, kwargs) diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml index 5165ab312b..4bb8973224 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml @@ -455,7 +455,7 @@ parameters: etcd_tag: type: string description: tag of the etcd system container - default: 3.2.26 + default: v3.2.7 coredns_tag: type: string @@ -765,6 +765,13 @@ parameters: description: The ostree commit to deploy default: '' + use_podman: + type: boolean + description: > + if true, run system containers for kubernetes, etcd and heat-agent + default: + false + resources: ###################################################################### @@ -1081,6 +1088,7 @@ resources: npd_enabled: {get_param: npd_enabled} ostree_remote: {get_param: ostree_remote} ostree_commit: {get_param: ostree_commit} + use_podman: {get_param: use_podman} kube_cluster_config: condition: create_cluster_resources @@ -1246,6 +1254,7 @@ resources: auto_healing_controller: {get_param: auto_healing_controller} ostree_remote: {get_param: ostree_remote} ostree_commit: {get_param: ostree_commit} + use_podman: {get_param: use_podman} outputs: diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml index e8238393b5..99bcd007a8 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml @@ -538,6 +538,11 @@ parameters: type: string description: The ostree commit to deploy + use_podman: + type: boolean + description: > + if true, run system containers for kubernetes, etcd and heat-agent + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -586,6 +591,7 @@ resources: params: $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} + $USE_PODMAN: {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh master_config: @@ -699,6 +705,7 @@ resources: "$NPD_ENABLED": {get_param: npd_enabled} "$NODEGROUP_ROLE": {get_param: nodegroup_role} "$NODEGROUP_NAME": {get_param: nodegroup_name} + "$USE_PODMAN": {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh diff --git a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml index b2c5059a2c..48711159e4 100644 --- a/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml @@ -322,6 +322,11 @@ parameters: description: The ostree commit to deploy default: '' + use_podman: + type: boolean + description: > + if true, run system containers for kubernetes, etcd and heat-agent + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -351,6 +356,7 @@ resources: params: $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix} $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag} + $USE_PODMAN: {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh ###################################################################### @@ -422,6 +428,7 @@ resources: $NPD_ENABLED: {get_param: npd_enabled} $NODEGROUP_ROLE: {get_param: nodegroup_role} $NODEGROUP_NAME: {get_param: nodegroup_name} + $USE_PODMAN: {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh - get_file: ../../common/templates/fragments/configure-docker-registry.sh diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml index d6ad2e0c58..1039623b6b 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml @@ -755,6 +755,25 @@ parameters: default: true + ostree_remote: + type: string + description: This parameter is ignored for k8s_fedora_coreos. + default: '' + + ostree_commit: + type: string + description: This parameter is ignored for k8s_fedora_coreos. + default: '' + + use_podman: + type: boolean + description: > + If true, run system containers for kubernetes, etcd and heat-agent + default: + true + constraints: + - allowed_values: [true] + resources: ###################################################################### @@ -1070,6 +1089,9 @@ resources: min_node_count: {get_param: min_node_count} max_node_count: {get_param: max_node_count} npd_enabled: {get_param: npd_enabled} + ostree_remote: {get_param: ostree_remote} + ostree_commit: {get_param: ostree_commit} + use_podman: {get_param: use_podman} kube_cluster_config: condition: create_cluster_resources @@ -1234,6 +1256,9 @@ resources: auto_healing_enabled: {get_param: auto_healing_enabled} npd_enabled: {get_param: npd_enabled} auto_healing_controller: {get_param: auto_healing_controller} + ostree_remote: {get_param: ostree_remote} + ostree_commit: {get_param: ostree_commit} + use_podman: {get_param: use_podman} outputs: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml index 753579e25d..339e705880 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml @@ -534,6 +534,19 @@ parameters: default: true + ostree_remote: + type: string + description: The ostree remote branch to upgrade + + ostree_commit: + type: string + description: The ostree commit to deploy + + use_podman: + type: boolean + description: > + If true, run system containers for kubernetes, etcd and heat-agent + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -690,6 +703,7 @@ resources: "$NPD_ENABLED": {get_param: npd_enabled} "$NODEGROUP_ROLE": {get_param: nodegroup_role} "$NODEGROUP_NAME": {get_param: nodegroup_name} + "$USE_PODMAN": {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh @@ -834,6 +848,8 @@ resources: group: script inputs: - name: kube_tag_input + - name: ostree_remote_input + - name: ostree_commit_input config: get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh @@ -846,6 +862,8 @@ resources: actions: ['UPDATE'] input_values: kube_tag_input: {get_param: kube_tag} + ostree_remote_input: {get_param: ostree_remote} + ostree_commit_input: {get_param: ostree_commit} outputs: diff --git a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml index 0daa775a75..1d0cdf8ecf 100644 --- a/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml +++ b/magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml @@ -316,6 +316,19 @@ parameters: default: true + ostree_remote: + type: string + description: The ostree remote branch to upgrade + + ostree_commit: + type: string + description: The ostree commit to deploy + + use_podman: + type: boolean + description: > + If true, run system containers for kubernetes, etcd and heat-agent + conditions: image_based: {equals: [{get_param: boot_volume_size}, 0]} @@ -411,6 +424,7 @@ resources: $NPD_ENABLED: {get_param: npd_enabled} $NODEGROUP_ROLE: {get_param: nodegroup_role} $NODEGROUP_NAME: {get_param: nodegroup_name} + $USE_PODMAN: {get_param: use_podman} - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh - get_file: ../../common/templates/fragments/configure-docker-registry.sh @@ -521,6 +535,8 @@ resources: group: script inputs: - name: kube_tag_input + - name: ostree_remote_input + - name: ostree_commit_input config: get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh @@ -533,6 +549,8 @@ resources: actions: ['UPDATE'] input_values: kube_tag_input: {get_param: kube_tag} + ostree_remote_input: {get_param: ostree_remote} + ostree_commit_input: {get_param: ostree_commit} outputs: diff --git a/magnum/tests/unit/drivers/test_template_definition.py b/magnum/tests/unit/drivers/test_template_definition.py index a3afffaefe..9e8710892f 100644 --- a/magnum/tests/unit/drivers/test_template_definition.py +++ b/magnum/tests/unit/drivers/test_template_definition.py @@ -572,6 +572,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): etcd_volume_type = mock_cluster.labels.get('etcd_volume_type') ostree_remote = mock_cluster.labels.get('ostree_remote') ostree_commit = mock_cluster.labels.get('ostree_commit') + use_podman = mock_cluster.labels.get('use_podman') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -658,6 +659,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'etcd_volume_type': etcd_volume_type, 'ostree_remote': ostree_remote, 'ostree_commit': ostree_commit, + 'use_podman': use_podman, }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, @@ -1012,6 +1014,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): etcd_volume_type = mock_cluster.labels.get('etcd_volume_type') ostree_remote = mock_cluster.labels.get('ostree_remote') ostree_commit = mock_cluster.labels.get('ostree_commit') + use_podman = mock_cluster.labels.get('use_podman') k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition() @@ -1100,6 +1103,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase): 'etcd_volume_type': etcd_volume_type, 'ostree_remote': ostree_remote, 'ostree_commit': ostree_commit, + 'use_podman': use_podman, }} mock_get_params.assert_called_once_with(mock_context, mock_cluster_template, diff --git a/releasenotes/notes/use_podman-39532143be2296c2.yaml b/releasenotes/notes/use_podman-39532143be2296c2.yaml new file mode 100644 index 0000000000..c3d9fbaed2 --- /dev/null +++ b/releasenotes/notes/use_podman-39532143be2296c2.yaml @@ -0,0 +1,17 @@ +--- +features: + - | + Choose whether system containers etcd, kubernetes and the heat-agent will + be installed with podman or atomic. This label is relevant for + k8s_fedora drivers. + + k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be + used pulling containers from docker.io/openstackmagnum. use_podman=true + is accepted as well, which will pull containers by k8s.gcr.io. + + k8s_fedora_coreos_v1 defaults and accepts only use_podman=true. + + Note that, to use kubernetes version greater or equal to v1.16.0 with the + k8s_fedora_atomic_v1 driver, you need to set use_podman=true. This is + necessary since v1.16 dropped the --containerized flag in kubelet. + https://github.com/kubernetes/kubernetes/pull/80043/files