k8s_fedora: Add use_podman label

Choose whether system containers etcd, kubernetes and the heat-agent will be
installed with podman or atomic. This label is relevant for k8s_fedora drivers.

k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be used
pulling containers from docker.io/openstackmagnum. use_podman=true is accepted
as well, which will pull containers by k8s.gcr.io.

k8s_fedora_coreos_v1 defaults and accepts only use_podman=true.

Fix upgrade for k8s_fedora_coreos_v1 and magnum-cordon systemd unit.

Task: 37242
Story: 2005201

Change-Id: I0d5e4e059cd4f0458746df7c09d2fd47c389c6a0
Signed-off-by: Spyros Trigazis <spyridon.trigazis@cern.ch>

doc/source/user/index.rst

@@ -429,6 +429,9 @@ the table are linked to more details elsewhere in the user guide.
 | `npd_enabled`_                        | - true             | true          |
 |                                       | - false            |               |
 +---------------------------------------+--------------------+---------------+
+| `use_podman`_                         | - true             | see below     |
+|                                       | - false            |               |
++---------------------------------------+--------------------+---------------+
 
 .. _cluster:
 
@@ -1372,7 +1375,21 @@ _`max_node_count`
 _`npd_enabled`
   Set Node Problem Detector service enabled or disabled. Default enabled.
 
+_`use_podman`
+  Choose whether system containers etcd, kubernetes and the heat-agent will
+  be installed with podman or atomic. This label is relevant for
+  k8s_fedora drivers.
+
+  k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be
+  used pulling containers from docker.io/openstackmagnum. use_podman=true
+  is accepted as well, which will pull containers by k8s.gcr.io.
+
+  k8s_fedora_coreos_v1 defaults and accepts only use_podman=true.
 
+  Note that, to use kubernetes version greater or equal to v1.16.0 with the
+  k8s_fedora_atomic_v1 driver, you need to set use_podman=true. This is
+  necessary since v1.16 dropped the --containerized flag in kubelet.
+  https://github.com/kubernetes/kubernetes/pull/80043/files
 
 External load balancer for services
 -----------------------------------

magnum/drivers/common/templates/kubernetes/fragments/configure-etcd.sh

@@ -50,7 +50,8 @@ if [ -n "$ETCD_VOLUME_SIZE" ] && [ "$ETCD_VOLUME_SIZE" -gt 0 ]; then
 
 fi
 
-cat > /etc/systemd/system/etcd.service <<EOF
+if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+    cat > /etc/systemd/system/etcd.service <<EOF
 [Unit]
 Description=Etcd server
 After=network-online.target
@@ -73,6 +74,14 @@ ExecStop=/bin/podman stop etcd
 [Install]
 WantedBy=multi-user.target
 EOF
+else
+    _prefix=${CONTAINER_INFRA_PREFIX:-"docker.io/openstackmagnum/"}
+    $ssh_cmd atomic install \
+    --system-package no \
+    --system \
+    --storage ostree \
+    --name=etcd ${_prefix}etcd:${ETCD_TAG}
+fi
 
 
 if [ -z "$KUBE_NODE_IP" ]; then
@@ -154,3 +163,34 @@ peer-transport-security:
   trusted-ca-file: $cert_dir/ca.crt
 EOF
 fi
+# backwards compatible conf file
+cat > /etc/etcd/etcd.conf <<EOF
+ETCD_NAME="$INSTANCE_NAME"
+ETCD_DATA_DIR="/var/lib/etcd/default.etcd"
+ETCD_LISTEN_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_LISTEN_PEER_URLS="$protocol://$myip:2380"
+ETCD_ADVERTISE_CLIENT_URLS="$protocol://$myip:2379,http://127.0.0.1:2379"
+ETCD_INITIAL_ADVERTISE_PEER_URLS="$protocol://$myip:2380"
+ETCD_DISCOVERY="$ETCD_DISCOVERY_URL"
+EOF
+
+if [ "$TLS_DISABLED" = "False" ]; then
+
+cat >> /etc/etcd/etcd.conf <<EOF
+ETCD_CA_FILE=$cert_dir/ca.crt
+ETCD_TRUSTED_CA_FILE=$cert_dir/ca.crt
+ETCD_CERT_FILE=$cert_dir/server.crt
+ETCD_KEY_FILE=$cert_dir/server.key
+ETCD_CLIENT_CERT_AUTH=true
+ETCD_PEER_CA_FILE=$cert_dir/ca.crt
+ETCD_PEER_TRUSTED_CA_FILE=$cert_dir/ca.crt
+ETCD_PEER_CERT_FILE=$cert_dir/server.crt
+ETCD_PEER_KEY_FILE=$cert_dir/server.key
+ETCD_PEER_CLIENT_CERT_AUTH=true
+EOF
+
+fi
+
+if [ -n "$HTTP_PROXY" ]; then
+    echo "ETCD_DISCOVERY_PROXY=$HTTP_PROXY" >> /etc/etcd/etcd.conf
+fi

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-master.sh

@@ -77,7 +77,8 @@ KUBE_PROXY_ARGS=""
 EOF
 
 
-cat > /etc/systemd/system/kube-apiserver.service <<EOF
+if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+    cat > /etc/systemd/system/kube-apiserver.service <<EOF
 [Unit]
 Description=kube-apiserver via Hyperkube
 [Service]
@@ -105,7 +106,7 @@ RestartSec=10
 WantedBy=multi-user.target
 EOF
 
-cat > /etc/systemd/system/kube-controller-manager.service <<EOF
+    cat > /etc/systemd/system/kube-controller-manager.service <<EOF
 [Unit]
 Description=kube-controller-manager via Hyperkube
 [Service]
@@ -133,7 +134,7 @@ RestartSec=10
 WantedBy=multi-user.target
 EOF
 
-cat > /etc/systemd/system/kube-scheduler.service <<EOF
+    cat > /etc/systemd/system/kube-scheduler.service <<EOF
 [Unit]
 Description=kube-scheduler via Hyperkube
 [Service]
@@ -162,7 +163,7 @@ EOF
 
 
 
-cat > /etc/systemd/system/kubelet.service <<EOF
+    cat > /etc/systemd/system/kubelet.service <<EOF
 [Unit]
 Description=Kubelet via Hyperkube (System Container)
 [Service]
@@ -207,7 +208,7 @@ RestartSec=10
 WantedBy=multi-user.target
 EOF
 
-cat > /etc/systemd/system/kube-proxy.service <<EOF
+    cat > /etc/systemd/system/kube-proxy.service <<EOF
 [Unit]
 Description=kube-proxy via Hyperkube
 [Service]
@@ -237,7 +238,21 @@ RestartSec=10
 [Install]
 WantedBy=multi-user.target
 EOF
-
+else
+    _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
+    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
+    mkdir -p /srv/magnum/kubernetes/
+    cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
+#!/bin/bash -x
+atomic install --storage ostree --system --set=ADDTL_MOUNTS='${_addtl_mounts}' --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
+EOF
+    chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
+    $ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
+fi
 
 CERT_DIR=/etc/kubernetes/certs
 

magnum/drivers/common/templates/kubernetes/fragments/configure-kubernetes-minion.sh

@@ -61,7 +61,8 @@ EOF
 cat > /etc/kubernetes/proxy <<EOF
 KUBE_PROXY_ARGS=""
 EOF
-cat > /etc/systemd/system/kubelet.service <<EOF
+if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+    cat > /etc/systemd/system/kubelet.service <<EOF
 [Unit]
 Description=Kubelet via Hyperkube (System Container)
 [Service]
@@ -73,6 +74,7 @@ ExecStartPre=/bin/mkdir -p /etc/kubernetes/manifests
 ExecStartPre=/bin/mkdir -p /var/lib/calico
 ExecStartPre=/bin/mkdir -p /var/lib/kubelet/volumeplugins
 ExecStartPre=/bin/mkdir -p /opt/cni/bin
+ExecStartPre=-/bin/bash -c '/usr/bin/podman run --privileged --user root --net host --rm --volume /usr/local/bin:/host/usr/local/bin \${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:\${KUBE_TAG} /bin/sh -c "cp /usr/local/bin/kubectl /host/usr/local/bin/kubectl"'
 ExecStartPre=-/usr/bin/podman rm kubelet
 ExecStart=/bin/bash -c '/usr/bin/podman run --name kubelet \\
     --privileged \\
@@ -106,7 +108,7 @@ RestartSec=10
 WantedBy=multi-user.target
 EOF
 
-cat > /etc/systemd/system/kube-proxy.service <<EOF
+    cat > /etc/systemd/system/kube-proxy.service <<EOF
 [Unit]
 Description=kube-proxy via Hyperkube
 [Service]
@@ -136,6 +138,21 @@ RestartSec=10
 [Install]
 WantedBy=multi-user.target
 EOF
+else
+    _prefix=${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}
+    _addtl_mounts=',{"type":"bind","source":"/opt/cni","destination":"/opt/cni","options":["bind","rw","slave","mode=777"]},{"type":"bind","source":"/var/lib/docker","destination":"/var/lib/docker","options":["bind","rw","slave","mode=755"]}'
+    mkdir -p /srv/magnum/kubernetes/
+    cat > /srv/magnum/kubernetes/install-kubernetes.sh <<EOF
+#!/bin/bash -x
+atomic install --storage ostree --system --set=ADDTL_MOUNTS='${_addtl_mounts}' --system-package=no --name=kubelet ${_prefix}kubernetes-kubelet:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-apiserver ${_prefix}kubernetes-apiserver:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-controller-manager ${_prefix}kubernetes-controller-manager:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-scheduler ${_prefix}kubernetes-scheduler:${KUBE_TAG}
+atomic install --storage ostree --system --system-package=no --name=kube-proxy ${_prefix}kubernetes-proxy:${KUBE_TAG}
+EOF
+    chmod +x /srv/magnum/kubernetes/install-kubernetes.sh
+    $ssh_cmd "/srv/magnum/kubernetes/install-kubernetes.sh"
+fi
 
 CERT_DIR=/etc/kubernetes/certs
 ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}

magnum/drivers/common/templates/kubernetes/fragments/start-container-agent.sh

@@ -51,7 +51,8 @@ systemctl restart sshd
 
 _prefix="${CONTAINER_INFRA_PREFIX:-docker.io/openstackmagnum/}"
 
-cat > /etc/systemd/system/heat-container-agent.service <<EOF
+if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+    cat > /etc/systemd/system/heat-container-agent.service <<EOF
 [Unit]
 Description=Run heat-container-agent
 After=network-online.target
@@ -87,6 +88,15 @@ ExecStop=/bin/podman stop heat-container-agent
 [Install]
 WantedBy=multi-user.target
 EOF
+else
+    atomic install \
+    --storage ostree \
+    --system \
+    --system-package no \
+    --set REQUESTS_CA_BUNDLE=/etc/pki/tls/certs/ca-bundle.crt \
+    --name heat-container-agent \
+    "${_prefix}heat-container-agent:${HEAT_CONTAINER_AGENT_TAG}"
+fi
 
 systemctl enable heat-container-agent
 systemctl start heat-container-agent

magnum/drivers/common/templates/kubernetes/fragments/upgrade-kubernetes.sh

@@ -5,10 +5,14 @@ set -x
 
 ssh_cmd="ssh -F /srv/magnum/.ssh/config root@localhost"
 KUBECONFIG="/etc/kubernetes/kubelet-config.yaml"
+if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+    kubecontrol="/var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig $KUBECONFIG"
+else
+    kubecontrol="/usr/local/bin/kubectl --kubeconfig $KUBECONFIG"
+fi
 new_kube_tag="$kube_tag_input"
 new_ostree_remote="$ostree_remote_input"
 new_ostree_commit="$ostree_commit_input"
-HOSTNAME_OVERRIDE="$(cat /etc/hostname | head -1 | sed 's/\.novalocal//')"
 
 function drain {
     # If there is only one master and this is the master node, skip the drain, just cordon it
@@ -26,19 +30,50 @@ if [ "${new_kube_tag}" != "${KUBE_TAG}" ]; then
 
     drain
 
-    SERVICE_LIST=$($ssh_cmd podman ps -f name=kube --format {{.Names}})
+    if [ "$(echo $USE_PODMAN | tr '[:upper:]' '[:lower:]')" == "true" ]; then
+        SERVICE_LIST=$($ssh_cmd podman ps -f name=kube --format {{.Names}})
 
-    for service in ${SERVICE_LIST}; do
-        ${ssh_cmd} systemctl stop ${service}
-        ${ssh_cmd} podman rm ${service}
-    done
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} systemctl stop ${service}
+            ${ssh_cmd} podman rm ${service}
+        done
 
-    ${ssh_cmd} podman rmi ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:${KUBE_TAG}
-    echo "KUBE_TAG=$new_kube_tag" >> /etc/sysconfig/heat-params
+        ${ssh_cmd} podman rmi ${CONTAINER_INFRA_PREFIX:-k8s.gcr.io/}hyperkube:${KUBE_TAG}
+        echo "KUBE_TAG=$new_kube_tag" >> /etc/sysconfig/heat-params
 
-    for service in ${SERVICE_LIST}; do
-        ${ssh_cmd} systemctl start ${service}
-    done
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} systemctl start ${service}
+        done
+    else
+        declare -A service_image_mapping
+        service_image_mapping=( ["kubelet"]="kubernetes-kubelet" ["kube-controller-manager"]="kubernetes-controller-manager" ["kube-scheduler"]="kubernetes-scheduler" ["kube-proxy"]="kubernetes-proxy" ["kube-apiserver"]="kubernetes-apiserver" )
+
+        SERVICE_LIST=$($ssh_cmd atomic containers list -f container=kube -q --no-trunc)
+
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} systemctl stop ${service}
+        done
+
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} atomic pull --storage ostree "docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag}"
+        done
+
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} atomic containers update --rebase docker.io/openstackmagnum/${service_image_mapping[${service}]}:${new_kube_tag} ${service}
+        done
+
+        for service in ${SERVICE_LIST}; do
+            systemctl restart ${service}
+        done
+
+        ${ssh_cmd} /var/lib/containers/atomic/heat-container-agent.0/rootfs/usr/bin/kubectl --kubeconfig /etc/kubernetes/kubelet-config.yaml uncordon ${INSTANCE_NAME}
+
+        for service in ${SERVICE_LIST}; do
+            ${ssh_cmd} atomic --assumeyes images "delete docker.io/openstackmagnum/${service_image_mapping[${service}]}:${KUBE_TAG}"
+        done
+
+        ${ssh_cmd} atomic images prune
+    fi
 
     i=0
     until kubectl uncordon ${INSTANCE_NAME}
@@ -61,7 +96,7 @@ After=network.target kubelet.service
 [Service]
 Restart=Always
 RemainAfterExit=yes
-ExecStart=${kubecontrol} uncordon ${HOSTNAME_OVERRIDE}
+ExecStart=${kubecontrol} uncordon ${INSTANCE_NAME}
 
 [Install]
 WantedBy=multi-user.target

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params-master.sh

@@ -108,6 +108,7 @@ HEAT_PARAMS=/etc/sysconfig/heat-params
       NPD_ENABLED="$NPD_ENABLED"
       NODEGROUP_ROLE="$NODEGROUP_ROLE"
       NODEGROUP_NAME="$NODEGROUP_NAME"
+      USE_PODMAN="$USE_PODMAN"
 EOF
 }
 

magnum/drivers/common/templates/kubernetes/fragments/write-heat-params.sh

@@ -60,6 +60,7 @@ AUTO_HEALING_ENABLED="$AUTO_HEALING_ENABLED"
 AUTO_HEALING_CONTROLLER="$AUTO_HEALING_CONTROLLER"
 NODEGROUP_ROLE="$NODEGROUP_ROLE"
 NODEGROUP_NAME="$NODEGROUP_NAME"
+USE_PODMAN="$USE_PODMAN"
 EOF
 }
 

magnum/drivers/heat/k8s_fedora_template_def.py

@@ -99,7 +99,8 @@ class K8sFedoraTemplateDefinition(k8s_template_def.K8sTemplateDefinition):
                       'auto_healing_controller', 'magnum_auto_healer_tag',
                       'draino_tag', 'autoscaler_tag',
                       'min_node_count', 'max_node_count', 'npd_enabled',
-                      'ostree_remote', 'ostree_commit']
+                      'ostree_remote', 'ostree_commit',
+                      'use_podman']
 
         labels = self._get_relevant_labels(cluster, kwargs)
 

magnum/drivers/k8s_fedora_atomic_v1/templates/kubecluster.yaml

@@ -455,7 +455,7 @@ parameters:
   etcd_tag:
     type: string
     description: tag of the etcd system container
-    default: 3.2.26
+    default: v3.2.7
 
   coredns_tag:
     type: string
@@ -765,6 +765,13 @@ parameters:
     description: The ostree commit to deploy
     default: ''
 
+  use_podman:
+    type: boolean
+    description: >
+      if true, run system containers for kubernetes, etcd and heat-agent
+    default:
+      false
+
 resources:
 
   ######################################################################
@@ -1081,6 +1088,7 @@ resources:
           npd_enabled: {get_param: npd_enabled}
           ostree_remote: {get_param: ostree_remote}
           ostree_commit: {get_param: ostree_commit}
+          use_podman: {get_param: use_podman}
 
   kube_cluster_config:
     condition: create_cluster_resources
@@ -1246,6 +1254,7 @@ resources:
           auto_healing_controller: {get_param: auto_healing_controller}
           ostree_remote: {get_param: ostree_remote}
           ostree_commit: {get_param: ostree_commit}
+          use_podman: {get_param: use_podman}
 
 outputs:
 

magnum/drivers/k8s_fedora_atomic_v1/templates/kubemaster.yaml

@@ -538,6 +538,11 @@ parameters:
     type: string
     description: The ostree commit to deploy
 
+  use_podman:
+    type: boolean
+    description: >
+      if true, run system containers for kubernetes, etcd and heat-agent
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -586,6 +591,7 @@ resources:
                 params:
                   $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
                   $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
+                  $USE_PODMAN: {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh
 
   master_config:
@@ -699,6 +705,7 @@ resources:
                   "$NPD_ENABLED": {get_param: npd_enabled}
                   "$NODEGROUP_ROLE": {get_param: nodegroup_role}
                   "$NODEGROUP_NAME": {get_param: nodegroup_name}
+                  "$USE_PODMAN": {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
             - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh
             - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh

magnum/drivers/k8s_fedora_atomic_v1/templates/kubeminion.yaml

@@ -322,6 +322,11 @@ parameters:
     description: The ostree commit to deploy
     default: ''
 
+  use_podman:
+    type: boolean
+    description: >
+      if true, run system containers for kubernetes, etcd and heat-agent
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -351,6 +356,7 @@ resources:
                 params:
                   $CONTAINER_INFRA_PREFIX: {get_param: container_infra_prefix}
                   $HEAT_CONTAINER_AGENT_TAG: {get_param: heat_container_agent_tag}
+                  $USE_PODMAN: {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/disable-selinux.sh
 
   ######################################################################
@@ -422,6 +428,7 @@ resources:
                   $NPD_ENABLED: {get_param: npd_enabled}
                   $NODEGROUP_ROLE: {get_param: nodegroup_role}
                   $NODEGROUP_NAME: {get_param: nodegroup_name}
+                  $USE_PODMAN: {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
             - get_file: ../../common/templates/fragments/configure-docker-registry.sh

magnum/drivers/k8s_fedora_coreos_v1/templates/kubecluster.yaml

@@ -755,6 +755,25 @@ parameters:
     default:
       true
 
+  ostree_remote:
+    type: string
+    description: This parameter is ignored for k8s_fedora_coreos.
+    default: ''
+
+  ostree_commit:
+    type: string
+    description: This parameter is ignored for k8s_fedora_coreos.
+    default: ''
+
+  use_podman:
+    type: boolean
+    description: >
+      If true, run system containers for kubernetes, etcd and heat-agent
+    default:
+      true
+    constraints:
+      - allowed_values: [true]
+
 resources:
 
   ######################################################################
@@ -1070,6 +1089,9 @@ resources:
           min_node_count: {get_param: min_node_count}
           max_node_count: {get_param: max_node_count}
           npd_enabled: {get_param: npd_enabled}
+          ostree_remote: {get_param: ostree_remote}
+          ostree_commit: {get_param: ostree_commit}
+          use_podman: {get_param: use_podman}
 
   kube_cluster_config:
     condition: create_cluster_resources
@@ -1234,6 +1256,9 @@ resources:
           auto_healing_enabled: {get_param: auto_healing_enabled}
           npd_enabled: {get_param: npd_enabled}
           auto_healing_controller: {get_param: auto_healing_controller}
+          ostree_remote: {get_param: ostree_remote}
+          ostree_commit: {get_param: ostree_commit}
+          use_podman: {get_param: use_podman}
 
 outputs:
 

magnum/drivers/k8s_fedora_coreos_v1/templates/kubemaster.yaml

@@ -534,6 +534,19 @@ parameters:
     default:
       true
 
+  ostree_remote:
+    type: string
+    description: The ostree remote branch to upgrade
+
+  ostree_commit:
+    type: string
+    description: The ostree commit to deploy
+
+  use_podman:
+    type: boolean
+    description: >
+      If true, run system containers for kubernetes, etcd and heat-agent
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -690,6 +703,7 @@ resources:
                   "$NPD_ENABLED": {get_param: npd_enabled}
                   "$NODEGROUP_ROLE": {get_param: nodegroup_role}
                   "$NODEGROUP_NAME": {get_param: nodegroup_name}
+                  "$USE_PODMAN": {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/make-cert.sh
             - get_file: ../../common/templates/kubernetes/fragments/configure-etcd.sh
             - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
@@ -834,6 +848,8 @@ resources:
       group: script
       inputs:
       - name: kube_tag_input
+      - name: ostree_remote_input
+      - name: ostree_commit_input
       config:
         get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
 
@@ -846,6 +862,8 @@ resources:
       actions: ['UPDATE']
       input_values:
         kube_tag_input: {get_param: kube_tag}
+        ostree_remote_input: {get_param: ostree_remote}
+        ostree_commit_input: {get_param: ostree_commit}
 
 outputs:
 

magnum/drivers/k8s_fedora_coreos_v1/templates/kubeminion.yaml

@@ -316,6 +316,19 @@ parameters:
     default:
       true
 
+  ostree_remote:
+    type: string
+    description: The ostree remote branch to upgrade
+
+  ostree_commit:
+    type: string
+    description: The ostree commit to deploy
+
+  use_podman:
+    type: boolean
+    description: >
+      If true, run system containers for kubernetes, etcd and heat-agent
+
 conditions:
 
   image_based: {equals: [{get_param: boot_volume_size}, 0]}
@@ -411,6 +424,7 @@ resources:
                   $NPD_ENABLED: {get_param: npd_enabled}
                   $NODEGROUP_ROLE: {get_param: nodegroup_role}
                   $NODEGROUP_NAME: {get_param: nodegroup_name}
+                  $USE_PODMAN: {get_param: use_podman}
             - get_file: ../../common/templates/kubernetes/fragments/write-kube-os-config.sh
             - get_file: ../../common/templates/kubernetes/fragments/make-cert-client.sh
             - get_file: ../../common/templates/fragments/configure-docker-registry.sh
@@ -521,6 +535,8 @@ resources:
       group: script
       inputs:
       - name: kube_tag_input
+      - name: ostree_remote_input
+      - name: ostree_commit_input
       config:
         get_file: ../../common/templates/kubernetes/fragments/upgrade-kubernetes.sh
 
@@ -533,6 +549,8 @@ resources:
       actions: ['UPDATE']
       input_values:
         kube_tag_input: {get_param: kube_tag}
+        ostree_remote_input: {get_param: ostree_remote}
+        ostree_commit_input: {get_param: ostree_commit}
 
 outputs:
 

magnum/tests/unit/drivers/test_template_definition.py

@@ -572,6 +572,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         etcd_volume_type = mock_cluster.labels.get('etcd_volume_type')
         ostree_remote = mock_cluster.labels.get('ostree_remote')
         ostree_commit = mock_cluster.labels.get('ostree_commit')
+        use_podman = mock_cluster.labels.get('use_podman')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -658,6 +659,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'etcd_volume_type': etcd_volume_type,
             'ostree_remote': ostree_remote,
             'ostree_commit': ostree_commit,
+            'use_podman': use_podman,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,
@@ -1012,6 +1014,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
         etcd_volume_type = mock_cluster.labels.get('etcd_volume_type')
         ostree_remote = mock_cluster.labels.get('ostree_remote')
         ostree_commit = mock_cluster.labels.get('ostree_commit')
+        use_podman = mock_cluster.labels.get('use_podman')
 
         k8s_def = k8sa_tdef.AtomicK8sTemplateDefinition()
 
@@ -1100,6 +1103,7 @@ class AtomicK8sTemplateDefinitionTestCase(BaseK8sTemplateDefinitionTestCase):
             'etcd_volume_type': etcd_volume_type,
             'ostree_remote': ostree_remote,
             'ostree_commit': ostree_commit,
+            'use_podman': use_podman,
         }}
         mock_get_params.assert_called_once_with(mock_context,
                                                 mock_cluster_template,

releasenotes/notes/use_podman-39532143be2296c2.yaml

@@ -0,0 +1,17 @@
+---
+features:
+  - |
+    Choose whether system containers etcd, kubernetes and the heat-agent will
+    be installed with podman or atomic. This label is relevant for
+    k8s_fedora drivers.
+
+    k8s_fedora_atomic_v1 defaults to use_podman=false, meaning atomic will be
+    used pulling containers from docker.io/openstackmagnum. use_podman=true
+    is accepted as well, which will pull containers by k8s.gcr.io.
+
+    k8s_fedora_coreos_v1 defaults and accepts only use_podman=true.
+
+    Note that, to use kubernetes version greater or equal to v1.16.0 with the
+    k8s_fedora_atomic_v1 driver, you need to set use_podman=true. This is
+    necessary since v1.16 dropped the --containerized flag in kubelet.
+    https://github.com/kubernetes/kubernetes/pull/80043/files