Deprecate running kuryr-k8s without kuryr-daemon

This commit implements what was discussed on the PTG, i.e. deprecation
of running Kuryr-Kubernetes without kuryr-daemon services. This commit
includes changes in configuration defaults, sample local.conf files,
documentation, gates and a release note explaining the change.

Change-Id: I152c81797cb83237af4917a4487cb1f1918270aa

.zuul.yaml

@@ -17,20 +17,20 @@
       jobs:
         - kuryr-kubernetes-tempest-octavia
         - kuryr-kubernetes-tempest-daemon-octavia
-        - kuryr-kubernetes-tempest-openshift-octavia
+        - kuryr-kubernetes-tempest-daemon-openshift-octavia
         - kuryr-kubernetes-tempest-lbaasv2
         - kuryr-kubernetes-tempest-daemon-lbaasv2
-        - kuryr-kubernetes-tempest-openshift-lbaasv2
+        - kuryr-kubernetes-tempest-daemon-openshift-lbaasv2
         - kuryr-kubernetes-tempest-dragonflow
         - kuryr-kubernetes-tempest-ovn
     gate:
       jobs:
         - kuryr-kubernetes-tempest-octavia
+        - kuryr-kubernetes-tempest-daemon-octavia
         - kuryr-kubernetes-tempest-lbaasv2
+        - kuryr-kubernetes-tempest-daemon-lbaasv2
     experimental:
       jobs:
-        - kuryr-kubernetes-tempest-daemon-openshift-octavia
-        - kuryr-kubernetes-tempest-daemon-openshift-lbaasv2
         - kuryr-kubernetes-tempest-multinode-octavia
         - kuryr-kubernetes-tempest-multinode-lbaasv2
         - kuryr-kubernetes-tempest-octavia-centos-7

cni.Dockerfile

@@ -21,7 +21,7 @@ ARG CNI_CONFIG_DIR_PATH=/etc/cni/net.d
 ENV CNI_CONFIG_DIR_PATH ${CNI_CONFIG_DIR_PATH}
 ARG CNI_BIN_DIR_PATH=/opt/cni/bin
 ENV CNI_BIN_DIR_PATH ${CNI_BIN_DIR_PATH}
-ARG CNI_DAEMON=False
+ARG CNI_DAEMON=True
 ENV CNI_DAEMON ${CNI_DAEMON}
 
 VOLUME [ "/sys/fs/cgroup" ]

devstack/local.conf.df.sample

@@ -181,6 +181,17 @@ enable_service kubelet
 # resource events and convert them to Neutron actions
 enable_service kuryr-kubernetes
 
+# Kuryr Daemon
+# ============
+#
+# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
+# driver and the driver will pass requests to Kuryr daemon running on the node,
+# instead of processing them on its own. This limits the number of Kubernetes
+# API requests (as only Kuryr Daemon will watch for new pod events) and should
+# increase scalability in environments that often delete and create pods.
+# Since Rocky release this is a default deployment configuration.
+enable_service kuryr-daemon
+
 # Kuryr POD VIF Driver
 # ====================
 #

devstack/local.conf.odl.sample

@@ -145,6 +145,17 @@ enable_service kubelet
 # resource events and convert them to Neutron actions
 enable_service kuryr-kubernetes
 
+# Kuryr Daemon
+# ============
+#
+# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
+# driver and the driver will pass requests to Kuryr daemon running on the node,
+# instead of processing them on its own. This limits the number of Kubernetes
+# API requests (as only Kuryr Daemon will watch for new pod events) and should
+# increase scalability in environments that often delete and create pods.
+# Since Rocky release this is a default deployment configuration.
+enable_service kuryr-daemon
+
 # Kuryr POD VIF Driver
 # ====================
 #

devstack/local.conf.openshift.sample

@@ -154,6 +154,17 @@ DOCKER_CGROUP_DRIVER="systemd"
 # resource events and convert them to Neutron actions
 enable_service kuryr-kubernetes
 
+# Kuryr Daemon
+# ============
+#
+# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
+# driver and the driver will pass requests to Kuryr daemon running on the node,
+# instead of processing them on its own. This limits the number of Kubernetes
+# API requests (as only Kuryr Daemon will watch for new pod events) and should
+# increase scalability in environments that often delete and create pods.
+# Since Rocky release this is a default deployment configuration.
+enable_service kuryr-daemon
+
 # Containerized Kuryr
 # ===================
 #

devstack/local.conf.ovn.sample

@@ -191,7 +191,7 @@ enable_service kuryr-kubernetes
 # instead of processing them on its own. This limits the number of Kubernetes
 # API requests (as only Kuryr Daemon will watch for new pod events) and should
 # increase scalability in environments that often delete and create pods.
-# To enable kuryr-daemon uncomment next line.
+# Since Rocky release this is a default deployment configuration.
 enable_service kuryr-daemon
 
 

devstack/local.conf.pod-in-vm.overcloud.sample

@@ -38,6 +38,7 @@ enable_service kubernetes-controller-manager
 enable_service kubernetes-scheduler
 enable_service kubelet
 enable_service kuryr-kubernetes
+enable_service kuryr-daemon
 
 KURYR_POD_VIF_DRIVER=nested-vlan
 

devstack/local.conf.sample

@@ -182,13 +182,13 @@ enable_service kuryr-kubernetes
 # Kuryr Daemon
 # ============
 #
-# Kuryr can run CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
+# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
 # driver and the driver will pass requests to Kuryr daemon running on the node,
 # instead of processing them on its own. This limits the number of Kubernetes
 # API requests (as only Kuryr Daemon will watch for new pod events) and should
 # increase scalability in environments that often delete and create pods.
-# To enable kuryr-daemon uncomment next line.
+# Since Rocky release this is a default deployment configuration.
-# enable_service kuryr-daemon
+enable_service kuryr-daemon
 
 
 # Containerized Kuryr

devstack/plugin.sh

@@ -82,7 +82,6 @@ function configure_kuryr {
     fi
 
     if is_service_enabled kuryr-daemon; then
-        iniset "$KURYR_CONFIG" cni_daemon daemon_enabled True
         iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR"
         create_kuryr_lock_dir
         if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
@@ -91,6 +90,8 @@ function configure_kuryr {
             iniset "$KURYR_CONFIG" cni_daemon docker_mode True
             iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc"
         fi
+    else
+        iniset "$KURYR_CONFIG" cni_daemon daemon_enabled False
     fi
 
     create_kuryr_cache_dir

doc/source/devref/kuryr_kubernetes_design.rst

@@ -165,45 +165,19 @@ CNI driver to complete pod handling.
 The NeutronPodVifDriver is the default driver that creates neutron port upon
 Pod addition and deletes port upon Pod removal.
 
-CNI Driver
-----------
-Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin <http://kubernetes.io/docs/admin/network-plugins/#cni>`_
-and introduces Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes
-CNI Driver should get all information required to plug and bind Pod via
-kubernetes control plane and should not depend on Neutron. CNI plugin/driver
-is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore it is
-expected to return when either success or error state determined.
-
-Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node
-environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the
-contract that defines Pod annotation that Controller Server adds and CNI
-driver reads. The contract is `os_vif VIF <https://github.com/openstack/os-vif/blob/master/os_vif/objects/vif.py>`_
-
-With VIF object loaded from the Pod object annotation, the CNI driver performs
-Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and
-unplug operations. The CNI driver should complete its job and return control to
-Kubelet when all the network plugging is completed.
-In the cases when Neutron initially creates port in 'Down' state, CNI driver
-will plug the Pod, but will have to watch the Pod annotations for vif state
-change to 'Active' before returning the control to the caller.
-
-.. image:: ../../images/pod_creation_flow.png
-    :alt: Controller-CNI interaction
-    :align: center
-    :width: 100%
-
 .. _cni-daemon:
 
 CNI Daemon
 ----------
 
-CNI Daemon is an optional service that should run on every Kubernetes node. It
+CNI Daemon is a service that should run on every Kubernetes node. Starting from
-is responsible for watching pod events on the node it's running on, answering
+Rocky release it should be seen as a default supported deployment option.
-calls from CNI Driver and attaching VIFs when they are ready. In the future
+It is responsible for watching pod events on the node it's running on,
-it will also keep information about pooled ports in memory. This helps to limit
+answering calls from CNI Driver and attaching VIFs when they are ready. In the
-the number of processes spawned when creating multiple Pods, as a single
+future it will also keep information about pooled ports in memory. This helps
-Watcher is enough for each node and CNI Driver will only wait on local network
+to limit the number of processes spawned when creating multiple Pods, as a
-socket for response from the Daemon.
+single Watcher is enough for each node and CNI Driver will only wait on local
+network socket for response from the Daemon.
 
 Currently CNI Daemon consists of two processes i.e. Watcher and Server.
 Processes communicate between each other using Python's
@@ -252,6 +226,44 @@ deserialized using o.vo's ``obj_from_primitive()`` method.
 When running in daemonized mode, CNI Driver will call CNI Daemon over those APIs
 to perform its tasks and wait on socket for result.
 
+CNI Driver (deprecated)
+-----------------------
+
+.. warning::
+    Running with CNI Driver in this mode is deprecated since Rocky release.
+    Currently the preferred way of deploying kuryr-kubernetes is with
+    kuryr-daemon that takes over most of the CNI Driver tasks. In that case CNI
+    driver becomes a thin client that passes CNI ADD and DEL requests to
+    kuryr-daemon instance via its HTTP API.
+
+Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin
+<http://kubernetes.io/docs/admin/network-plugins/#cni>`_ and introduces
+Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes
+CNI Driver should get all information required to plug and bind Pod via
+kubernetes control plane and should not depend on Neutron. CNI plugin/driver
+is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore
+it is expected to return when either success or error state determined.
+
+Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node
+environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the
+contract that defines Pod annotation that Controller Server adds and CNI
+driver reads. The contract is `os_vif VIF
+<https://github.com/openstack/os-vif/blob/master/os_vif/objects/vif.py>`_
+
+With VIF object loaded from the Pod object annotation, the CNI driver performs
+Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and
+unplug operations. The CNI driver should complete its job and return control to
+Kubelet when all the network plugging is completed.
+In the cases when Neutron initially creates port in 'Down' state, CNI driver
+will plug the Pod, but will have to watch the Pod annotations for vif state
+change to 'Active' before returning the control to the caller.
+
+.. image:: ../../images/pod_creation_flow.png
+    :alt: Controller-CNI interaction
+    :align: center
+    :width: 100%
+
+
 Kubernetes Documentation
 ------------------------
 The `Kubernetes reference documentation <https://kubernetes.io/docs/reference/>`_

doc/source/installation/devstack/basic.rst

@@ -48,8 +48,9 @@ Now edit ``devstack/local.conf`` to set up some initial options:
   omitted.
 * If you already have Docker installed on the machine, you can comment out line
   starting with ``enable_plugin devstack-plugin-container``.
-* If you want to enable kuryr-daemon uncomment ``enable_service kuryr-daemon``
+* If you want to disable kuryr-daemon add ``disable_service kuryr-daemon``
-  line.
+  line. Please note that running without kuryr-daemon was deprecated in Rocky
+  release.
 
 Once ``local.conf`` is configured, you can start the installation: ::
 

doc/source/installation/manual.rst

@@ -157,20 +157,15 @@ to work correctly::
     deactivate
     sudo pip install 'oslo.privsep>=1.20.0' 'os-vif>=1.5.0'
 
-Configure Kuryr CNI Daemon (optional)
+Configure Kuryr CNI Daemon
 -------------------------------------
 
-Kuryr CNI Daemon is an optional service designed to increased scalability of
+Kuryr CNI Daemon is a service designed to increased scalability of the Kuryr
-the Kuryr operations done on Kubernetes nodes. More information can be found on
+operations done on Kubernetes nodes. More information can be found on
 :ref:`cni-daemon` page.
 
-If you want to use Kuryr CNI Daemon, it needs to be installed on every
+Kuryr CNI Daemon, should be installed on every Kubernetes node, so following
-Kubernetes node, so following steps need to be repeated.
+steps need to be repeated.
-
-Edit ``kuryr.conf``::
-
-    [cni_daemon]
-    daemon_enabled=True
 
 .. note::
   You can tweak configuration of some timeouts to match your environment. It's

kuryr_kubernetes/cni/main.py

@@ -20,6 +20,7 @@ import sys
 import os_vif
 from oslo_config import cfg
 from oslo_log import log as logging
+from oslo_log import versionutils
 from oslo_serialization import jsonutils
 
 from kuryr_kubernetes.cni import api as cni_api
@@ -56,6 +57,13 @@ def run():
     if CONF.cni_daemon.daemon_enabled:
         runner = cni_api.CNIDaemonizedRunner()
     else:
+        # TODO(dulek): Switch that to versionutils.deprecation_warning once
+        #              bug 1754087 is fixed.
+        versionutils.report_deprecated_feature(
+            LOG,
+            'Deploying kuryr-kubernetes without kuryr-daemon service is '
+            'deprecated since Rocky release and may be removed in future '
+            'releases.')
         runner = cni_api.CNIStandaloneRunner(k8s_cni.K8sCNIPlugin())
     LOG.info("Using '%s' ", runner.__class__.__name__)
 

kuryr_kubernetes/config.py

@@ -33,7 +33,11 @@ kuryr_k8s_opts = [
 daemon_opts = [
     cfg.BoolOpt('daemon_enabled',
                 help=_('Enable CNI Daemon configuration.'),
-                default=False),
+                default=True,
+                deprecated_for_removal=True,
+                deprecated_reason="Deployment without kuryr-daemon is now "
+                                  "deprecated.",
+                deprecated_since="Rocky"),
     cfg.StrOpt('bind_address',
                help=_('Bind address for CNI daemon HTTP server. It is '
                       'recommened to allow only local connections.'),

releasenotes/notes/deprecate-non-daemonized-6dd2154238b1628c.yaml

@@ -0,0 +1,18 @@
+---
+upgrade:
+  - |
+    Legacy Kuryr deployment without running kuryr-daemon is now considered
+    deprecated. That possibility will be completely removed in one of the next
+    releases. Please note that this means that ``[cni_daemon]daemon_enabled``
+    option will default to ``True``.
+deprecations:
+  - |
+    Running Kuryr-Kubernetes without kuryr-daemon service is now deprecated.
+    Motivations for that move include:
+
+    * Discoveries of bugs that are much easier to fix in kuryr-daemon.
+    * Further improvements in Kuryr scalability (e.g. moving choosing VIF from
+      pool into kuryr-daemon) are only possible when kuryr-daemon is present.
+
+    Possibility of running Kuryr-Kubernetes without kuryr-daemon will be
+    removed in one of the future releases.

tools/build_cni_daemonset_image

@@ -2,7 +2,7 @@
 
 CNI_BIN_DIR=$1
 CNI_CONF_DIR=$2
-CNI_DAEMON=${3:-"False"}
+CNI_DAEMON=${3:-"True"}
 CNI_TAG="kuryr/cni"
 
 # create cni daemonset image