diff --git a/.zuul.yaml b/.zuul.yaml index d48580594..abb19333f 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -17,20 +17,20 @@ jobs: - kuryr-kubernetes-tempest-octavia - kuryr-kubernetes-tempest-daemon-octavia - - kuryr-kubernetes-tempest-openshift-octavia + - kuryr-kubernetes-tempest-daemon-openshift-octavia - kuryr-kubernetes-tempest-lbaasv2 - kuryr-kubernetes-tempest-daemon-lbaasv2 - - kuryr-kubernetes-tempest-openshift-lbaasv2 + - kuryr-kubernetes-tempest-daemon-openshift-lbaasv2 - kuryr-kubernetes-tempest-dragonflow - kuryr-kubernetes-tempest-ovn gate: jobs: - kuryr-kubernetes-tempest-octavia + - kuryr-kubernetes-tempest-daemon-octavia - kuryr-kubernetes-tempest-lbaasv2 + - kuryr-kubernetes-tempest-daemon-lbaasv2 experimental: jobs: - - kuryr-kubernetes-tempest-daemon-openshift-octavia - - kuryr-kubernetes-tempest-daemon-openshift-lbaasv2 - kuryr-kubernetes-tempest-multinode-octavia - kuryr-kubernetes-tempest-multinode-lbaasv2 - kuryr-kubernetes-tempest-octavia-centos-7 diff --git a/cni.Dockerfile b/cni.Dockerfile index 55985345a..f83a5935a 100644 --- a/cni.Dockerfile +++ b/cni.Dockerfile @@ -21,7 +21,7 @@ ARG CNI_CONFIG_DIR_PATH=/etc/cni/net.d ENV CNI_CONFIG_DIR_PATH ${CNI_CONFIG_DIR_PATH} ARG CNI_BIN_DIR_PATH=/opt/cni/bin ENV CNI_BIN_DIR_PATH ${CNI_BIN_DIR_PATH} -ARG CNI_DAEMON=False +ARG CNI_DAEMON=True ENV CNI_DAEMON ${CNI_DAEMON} VOLUME [ "/sys/fs/cgroup" ] diff --git a/devstack/local.conf.df.sample b/devstack/local.conf.df.sample index fb79d2839..d496aa971 100644 --- a/devstack/local.conf.df.sample +++ b/devstack/local.conf.df.sample @@ -181,6 +181,17 @@ enable_service kubelet # resource events and convert them to Neutron actions enable_service kuryr-kubernetes +# Kuryr Daemon +# ============ +# +# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI +# driver and the driver will pass requests to Kuryr daemon running on the node, +# instead of processing them on its own. This limits the number of Kubernetes +# API requests (as only Kuryr Daemon will watch for new pod events) and should +# increase scalability in environments that often delete and create pods. +# Since Rocky release this is a default deployment configuration. +enable_service kuryr-daemon + # Kuryr POD VIF Driver # ==================== # diff --git a/devstack/local.conf.odl.sample b/devstack/local.conf.odl.sample index 2482357d4..01210c501 100644 --- a/devstack/local.conf.odl.sample +++ b/devstack/local.conf.odl.sample @@ -145,6 +145,17 @@ enable_service kubelet # resource events and convert them to Neutron actions enable_service kuryr-kubernetes +# Kuryr Daemon +# ============ +# +# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI +# driver and the driver will pass requests to Kuryr daemon running on the node, +# instead of processing them on its own. This limits the number of Kubernetes +# API requests (as only Kuryr Daemon will watch for new pod events) and should +# increase scalability in environments that often delete and create pods. +# Since Rocky release this is a default deployment configuration. +enable_service kuryr-daemon + # Kuryr POD VIF Driver # ==================== # diff --git a/devstack/local.conf.openshift.sample b/devstack/local.conf.openshift.sample index 35e1b723f..cf51008b6 100644 --- a/devstack/local.conf.openshift.sample +++ b/devstack/local.conf.openshift.sample @@ -154,6 +154,17 @@ DOCKER_CGROUP_DRIVER="systemd" # resource events and convert them to Neutron actions enable_service kuryr-kubernetes +# Kuryr Daemon +# ============ +# +# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI +# driver and the driver will pass requests to Kuryr daemon running on the node, +# instead of processing them on its own. This limits the number of Kubernetes +# API requests (as only Kuryr Daemon will watch for new pod events) and should +# increase scalability in environments that often delete and create pods. +# Since Rocky release this is a default deployment configuration. +enable_service kuryr-daemon + # Containerized Kuryr # =================== # diff --git a/devstack/local.conf.ovn.sample b/devstack/local.conf.ovn.sample index 30000f0e7..fd56e3bf8 100644 --- a/devstack/local.conf.ovn.sample +++ b/devstack/local.conf.ovn.sample @@ -191,7 +191,7 @@ enable_service kuryr-kubernetes # instead of processing them on its own. This limits the number of Kubernetes # API requests (as only Kuryr Daemon will watch for new pod events) and should # increase scalability in environments that often delete and create pods. -# To enable kuryr-daemon uncomment next line. +# Since Rocky release this is a default deployment configuration. enable_service kuryr-daemon diff --git a/devstack/local.conf.pod-in-vm.overcloud.sample b/devstack/local.conf.pod-in-vm.overcloud.sample index 8889b37ce..47396b53d 100644 --- a/devstack/local.conf.pod-in-vm.overcloud.sample +++ b/devstack/local.conf.pod-in-vm.overcloud.sample @@ -38,6 +38,7 @@ enable_service kubernetes-controller-manager enable_service kubernetes-scheduler enable_service kubelet enable_service kuryr-kubernetes +enable_service kuryr-daemon KURYR_POD_VIF_DRIVER=nested-vlan diff --git a/devstack/local.conf.sample b/devstack/local.conf.sample index 2004e04a3..8f62099d8 100644 --- a/devstack/local.conf.sample +++ b/devstack/local.conf.sample @@ -182,13 +182,13 @@ enable_service kuryr-kubernetes # Kuryr Daemon # ============ # -# Kuryr can run CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI +# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI # driver and the driver will pass requests to Kuryr daemon running on the node, # instead of processing them on its own. This limits the number of Kubernetes # API requests (as only Kuryr Daemon will watch for new pod events) and should # increase scalability in environments that often delete and create pods. -# To enable kuryr-daemon uncomment next line. -# enable_service kuryr-daemon +# Since Rocky release this is a default deployment configuration. +enable_service kuryr-daemon # Containerized Kuryr diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 7e985cf6d..4257f7896 100644 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -82,7 +82,6 @@ function configure_kuryr { fi if is_service_enabled kuryr-daemon; then - iniset "$KURYR_CONFIG" cni_daemon daemon_enabled True iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR" create_kuryr_lock_dir if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then @@ -91,6 +90,8 @@ function configure_kuryr { iniset "$KURYR_CONFIG" cni_daemon docker_mode True iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc" fi + else + iniset "$KURYR_CONFIG" cni_daemon daemon_enabled False fi create_kuryr_cache_dir diff --git a/doc/source/devref/kuryr_kubernetes_design.rst b/doc/source/devref/kuryr_kubernetes_design.rst index 18b0da084..69df528e9 100644 --- a/doc/source/devref/kuryr_kubernetes_design.rst +++ b/doc/source/devref/kuryr_kubernetes_design.rst @@ -165,45 +165,19 @@ CNI driver to complete pod handling. The NeutronPodVifDriver is the default driver that creates neutron port upon Pod addition and deletes port upon Pod removal. -CNI Driver ----------- -Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin `_ -and introduces Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes -CNI Driver should get all information required to plug and bind Pod via -kubernetes control plane and should not depend on Neutron. CNI plugin/driver -is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore it is -expected to return when either success or error state determined. - -Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node -environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the -contract that defines Pod annotation that Controller Server adds and CNI -driver reads. The contract is `os_vif VIF `_ - -With VIF object loaded from the Pod object annotation, the CNI driver performs -Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and -unplug operations. The CNI driver should complete its job and return control to -Kubelet when all the network plugging is completed. -In the cases when Neutron initially creates port in 'Down' state, CNI driver -will plug the Pod, but will have to watch the Pod annotations for vif state -change to 'Active' before returning the control to the caller. - -.. image:: ../../images/pod_creation_flow.png - :alt: Controller-CNI interaction - :align: center - :width: 100% - .. _cni-daemon: CNI Daemon ---------- -CNI Daemon is an optional service that should run on every Kubernetes node. It -is responsible for watching pod events on the node it's running on, answering -calls from CNI Driver and attaching VIFs when they are ready. In the future -it will also keep information about pooled ports in memory. This helps to limit -the number of processes spawned when creating multiple Pods, as a single -Watcher is enough for each node and CNI Driver will only wait on local network -socket for response from the Daemon. +CNI Daemon is a service that should run on every Kubernetes node. Starting from +Rocky release it should be seen as a default supported deployment option. +It is responsible for watching pod events on the node it's running on, +answering calls from CNI Driver and attaching VIFs when they are ready. In the +future it will also keep information about pooled ports in memory. This helps +to limit the number of processes spawned when creating multiple Pods, as a +single Watcher is enough for each node and CNI Driver will only wait on local +network socket for response from the Daemon. Currently CNI Daemon consists of two processes i.e. Watcher and Server. Processes communicate between each other using Python's @@ -252,6 +226,44 @@ deserialized using o.vo's ``obj_from_primitive()`` method. When running in daemonized mode, CNI Driver will call CNI Daemon over those APIs to perform its tasks and wait on socket for result. +CNI Driver (deprecated) +----------------------- + +.. warning:: + Running with CNI Driver in this mode is deprecated since Rocky release. + Currently the preferred way of deploying kuryr-kubernetes is with + kuryr-daemon that takes over most of the CNI Driver tasks. In that case CNI + driver becomes a thin client that passes CNI ADD and DEL requests to + kuryr-daemon instance via its HTTP API. + +Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin +`_ and introduces +Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes +CNI Driver should get all information required to plug and bind Pod via +kubernetes control plane and should not depend on Neutron. CNI plugin/driver +is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore +it is expected to return when either success or error state determined. + +Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node +environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the +contract that defines Pod annotation that Controller Server adds and CNI +driver reads. The contract is `os_vif VIF +`_ + +With VIF object loaded from the Pod object annotation, the CNI driver performs +Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and +unplug operations. The CNI driver should complete its job and return control to +Kubelet when all the network plugging is completed. +In the cases when Neutron initially creates port in 'Down' state, CNI driver +will plug the Pod, but will have to watch the Pod annotations for vif state +change to 'Active' before returning the control to the caller. + +.. image:: ../../images/pod_creation_flow.png + :alt: Controller-CNI interaction + :align: center + :width: 100% + + Kubernetes Documentation ------------------------ The `Kubernetes reference documentation `_ diff --git a/doc/source/installation/devstack/basic.rst b/doc/source/installation/devstack/basic.rst index 245717177..8f487f28d 100644 --- a/doc/source/installation/devstack/basic.rst +++ b/doc/source/installation/devstack/basic.rst @@ -48,8 +48,9 @@ Now edit ``devstack/local.conf`` to set up some initial options: omitted. * If you already have Docker installed on the machine, you can comment out line starting with ``enable_plugin devstack-plugin-container``. -* If you want to enable kuryr-daemon uncomment ``enable_service kuryr-daemon`` - line. +* If you want to disable kuryr-daemon add ``disable_service kuryr-daemon`` + line. Please note that running without kuryr-daemon was deprecated in Rocky + release. Once ``local.conf`` is configured, you can start the installation: :: diff --git a/doc/source/installation/manual.rst b/doc/source/installation/manual.rst index 5c8bd9d70..9709f01f1 100644 --- a/doc/source/installation/manual.rst +++ b/doc/source/installation/manual.rst @@ -157,20 +157,15 @@ to work correctly:: deactivate sudo pip install 'oslo.privsep>=1.20.0' 'os-vif>=1.5.0' -Configure Kuryr CNI Daemon (optional) +Configure Kuryr CNI Daemon ------------------------------------- -Kuryr CNI Daemon is an optional service designed to increased scalability of -the Kuryr operations done on Kubernetes nodes. More information can be found on +Kuryr CNI Daemon is a service designed to increased scalability of the Kuryr +operations done on Kubernetes nodes. More information can be found on :ref:`cni-daemon` page. -If you want to use Kuryr CNI Daemon, it needs to be installed on every -Kubernetes node, so following steps need to be repeated. - -Edit ``kuryr.conf``:: - - [cni_daemon] - daemon_enabled=True +Kuryr CNI Daemon, should be installed on every Kubernetes node, so following +steps need to be repeated. .. note:: You can tweak configuration of some timeouts to match your environment. It's diff --git a/kuryr_kubernetes/cni/main.py b/kuryr_kubernetes/cni/main.py index 7989378a6..8554d4bbb 100644 --- a/kuryr_kubernetes/cni/main.py +++ b/kuryr_kubernetes/cni/main.py @@ -20,6 +20,7 @@ import sys import os_vif from oslo_config import cfg from oslo_log import log as logging +from oslo_log import versionutils from oslo_serialization import jsonutils from kuryr_kubernetes.cni import api as cni_api @@ -56,6 +57,13 @@ def run(): if CONF.cni_daemon.daemon_enabled: runner = cni_api.CNIDaemonizedRunner() else: + # TODO(dulek): Switch that to versionutils.deprecation_warning once + # bug 1754087 is fixed. + versionutils.report_deprecated_feature( + LOG, + 'Deploying kuryr-kubernetes without kuryr-daemon service is ' + 'deprecated since Rocky release and may be removed in future ' + 'releases.') runner = cni_api.CNIStandaloneRunner(k8s_cni.K8sCNIPlugin()) LOG.info("Using '%s' ", runner.__class__.__name__) diff --git a/kuryr_kubernetes/config.py b/kuryr_kubernetes/config.py index 215aa96bf..6095e9c2a 100644 --- a/kuryr_kubernetes/config.py +++ b/kuryr_kubernetes/config.py @@ -33,7 +33,11 @@ kuryr_k8s_opts = [ daemon_opts = [ cfg.BoolOpt('daemon_enabled', help=_('Enable CNI Daemon configuration.'), - default=False), + default=True, + deprecated_for_removal=True, + deprecated_reason="Deployment without kuryr-daemon is now " + "deprecated.", + deprecated_since="Rocky"), cfg.StrOpt('bind_address', help=_('Bind address for CNI daemon HTTP server. It is ' 'recommened to allow only local connections.'), diff --git a/releasenotes/notes/deprecate-non-daemonized-6dd2154238b1628c.yaml b/releasenotes/notes/deprecate-non-daemonized-6dd2154238b1628c.yaml new file mode 100644 index 000000000..1888863bd --- /dev/null +++ b/releasenotes/notes/deprecate-non-daemonized-6dd2154238b1628c.yaml @@ -0,0 +1,18 @@ +--- +upgrade: + - | + Legacy Kuryr deployment without running kuryr-daemon is now considered + deprecated. That possibility will be completely removed in one of the next + releases. Please note that this means that ``[cni_daemon]daemon_enabled`` + option will default to ``True``. +deprecations: + - | + Running Kuryr-Kubernetes without kuryr-daemon service is now deprecated. + Motivations for that move include: + + * Discoveries of bugs that are much easier to fix in kuryr-daemon. + * Further improvements in Kuryr scalability (e.g. moving choosing VIF from + pool into kuryr-daemon) are only possible when kuryr-daemon is present. + + Possibility of running Kuryr-Kubernetes without kuryr-daemon will be + removed in one of the future releases. diff --git a/tools/build_cni_daemonset_image b/tools/build_cni_daemonset_image index 9157eb7b6..033b88b89 100755 --- a/tools/build_cni_daemonset_image +++ b/tools/build_cni_daemonset_image @@ -2,7 +2,7 @@ CNI_BIN_DIR=$1 CNI_CONF_DIR=$2 -CNI_DAEMON=${3:-"False"} +CNI_DAEMON=${3:-"True"} CNI_TAG="kuryr/cni" # create cni daemonset image