Deprecate running kuryr-k8s without kuryr-daemon
This commit implements what was discussed on the PTG, i.e. deprecation of running Kuryr-Kubernetes without kuryr-daemon services. This commit includes changes in configuration defaults, sample local.conf files, documentation, gates and a release note explaining the change. Change-Id: I152c81797cb83237af4917a4487cb1f1918270aa
This commit is contained in:
Michał Dulko
parent
c16815d544
commit
898abb4a75
|
|
@ -17,20 +17,20 @@
|
|||
jobs:
|
||||
- kuryr-kubernetes-tempest-octavia
|
||||
- kuryr-kubernetes-tempest-daemon-octavia
|
||||
- kuryr-kubernetes-tempest-openshift-octavia
|
||||
- kuryr-kubernetes-tempest-daemon-openshift-octavia
|
||||
- kuryr-kubernetes-tempest-lbaasv2
|
||||
- kuryr-kubernetes-tempest-daemon-lbaasv2
|
||||
- kuryr-kubernetes-tempest-openshift-lbaasv2
|
||||
- kuryr-kubernetes-tempest-daemon-openshift-lbaasv2
|
||||
- kuryr-kubernetes-tempest-dragonflow
|
||||
- kuryr-kubernetes-tempest-ovn
|
||||
gate:
|
||||
jobs:
|
||||
- kuryr-kubernetes-tempest-octavia
|
||||
- kuryr-kubernetes-tempest-daemon-octavia
|
||||
- kuryr-kubernetes-tempest-lbaasv2
|
||||
- kuryr-kubernetes-tempest-daemon-lbaasv2
|
||||
experimental:
|
||||
jobs:
|
||||
- kuryr-kubernetes-tempest-daemon-openshift-octavia
|
||||
- kuryr-kubernetes-tempest-daemon-openshift-lbaasv2
|
||||
- kuryr-kubernetes-tempest-multinode-octavia
|
||||
- kuryr-kubernetes-tempest-multinode-lbaasv2
|
||||
- kuryr-kubernetes-tempest-octavia-centos-7
|
||||
|
|
|
|||
|
|
@ -21,7 +21,7 @@ ARG CNI_CONFIG_DIR_PATH=/etc/cni/net.d
|
|||
ENV CNI_CONFIG_DIR_PATH ${CNI_CONFIG_DIR_PATH}
|
||||
ARG CNI_BIN_DIR_PATH=/opt/cni/bin
|
||||
ENV CNI_BIN_DIR_PATH ${CNI_BIN_DIR_PATH}
|
||||
ARG CNI_DAEMON=False
|
||||
ARG CNI_DAEMON=True
|
||||
ENV CNI_DAEMON ${CNI_DAEMON}
|
||||
|
||||
VOLUME [ "/sys/fs/cgroup" ]
|
||||
|
|
|
|||
|
|
@ -181,6 +181,17 @@ enable_service kubelet
|
|||
# resource events and convert them to Neutron actions
|
||||
enable_service kuryr-kubernetes
|
||||
|
||||
# Kuryr Daemon
|
||||
# ============
|
||||
#
|
||||
# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
|
||||
# driver and the driver will pass requests to Kuryr daemon running on the node,
|
||||
# instead of processing them on its own. This limits the number of Kubernetes
|
||||
# API requests (as only Kuryr Daemon will watch for new pod events) and should
|
||||
# increase scalability in environments that often delete and create pods.
|
||||
# Since Rocky release this is a default deployment configuration.
|
||||
enable_service kuryr-daemon
|
||||
|
||||
# Kuryr POD VIF Driver
|
||||
# ====================
|
||||
#
|
||||
|
|
|
|||
|
|
@ -145,6 +145,17 @@ enable_service kubelet
|
|||
# resource events and convert them to Neutron actions
|
||||
enable_service kuryr-kubernetes
|
||||
|
||||
# Kuryr Daemon
|
||||
# ============
|
||||
#
|
||||
# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
|
||||
# driver and the driver will pass requests to Kuryr daemon running on the node,
|
||||
# instead of processing them on its own. This limits the number of Kubernetes
|
||||
# API requests (as only Kuryr Daemon will watch for new pod events) and should
|
||||
# increase scalability in environments that often delete and create pods.
|
||||
# Since Rocky release this is a default deployment configuration.
|
||||
enable_service kuryr-daemon
|
||||
|
||||
# Kuryr POD VIF Driver
|
||||
# ====================
|
||||
#
|
||||
|
|
|
|||
|
|
@ -154,6 +154,17 @@ DOCKER_CGROUP_DRIVER="systemd"
|
|||
# resource events and convert them to Neutron actions
|
||||
enable_service kuryr-kubernetes
|
||||
|
||||
# Kuryr Daemon
|
||||
# ============
|
||||
#
|
||||
# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
|
||||
# driver and the driver will pass requests to Kuryr daemon running on the node,
|
||||
# instead of processing them on its own. This limits the number of Kubernetes
|
||||
# API requests (as only Kuryr Daemon will watch for new pod events) and should
|
||||
# increase scalability in environments that often delete and create pods.
|
||||
# Since Rocky release this is a default deployment configuration.
|
||||
enable_service kuryr-daemon
|
||||
|
||||
# Containerized Kuryr
|
||||
# ===================
|
||||
#
|
||||
|
|
|
|||
|
|
@ -191,7 +191,7 @@ enable_service kuryr-kubernetes
|
|||
# instead of processing them on its own. This limits the number of Kubernetes
|
||||
# API requests (as only Kuryr Daemon will watch for new pod events) and should
|
||||
# increase scalability in environments that often delete and create pods.
|
||||
# To enable kuryr-daemon uncomment next line.
|
||||
# Since Rocky release this is a default deployment configuration.
|
||||
enable_service kuryr-daemon
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ enable_service kubernetes-controller-manager
|
|||
enable_service kubernetes-scheduler
|
||||
enable_service kubelet
|
||||
enable_service kuryr-kubernetes
|
||||
enable_service kuryr-daemon
|
||||
|
||||
KURYR_POD_VIF_DRIVER=nested-vlan
|
||||
|
||||
|
|
|
|||
|
|
@ -182,13 +182,13 @@ enable_service kuryr-kubernetes
|
|||
# Kuryr Daemon
|
||||
# ============
|
||||
#
|
||||
# Kuryr can run CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
|
||||
# Kuryr runs CNI plugin in daemonized way - i.e. kubelet will run kuryr CNI
|
||||
# driver and the driver will pass requests to Kuryr daemon running on the node,
|
||||
# instead of processing them on its own. This limits the number of Kubernetes
|
||||
# API requests (as only Kuryr Daemon will watch for new pod events) and should
|
||||
# increase scalability in environments that often delete and create pods.
|
||||
# To enable kuryr-daemon uncomment next line.
|
||||
# enable_service kuryr-daemon
|
||||
# Since Rocky release this is a default deployment configuration.
|
||||
enable_service kuryr-daemon
|
||||
|
||||
|
||||
# Containerized Kuryr
|
||||
|
|
|
|||
|
|
@ -82,7 +82,6 @@ function configure_kuryr {
|
|||
fi
|
||||
|
||||
if is_service_enabled kuryr-daemon; then
|
||||
iniset "$KURYR_CONFIG" cni_daemon daemon_enabled True
|
||||
iniset "$KURYR_CONFIG" oslo_concurrency lock_path "$KURYR_LOCK_DIR"
|
||||
create_kuryr_lock_dir
|
||||
if [ "$KURYR_K8S_CONTAINERIZED_DEPLOYMENT" == "True" ]; then
|
||||
|
|
@ -91,6 +90,8 @@ function configure_kuryr {
|
|||
iniset "$KURYR_CONFIG" cni_daemon docker_mode True
|
||||
iniset "$KURYR_CONFIG" cni_daemon netns_proc_dir "/host_proc"
|
||||
fi
|
||||
else
|
||||
iniset "$KURYR_CONFIG" cni_daemon daemon_enabled False
|
||||
fi
|
||||
|
||||
create_kuryr_cache_dir
|
||||
|
|
|
|||
|
|
@ -165,45 +165,19 @@ CNI driver to complete pod handling.
|
|||
The NeutronPodVifDriver is the default driver that creates neutron port upon
|
||||
Pod addition and deletes port upon Pod removal.
|
||||
|
||||
CNI Driver
|
||||
----------
|
||||
Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin <http://kubernetes.io/docs/admin/network-plugins/#cni>`_
|
||||
and introduces Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes
|
||||
CNI Driver should get all information required to plug and bind Pod via
|
||||
kubernetes control plane and should not depend on Neutron. CNI plugin/driver
|
||||
is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore it is
|
||||
expected to return when either success or error state determined.
|
||||
|
||||
Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node
|
||||
environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the
|
||||
contract that defines Pod annotation that Controller Server adds and CNI
|
||||
driver reads. The contract is `os_vif VIF <https://github.com/openstack/os-vif/blob/master/os_vif/objects/vif.py>`_
|
||||
|
||||
With VIF object loaded from the Pod object annotation, the CNI driver performs
|
||||
Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and
|
||||
unplug operations. The CNI driver should complete its job and return control to
|
||||
Kubelet when all the network plugging is completed.
|
||||
In the cases when Neutron initially creates port in 'Down' state, CNI driver
|
||||
will plug the Pod, but will have to watch the Pod annotations for vif state
|
||||
change to 'Active' before returning the control to the caller.
|
||||
|
||||
.. image:: ../../images/pod_creation_flow.png
|
||||
:alt: Controller-CNI interaction
|
||||
:align: center
|
||||
:width: 100%
|
||||
|
||||
.. _cni-daemon:
|
||||
|
||||
CNI Daemon
|
||||
----------
|
||||
|
||||
CNI Daemon is an optional service that should run on every Kubernetes node. It
|
||||
is responsible for watching pod events on the node it's running on, answering
|
||||
calls from CNI Driver and attaching VIFs when they are ready. In the future
|
||||
it will also keep information about pooled ports in memory. This helps to limit
|
||||
the number of processes spawned when creating multiple Pods, as a single
|
||||
Watcher is enough for each node and CNI Driver will only wait on local network
|
||||
socket for response from the Daemon.
|
||||
CNI Daemon is a service that should run on every Kubernetes node. Starting from
|
||||
Rocky release it should be seen as a default supported deployment option.
|
||||
It is responsible for watching pod events on the node it's running on,
|
||||
answering calls from CNI Driver and attaching VIFs when they are ready. In the
|
||||
future it will also keep information about pooled ports in memory. This helps
|
||||
to limit the number of processes spawned when creating multiple Pods, as a
|
||||
single Watcher is enough for each node and CNI Driver will only wait on local
|
||||
network socket for response from the Daemon.
|
||||
|
||||
Currently CNI Daemon consists of two processes i.e. Watcher and Server.
|
||||
Processes communicate between each other using Python's
|
||||
|
|
@ -252,6 +226,44 @@ deserialized using o.vo's ``obj_from_primitive()`` method.
|
|||
When running in daemonized mode, CNI Driver will call CNI Daemon over those APIs
|
||||
to perform its tasks and wait on socket for result.
|
||||
|
||||
CNI Driver (deprecated)
|
||||
-----------------------
|
||||
|
||||
.. warning::
|
||||
Running with CNI Driver in this mode is deprecated since Rocky release.
|
||||
Currently the preferred way of deploying kuryr-kubernetes is with
|
||||
kuryr-daemon that takes over most of the CNI Driver tasks. In that case CNI
|
||||
driver becomes a thin client that passes CNI ADD and DEL requests to
|
||||
kuryr-daemon instance via its HTTP API.
|
||||
|
||||
Kuryr kubernetes integration takes advantage of the kubernetes `CNI plugin
|
||||
<http://kubernetes.io/docs/admin/network-plugins/#cni>`_ and introduces
|
||||
Kuryr-K8s CNI Driver. Based on design decision, kuryr-kubernetes
|
||||
CNI Driver should get all information required to plug and bind Pod via
|
||||
kubernetes control plane and should not depend on Neutron. CNI plugin/driver
|
||||
is invoked in a blocking manner by kubelet (Kubernetes node agent), therefore
|
||||
it is expected to return when either success or error state determined.
|
||||
|
||||
Kuryr-K8s CNI Driver has 2 sources for Pod binding information: kubelet/node
|
||||
environment and Kubernetes API. The Kuryr-K8s Controller Service and CNI share the
|
||||
contract that defines Pod annotation that Controller Server adds and CNI
|
||||
driver reads. The contract is `os_vif VIF
|
||||
<https://github.com/openstack/os-vif/blob/master/os_vif/objects/vif.py>`_
|
||||
|
||||
With VIF object loaded from the Pod object annotation, the CNI driver performs
|
||||
Pod plugging. Kuryr-K8s CNI driver uses ov_vif library to perform Pod plug and
|
||||
unplug operations. The CNI driver should complete its job and return control to
|
||||
Kubelet when all the network plugging is completed.
|
||||
In the cases when Neutron initially creates port in 'Down' state, CNI driver
|
||||
will plug the Pod, but will have to watch the Pod annotations for vif state
|
||||
change to 'Active' before returning the control to the caller.
|
||||
|
||||
.. image:: ../../images/pod_creation_flow.png
|
||||
:alt: Controller-CNI interaction
|
||||
:align: center
|
||||
:width: 100%
|
||||
|
||||
|
||||
Kubernetes Documentation
|
||||
------------------------
|
||||
The `Kubernetes reference documentation <https://kubernetes.io/docs/reference/>`_
|
||||
|
|
|
|||
|
|
@ -48,8 +48,9 @@ Now edit ``devstack/local.conf`` to set up some initial options:
|
|||
omitted.
|
||||
* If you already have Docker installed on the machine, you can comment out line
|
||||
starting with ``enable_plugin devstack-plugin-container``.
|
||||
* If you want to enable kuryr-daemon uncomment ``enable_service kuryr-daemon``
|
||||
line.
|
||||
* If you want to disable kuryr-daemon add ``disable_service kuryr-daemon``
|
||||
line. Please note that running without kuryr-daemon was deprecated in Rocky
|
||||
release.
|
||||
|
||||
Once ``local.conf`` is configured, you can start the installation: ::
|
||||
|
||||
|
|
|
|||
|
|
@ -157,20 +157,15 @@ to work correctly::
|
|||
deactivate
|
||||
sudo pip install 'oslo.privsep>=1.20.0' 'os-vif>=1.5.0'
|
||||
|
||||
Configure Kuryr CNI Daemon (optional)
|
||||
Configure Kuryr CNI Daemon
|
||||
-------------------------------------
|
||||
|
||||
Kuryr CNI Daemon is an optional service designed to increased scalability of
|
||||
the Kuryr operations done on Kubernetes nodes. More information can be found on
|
||||
Kuryr CNI Daemon is a service designed to increased scalability of the Kuryr
|
||||
operations done on Kubernetes nodes. More information can be found on
|
||||
:ref:`cni-daemon` page.
|
||||
|
||||
If you want to use Kuryr CNI Daemon, it needs to be installed on every
|
||||
Kubernetes node, so following steps need to be repeated.
|
||||
|
||||
Edit ``kuryr.conf``::
|
||||
|
||||
[cni_daemon]
|
||||
daemon_enabled=True
|
||||
Kuryr CNI Daemon, should be installed on every Kubernetes node, so following
|
||||
steps need to be repeated.
|
||||
|
||||
.. note::
|
||||
You can tweak configuration of some timeouts to match your environment. It's
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import sys
|
|||
import os_vif
|
||||
from oslo_config import cfg
|
||||
from oslo_log import log as logging
|
||||
from oslo_log import versionutils
|
||||
from oslo_serialization import jsonutils
|
||||
|
||||
from kuryr_kubernetes.cni import api as cni_api
|
||||
|
|
@ -56,6 +57,13 @@ def run():
|
|||
if CONF.cni_daemon.daemon_enabled:
|
||||
runner = cni_api.CNIDaemonizedRunner()
|
||||
else:
|
||||
# TODO(dulek): Switch that to versionutils.deprecation_warning once
|
||||
# bug 1754087 is fixed.
|
||||
versionutils.report_deprecated_feature(
|
||||
LOG,
|
||||
'Deploying kuryr-kubernetes without kuryr-daemon service is '
|
||||
'deprecated since Rocky release and may be removed in future '
|
||||
'releases.')
|
||||
runner = cni_api.CNIStandaloneRunner(k8s_cni.K8sCNIPlugin())
|
||||
LOG.info("Using '%s' ", runner.__class__.__name__)
|
||||
|
||||
|
|
|
|||
|
|
@ -33,7 +33,11 @@ kuryr_k8s_opts = [
|
|||
daemon_opts = [
|
||||
cfg.BoolOpt('daemon_enabled',
|
||||
help=_('Enable CNI Daemon configuration.'),
|
||||
default=False),
|
||||
default=True,
|
||||
deprecated_for_removal=True,
|
||||
deprecated_reason="Deployment without kuryr-daemon is now "
|
||||
"deprecated.",
|
||||
deprecated_since="Rocky"),
|
||||
cfg.StrOpt('bind_address',
|
||||
help=_('Bind address for CNI daemon HTTP server. It is '
|
||||
'recommened to allow only local connections.'),
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
---
|
||||
upgrade:
|
||||
- |
|
||||
Legacy Kuryr deployment without running kuryr-daemon is now considered
|
||||
deprecated. That possibility will be completely removed in one of the next
|
||||
releases. Please note that this means that ``[cni_daemon]daemon_enabled``
|
||||
option will default to ``True``.
|
||||
deprecations:
|
||||
- |
|
||||
Running Kuryr-Kubernetes without kuryr-daemon service is now deprecated.
|
||||
Motivations for that move include:
|
||||
|
||||
* Discoveries of bugs that are much easier to fix in kuryr-daemon.
|
||||
* Further improvements in Kuryr scalability (e.g. moving choosing VIF from
|
||||
pool into kuryr-daemon) are only possible when kuryr-daemon is present.
|
||||
|
||||
Possibility of running Kuryr-Kubernetes without kuryr-daemon will be
|
||||
removed in one of the future releases.
|
||||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
CNI_BIN_DIR=$1
|
||||
CNI_CONF_DIR=$2
|
||||
CNI_DAEMON=${3:-"False"}
|
||||
CNI_DAEMON=${3:-"True"}
|
||||
CNI_TAG="kuryr/cni"
|
||||
|
||||
# create cni daemonset image
|
||||
|
|
|
|||
Loading…
Reference in New Issue