diff --git a/devstack/lib/cni/plugins b/devstack/lib/cni/plugins new file mode 100644 index 0000000..2577a59 --- /dev/null +++ b/devstack/lib/cni/plugins @@ -0,0 +1,93 @@ +#!/bin/bash +# +# lib/cni/plugins +# Common CNI plugins functions + +# Dependencies: +# ``functions`` file +# ``STACK_USER`` has to be defined + +# Save trace setting +_XTRACE_CONTAINER_CNI_PLUGINS=$(set +o | grep xtrace) +set +o xtrace + +# Defaults +# -------- + +CNI_PLUGINS_BIN_DIR=/opt/cni/bin +# install all plugins by default +CNI_PLUGINS_INSTALL_PLUGINS=${CNI_PLUGINS_INSTALL_PLUGINS:-flannel,ptp,host-local,portmap,tuning,vlan,host-device,sample,dhcp,ipvlan,macvlan,loopback,bridge} +CNI_PLUGINS_CONF_SOURCE_DIR=${CNI_PLUGINS_CONF_SOURCE_DIR:-$DEST/devstack-plugin-container/etc/cni/net.d} +CNI_PLUGINS_CONF_DIR=${CNI_PLUGINS_CONF_DIR:-/etc/cni/net.d} + +CNI_PLUGINS_VERSION=${CNI_PLUGINS_VERSION:-v0.7.1} +CNI_PLUGINS_SHA256_AMD64=${CNI_PLUGINS_SHA256_AMD64:-"6ecc5c7dbb8e4296b0d0d017e5440618e19605b9aa3b146a2c29af492f299dc7"} +CNI_PLUGINS_SHA256_ARM64=${CNI_PLUGINS_SHA256_ARM64:-"258080b94bfc54bd54fd0ea7494efc31806aa4b2836ba3f2d189e0fc16fab0ef"} +CNI_PLUGINS_SHA256_PPC64=${CNI_PLUGINS_SHA256_PPC64:-"a515c45a52e752249bb0e9feac1654c5d38974df6a36148778f6eeab9826f706"} +CNI_PLUGINS_SHA256_S390X=${CNI_PLUGINS_SHA256_S390X:-"24e31be69a012395f1026cd37d125f5f81001cfc36434d8f7a17b36bc5f1e6ad"} +# Make sure CNI plugins downloads the correct architecture +if is_arch "x86_64"; then + CNI_PLUGINS_ARCH="amd64" + CNI_PLUGINS_SHA256=${CNI_PLUGINS_SHA256:-$CNI_PLUGINS_SHA256_AMD64} +elif is_arch "aarch64"; then + CNI_PLUGINS_ARCH="arm64" + CNI_PLUGINS_SHA256=${CNI_PLUGINS_SHA256:-$CNI_PLUGINS_SHA256_ARM64} +elif is_arch "ppc64le"; then + CNI_PLUGINS_ARCH="ppc64le" + CNI_PLUGINS_SHA256=${CNI_PLUGINS_SHA256:-$CNI_PLUGINS_SHA256_PPC64} +elif is_arch "s390x"; then + CNI_PLUGINS_ARCH="s390x" + CNI_PLUGINS_SHA256=${CNI_PLUGINS_SHA256:-$CNI_PLUGINS_SHA256_S390X} +else + exit_distro_not_supported "invalid hardware type" +fi +CNI_PLUGINS_DOWNLOAD_URL=${CNI_PLUGINS_DOWNLOAD_URL:-https://github.com/containernetworking/plugins/releases/download} +CNI_PLUGINS_DOWNLOAD_FILE=cni-plugins-$CNI_PLUGINS_ARCH-$CNI_PLUGINS_VERSION.tgz +CNI_PLUGINS_DOWNLOAD_LOCATION=$CNI_PLUGINS_DOWNLOAD_URL/$CNI_PLUGINS_VERSION/$CNI_PLUGINS_DOWNLOAD_FILE + + +# Installs standard cni plugins. +function install_cni_plugins { + echo "Installing CNI standard plugins" + + # Download and cache the cni plugins tgz for subsequent use + local plugins_file + cni_plugins_file="$(get_extra_file $CNI_PLUGINS_DOWNLOAD_LOCATION)" + if [ ! -d "$FILES/cniplugins" ]; then + echo "${CNI_PLUGINS_SHA256} $cni_plugins_file" > $FILES/cniplugins.sha256sum + # remove the damaged file when checksum fails + sha256sum -c $FILES/cniplugins.sha256sum || (sudo rm -f $cni_plugins_file; exit 1) + + mkdir $FILES/cniplugins + tar xzvf $cni_plugins_file -C $FILES/cniplugins + fi + for plugin in ${CNI_PLUGINS_INSTALL_PLUGINS//,/ }; do + if [ $(ls $FILES/cniplugins/$plugin 2> /dev/null) ]; then + echo "Install plugin: $plugin" + sudo install -o "$STACK_USER" -m 0555 -D "$FILES/cniplugins/$plugin" \ + "$CNI_PLUGINS_BIN_DIR/$plugin" + else + echo "Skip installing plugin: $plugin" + fi + done +} + +# Configure cni plugins. +function configure_cni_plugins { + echo "Configuring CNI plugins" + + for plugin in ${CNI_PLUGINS_INSTALL_PLUGINS//,/ }; do + local source_config_file=$(ls ${CNI_PLUGINS_CONF_SOURCE_DIR}/*${plugin}.conf 2> /dev/null) + if [ $source_config_file ]; then + echo "Found config file for plugin: $plugin" + sudo install -o "$STACK_USER" -m 0664 -t "$CNI_PLUGINS_CONF_DIR" -D \ + "${source_config_file}" + else + echo "Config file not found for plugin: $plugin" + fi + done +} + + +# Restore xtrace +$_XTRACE_CONTAINER_CNI_PLUGINS diff --git a/devstack/lib/docker b/devstack/lib/docker index 9cb7d4c..c215f70 100644 --- a/devstack/lib/docker +++ b/devstack/lib/docker @@ -29,10 +29,14 @@ DOCKER_CGROUP_DRIVER=${DOCKER_CGROUP_DRIVER:-} # TODO(hongbin): deprecate and remove clear container ENABLE_CLEAR_CONTAINER=$(trueorfalse False ENABLE_CLEAR_CONTAINER) ENABLE_KATA_CONTAINERS=$(trueorfalse False ENABLE_KATA_CONTAINERS) +ENABLE_CONTAINERD_CRI=$(trueorfalse False ENABLE_CONTAINERD_CRI) ENABLE_LIVE_RESTORE=$(trueorfalse False ENABLE_LIVE_RESTORE) ENABLE_IPV6=$(trueorfalse False ENABLE_IPV6) KATA_BRANCH=${KATA_BRANCH:-master} +CONTAINERD_CONF_DIR=/etc/containerd +CONTAINERD_CONF=$CONTAINERD_CONF_DIR/config.toml + # Functions # --------- @@ -107,9 +111,27 @@ function install_docker { (>&2 echo "WARNING: Clear Container needs the CPU extensions svm or vmx which is not enabled. Skipping Clear Container installation.") fi fi + + if [[ "$ENABLE_CONTAINERD_CRI" == "True" ]]; then + source $DEST/devstack-plugin-container/devstack/lib/cni/plugins + install_cni_plugins + + source $DEST/devstack-plugin-container/devstack/lib/tools/crictl + install_crictl + fi } function configure_docker { + if [[ ${ENABLE_CONTAINERD_CRI} == "True" ]]; then + configure_containerd + + source $DEST/devstack-plugin-container/devstack/lib/cni/plugins + configure_cni_plugins + + source $DEST/devstack-plugin-container/devstack/lib/tools/crictl + configure_crictl + fi + # After an ./unstack it will be stopped. So it is ok if it returns exit-code == 1 sudo systemctl stop docker.service || true @@ -196,6 +218,17 @@ EOF sudo systemctl --no-block restart docker.service } +function configure_containerd { + sudo mkdir -p $CONTAINERD_CONF_DIR + sudo chown -R $STACK_USER $CONTAINERD_CONF_DIR + + containerd config default > $CONTAINERD_CONF + stack_user_gid=$(getent group $STACK_USER | cut -d: -f3) + sed -i "s/gid = [0-9]*/gid = ${stack_user_gid}/" $CONTAINERD_CONF + sed -i "s/level = \"\"/level = \"debug\"/" $CONTAINERD_CONF + sudo systemctl --no-block restart containerd.service +} + function stop_docker { sudo systemctl stop docker.service || true } diff --git a/devstack/lib/tools/crictl b/devstack/lib/tools/crictl new file mode 100644 index 0000000..5498879 --- /dev/null +++ b/devstack/lib/tools/crictl @@ -0,0 +1,76 @@ +#!/bin/bash +# +# lib/tools/crictl +# CRI command line tools functions + +# Dependencies: +# ``functions`` file +# ``STACK_USER`` has to be defined + +# Save trace setting +_XTRACE_CONTAINER_TOOLS_CRICTL=$(set +o | grep xtrace) +set +o xtrace + +# Defaults +# -------- + +CRICTL_BIN_DIR=/usr/local/bin + +CRICTL_VERSION=${CRICTL_VERSION:-v1.17.0} +CRICTL_SHA256_AMD64=${CRICTL_SHA256_AMD64:-"7b72073797f638f099ed19550d52e9b9067672523fc51b746e65d7aa0bafa414"} +CRICTL_SHA256_ARM64=${CRICTL_SHA256_ARM64:-"d89afd89c2852509fafeaff6534d456272360fcee732a8d0cb89476377387e12"} +CRICTL_SHA256_PPC64=${CRICTL_SHA256_PPC64:-"a61c52b9ac5bffe94ae4c09763083c60f3eccd30eb351017b310f32d1cafb855"} +CRICTL_SHA256_S390X=${CRICTL_SHA256_S390X:-"0db445f0b74ecb51708b710480a462b728174155c5f2709a39d1cc2dc975e350"} +# Make sure downloads the correct architecture +if is_arch "x86_64"; then + CRICTL_ARCH="amd64" + CRICTL_SHA256=${CRICTL_SHA256:-$CRICTL_SHA256_AMD64} +elif is_arch "aarch64"; then + CRICTL_ARCH="arm64" + CRICTL_SHA256=${CRICTL_SHA256:-$CRICTL_SHA256_ARM64} +elif is_arch "ppc64le"; then + CRICTL_ARCH="ppc64le" + CRICTL_SHA256=${CRICTL_SHA256:-$CRICTL_SHA256_PPC64} +elif is_arch "s390x"; then + CRICTL_ARCH="s390x" + CRICTL_SHA256=${CRICTL_SHA256:-$CRICTL_SHA256_S390X} +else + exit_distro_not_supported "invalid hardware type" +fi +CRICTL_DOWNLOAD_URL=${CRICTL_DOWNLOAD_URL:-https://github.com/kubernetes-sigs/cri-tools/releases/download} +CRICTL_DOWNLOAD_FILE=crictl-$CRICTL_VERSION-linux-$CRICTL_ARCH.tar.gz +CRICTL_DOWNLOAD_LOCATION=$CRICTL_DOWNLOAD_URL/$CRICTL_VERSION/$CRICTL_DOWNLOAD_FILE + + +# Installs crictl tools. +function install_crictl { + echo "Installing CRI command-line tools" + + # Download and cache the crictl tar for subsequent use + local crictl_file + crictl_file="$(get_extra_file $CRICTL_DOWNLOAD_LOCATION)" + if [ ! -f "$FILES/crictl" ]; then + echo "${CRICTL_SHA256} $crictl_file" > $FILES/crictl.sha256sum + # remove the damaged file when checksum fails + sha256sum -c $FILES/crictl.sha256sum || (sudo rm -f $crictl_file; exit 1) + + tar xzvf $crictl_file -C $FILES + sudo install -o "$STACK_USER" -m 0555 -D "$FILES/crictl" \ + "$CRICTL_BIN_DIR/crictl" + fi +} + +# Configure crictl tools. +function configure_crictl { + local crictl_config_file=/etc/crictl.yaml + cat </dev/null +runtime-endpoint: unix:///run/containerd/containerd.sock +image-endpoint: unix:///run/containerd/containerd.sock +timeout: 10 +debug: true +EOF +} + + +# Restore xtrace +$_XTRACE_CONTAINER_TOOLS_CRICTL diff --git a/devstack/settings b/devstack/settings index 95dd453..a104f7a 100644 --- a/devstack/settings +++ b/devstack/settings @@ -8,6 +8,7 @@ ENABLE_KATA_CONTAINERS=${ENABLE_KATA_CONTAINERS:-false} ENABLE_LIVE_RESTORE=${ENABLE_LIVE_RESTORE:-false} ENABLE_IPV6=${ENABLE_IPV6:-false} K8S_NETWORK_ADDON=${K8S_NETWORK_ADDON:-flannel} +ENABLE_CONTAINERD_CRI=${ENABLE_CONTAINERD_CRI:-false} # Enable container services enable_service container diff --git a/etc/cni/net.d/10-bridge.conf b/etc/cni/net.d/10-bridge.conf new file mode 100644 index 0000000..7ec6153 --- /dev/null +++ b/etc/cni/net.d/10-bridge.conf @@ -0,0 +1,15 @@ +{ + "cniVersion": "0.2.0", + "name": "mynet", + "type": "bridge", + "bridge": "cni0", + "isGateway": true, + "ipMasq": true, + "ipam": { + "type": "host-local", + "subnet": "10.22.0.0/16", + "routes": [ + { "dst": "0.0.0.0/0" } + ] + } +} diff --git a/etc/cni/net.d/99-loopback.conf b/etc/cni/net.d/99-loopback.conf new file mode 100644 index 0000000..9cec881 --- /dev/null +++ b/etc/cni/net.d/99-loopback.conf @@ -0,0 +1,5 @@ +{ + "cniVersion": "0.2.0", + "name": "lo", + "type": "loopback" +}