devstack: First version of kuryr-kubernetes plugin

At the moment, since the repo doesn't have CNI, and the event processing
pipeline is just a stub, it won't do practically anything, but it should
already be laying all the environment that we'll use to develop the
missing pieces and we can use it for the fullstack tests as well.

Partially implements: blueprint kuryr-k8s-integration
Change-Id: I7ca43ec7d0379d73273609078f5377ea6f55c363
Signed-off-by: Antoni Segura Puimedon <antonisp@celebdor.com>
This commit is contained in:
Antoni Segura Puimedon 2016-09-16 12:45:11 +02:00
parent 8f0a7cf0be
commit 2f82bb7ebe
No known key found for this signature in database
GPG Key ID: 2329618D2967720A
5 changed files with 575 additions and 0 deletions

131
devstack/local.conf.sample Normal file
View File

@ -0,0 +1,131 @@
[[local|localrc]]
# At the moment kuryr-kubernetes only works with Python35, thus, you should
# read https://wiki.openstack.org/wiki/Python3#Enable_Python_3_in_DevStack
# This devstack plugin should work with Ubuntu 16.04+ and Fedora 24+
# Make sure you have python3.5 and it's development package installed
USE_PYTHON3=True
PYTHON3_VERSION=3.5
enable_plugin kuryr-kubernetes \
https://git.openstack.org/openstack/kuryr-kubernetes
# If you do not want stacking to clone new versions of the enabled services,
# like for example when you did local modifications and need to ./unstack.sh
# and ./stack.sh again, uncomment the following
# RECLONE="no"
# Log settings for better readability
LOGFILE=devstack.log
LOG_COLOR=False
# If you want the screen tabs logged in a specific location, you can use:
# SCREEN_LOGDIR="${HOME}/devstack_logs"
# Credentials
ADMIN_PASSWORD=pass
DATABASE_PASSWORD=pass
RABBIT_PASSWORD=pass
SERVICE_PASSWORD=pass
SERVICE_TOKEN=pass
# Enable Keystone v3
IDENTITY_API_VERSION=3
# In pro of speed and being lightweight, we will be explicit in regards to
# which services we enable
ENABLED_SERVICES=""
# Neutron services
enable_service neutron
enable_service q-agt
enable_service q-dhcp
enable_service q-l3
enable_service q-svc
# Keystone
enable_service key
# dependencies
enable_service mysql
enable_service rabbit
# By default use all the services from the kuryr-kubernetes plugin
# Docker
# ======
# If you already have docker configured, running and with its socket writable
# by the stack user, you can omit the following line.
enable_service docker
# Etcd
# ====
# The default is for devstack to run etcd for you. You can select the image and
# version of it by uncommenting and modifying the following defaults.
# KURYR_ETCD_IMAGE="quay.io/coreos/etcd"
# KURYR_ETCD_VERSION="v3.0.8"
#
# You can select the listening and advertising client and peering Etcd
# addresses by uncommenting and changing from the following defaults:
# KURYR_ETCD_ADVERTISE_CLIENT_URL=http://my_host_ip:2379}
# KURYR_ETCD_ADVERTISE_PEER_URL=http://my_host_ip:2380}
# KURYR_ETCD_LISTEN_CLIENT_URL=http://0.0.0.0:2379}
# KURYR_ETCD_LISTEN_PEER_URL=http://0.0.0.0:2380}
#
# If you already have etcd configured and running, you can just comment out
enable_service etcd
# then uncomment and set the following line:
# KURYR_ETCD_CLIENT_URL="http://etcd_ip:etcd_client_port"
# Kubernetes
# ==========
#
# Kubernetes is run from the hyperkube docker image
# If you already have a Kubernetes deployment, you can use it instead and omit
# enabling the Kubernetes service (except Kubelet, which must be run by
# devstack so that it uses our development CNI driver.
#
# The default is, again, for devstack to run the Kubernetes services:
enable_service kubernetes-api
enable_service kubernetes-controller-manager
enable_service kubernetes-scheduler
# We use hyperkube to run the services. You can select the hyperkube image and/
# or version by uncommenting and setting the following ENV vars different
# to the following defaults:
# KURYR_HYPERKUBE_IMAGE="gcr.io/google_containers/hyperkube-amd64"
# KURYR_HYPERKUBE_VERSION="v1.3.7"
#
# If you have the 8080 port already bound to another service, you will need to
# have kubernetes API server bind to another port. In order to do that,
# uncomment and set a different port number in:
# KURYR_K8S_API_PORT="8080"
#
# If you want to test with a different range for the Cluster IPs uncomment and
# set the following ENV var to a different CIDR
# KURYR_K8S_CLUSTER_IP_RANGE="10.0.0.0/24"
#
# If, however, you are reusing an existing deployment, you should uncomment and
# set an ENV var so that the Kubelet devstack runs can find the API server:
# KURYR_K8S_API_URL="http://k8s_api_ip:k8s_api_port"
#
# Kubelet
# =======
#
# Kubelet should almost invariably be run by devstack
enable_service kubelet
# You can specify a different location for the hyperkube binary that will be
# extracted from the hyperkube container into the Host filesystem:
# KURYR_HYPERKUBE_BINARY=/usr/local/bin/hyperkube
#
# NOTE: KURYR_HYPERKUBE_IMAGE, KURYR_HYPERKUBE_VERSION also affect which
# the selected binary for the Kubelet.
# Kuryr watcher
# =============
#
# Just like the Kubelet, you'll want to have the watcher enabled. It is the
# part of the codebase that connects to the Kubernetes API server to read the
# resource events and convert them to Neutron actions
enable_service kuryr-kubernetes

392
devstack/plugin.sh Normal file
View File

@ -0,0 +1,392 @@
#!/bin/bash
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
# Save trace setting
XTRACE=$(set +o | grep xtrace)
set +o xtrace
function run_container {
# Runs a detached container and uses devstack's run process to monitor
# its logs
local name
name="$1"
shift
docker run --name "$name" --detach "$@"
run_process "$name" \
"docker logs -f $name"
}
function stop_container {
local name
name="$1"
docker kill "$name"
docker rm "$name"
stop_process "$name"
}
function create_kuryr_account {
if is_service_enabled kuryr-kubernetes; then
create_service_user "kuryr" "admin"
get_or_create_service "kuryr-kubernetes" "kuryr-kubernetes" \
"Kuryr-Kubernetes Service"
fi
}
function create_kuryr_cache_dir {
# Create cache directory
sudo install -d -o "$STACK_USER" "$KURYR_AUTH_CACHE_DIR"
if [[ ! "$KURYR_AUTH_CACHE_DIR" == "" ]]; then
rm -f "$KURYR_AUTH_CACHE_DIR"/*
fi
}
function get_distutils_data_path {
cat << EOF | python -
from __future__ import print_function
import distutils.dist
import distutils.command.install
inst = distutils.command.install.install(distutils.dist.Distribution())
inst.finalize_options()
print(inst.install_data)
EOF
}
function configure_kuryr {
sudo install -d -o "$STACK_USER" "$KURYR_CONFIG_DIR"
# TODO(apuimedo): remove when we have config generation
# (cd "$KURYR_HOME" && exec ./tools/generate_config_file_samples.sh)
# cp "$KURYR_HOME/etc/kuryr.conf.sample" "$KURYR_CONFIG"
# iniset -sudo ${KURYR_CONFIG} DEFAULT bindir \
# "$(get_distutils_data_path)/libexec/kuryr"
create_kuryr_cache_dir
# Neutron API server & Neutron plugin
if is_service_enabled kuryr-kubernetes; then
configure_auth_token_middleware "$KURYR_CONFIG" kuryr \
"$KURYR_AUTH_CACHE_DIR" neutron
fi
}
function check_docker {
if is_ubuntu; then
dpkg -s docker-engine > /dev/null 2>&1
else
rpm -q docker-engine > /dev/null 2>&1
fi
}
function get_container {
local image
local image_name
local version
image_name="$1"
version="${2:-latest}"
if [ "$image_name" == "" ]; then
return 0
fi
image="${image}:${version}"
if [ -z "$(docker images -q "$image")" ]; then
docker pull "$image"
fi
}
function prepare_etcd {
# Make Etcd data directory
sudo install -d -o "$STACK_USER" "$KURYR_ETCD_DATA_DIR"
# Get Etcd container
get_container "$KURYR_ETCD_IMAGE" "$KURYR_ETCD_VERSION"
}
function run_etcd {
run_container etcd \
--net host \
--volume="${KURYR_ETCD_DATA_DIR}:/var/etcd:rw" \
"${KURYR_ETCD_IMAGE}:${KURYR_ETCD_VERSION}" \
/usr/local/bin/etcd \
--name devstack \
--data-dir /var/etcd/data \
--initial-advertise-peer-urls "$KURYR_ETCD_ADVERTISE_PEER_URL" \
--listen-peer-urls "$KURYR_ETCD_LISTEN_PEER_URL" \
--listen-client-urls "$KURYR_ETCD_LISTEN_CLIENT_URL" \
--advertise-client-urls "$KURYR_ETCD_ADVERTISE_CLIENT_URL" \
--initial-cluster-token etcd-cluster-1 \
--initial-cluster "devstack=$KURYR_ETCD_PEER_URL" \
--initial-cluster-state new
}
function prepare_docker {
curl http://get.docker.com | sudo bash
# After an ./unstack it will be stopped. So it is OK if it returns
# exit-code == 1
sudo service docker stop || true
# Make sure there's no leftover Docker process and pidfile
sudo kill -s SIGTERM "$(cat /var/run/docker.pid)"
}
function run_docker {
run_process docker \
"sudo /usr/bin/docker daemon --debug=true \
-H unix://$KURYR_DOCKER_ENGINE_SOCKET_FILE"
# We put the stack user as owner of the socket so we do not need to
# run the Docker commands with sudo when developing.
echo -n "Waiting for Docker to create its socket file"
while [ ! -e "$KURYR_DOCKER_ENGINE_SOCKET_FILE" ]; do
echo -n "."
sleep 1
done
echo ""
sudo chown "$STACK_USER":docker "$KURYR_DOCKER_ENGINE_SOCKET_FILE"
}
function stop_docker {
stop_process docker
# Stop process does not handle well Docker 1.12+ new multi process
# split and doesn't kill them all. Let's leverage Docker's own pidfile
sudo kill -s SIGTERM "$(cat /var/run/docker.pid)"
}
function prepare_kubernetes_files {
# Sets up the base configuration for the Kubernetes API Server and the
# Controller Manager.
docker run \
--name devstack-k8s-setup-files \
--detach \
--volume "$KURYR_HYPERKUBE_DATA_DIR:/data:rw" \
"${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \
/setup-files.sh \
"IP:${HOST_IP},DNS:kubernetes,DNS:kubernetes.default,DNS:kubernetes.default.svc,DNS:kubernetes.default.svc.cluster.local"
}
function wait_for {
local name
local url
name="$1"
url="$2"
echo -n "Waiting for $name to respond"
until curl -o /dev/null -sIf "$url"; do
echo -n "."
sleep 1
done
echo ""
}
function run_k8s_api {
# Runs Hyperkube's Kubernetes API Server
wait_for "etcd" "${KURYR_ETCD_CLIENT_URL}/v2/machines"
run_container kubernetes-api \
--net host \
--volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \
"${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \
/hyperkube apiserver \
--service-cluster-ip-range="${KURYR_K8S_CLUSTER_IP_RANGE}" \
--insecure-bind-address=0.0.0.0 \
--insecure-port="${KURYR_K8S_API_PORT}" \
--etcd-servers="${KURYR_ETCD_CLIENT_URL}" \
--admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota \
--client-ca-file=/srv/kubernetes/ca.crt \
--basic-auth-file=/srv/kubernetes/basic_auth.csv \
--min-request-timeout=300 \
--tls-cert-file=/srv/kubernetes/server.cert \
--tls-private-key-file=/srv/kubernetes/server.key \
--token-auth-file=/srv/kubernetes/known_tokens.csv \
--allow-privileged=true \
--v=2 \
--logtostderr=true
}
function run_k8s_controller_manager {
# Runs Hyperkube's Kubernetes controller manager
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
run_container kubernetes-controller-manager \
--net host \
--volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \
"${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \
/hyperkube controller-manager \
--master="$KURYR_K8S_API_URL" \
--service-account-private-key-file=/srv/kubernetes/server.key \
--root-ca-file=/srv/kubernetes/ca.crt \
--min-resync-period=3m \
--v=2 \
--logtostderr=true
}
function run_k8s_scheduler {
# Runs Hyperkube's Kubernetes scheduler
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
run_container kubernetes-scheduler \
--net host \
--volume="${KURYR_HYPERKUBE_DATA_DIR}:/srv/kubernetes:rw" \
"${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" \
/hyperkube scheduler \
--master="$KURYR_K8S_API_URL" \
--v=2 \
--logtostderr=true
}
function extract_hyperkube {
local hyperkube_container
local tmp_hyperkube_path
tmp_hyperkube_path="/tmp/hyperkube"
hyperkube_container="$(docker ps -aq \
-f ancestor="${KURYR_HYPERKUBE_IMAGE}:${KURYR_HYPERKUBE_VERSION}" | \
head -1)"
docker cp "$hyperkube_container:/hyperkube" /tmp/hyperkube
sudo install -o "$STACK_USER" -m 0555 "$tmp_hyperkube_path" \
"$KURYR_HYPERKUBE_BINARY"
}
function prepare_kubelet {
local kubelet_plugin_dir
kubelet_plugin_dir="/usr/libexec/kubernetes/kubelet-plugins/net/exec"
sudo install -o "$STACK_USER" -m 0664 -D \
"${KURYR_HOME}${kubelet_plugin_dir}/kuryr.conf" \
"${kubelet_plugin_dir}/kuryr.conf"
}
function run_k8s_kubelet {
# Runs Hyperkube's Kubernetes kubelet from the extracted binary
#
# The reason for extracting the binary and running it in from the Host
# filesystem is so that we can leverage the binding utilities that network
# vendor devstack plugins may have installed (like ovs-vsctl). Also, it
# saves us from the arduous task of setting up mounts to the official image
# adding Python and all our CNI/binding dependencies.
local command
command="$KURYR_HYPERKUBE_BINARY kubelet\
--allow-privileged=true \
--api-servers=$KURYR_K8S_API_URL \
--v=2 \
--address='0.0.0.0' \
--enable-server \
network-plugin=cni"
wait_for "Kubernetes API Server" "$KURYR_K8S_API_URL"
run_process kubelet "$command"
}
# main loop
if is_service_enabled kuryr-kubernetes; then
if [[ "$1" == "stack" && "$2" == "install" ]]; then
setup_develop "$KURYR_HOME"
elif [[ "$1" == "stack" && "$2" == "post-config" ]]; then
create_kuryr_account
configure_kuryr
if is_service_enabled docker; then
check_docker || prepare_docker
run_docker
fi
if is_service_enabled etcd; then
prepare_etcd
run_etcd
fi
get_container "$KURYR_HYPERKUBE_IMAGE" "$KURYR_HYPERKUBE_VERSION"
prepare_kubernetes_files
if is_service_enabled kubernetes-api; then
run_k8s_api
fi
if is_service_enabled kubernetes-controller-manager; then
run_k8s_controller_manager
fi
if is_service_enabled kubernetes-scheduler; then
run_k8s_scheduler
fi
if is_service_enabled kubelet; then
prepare_kubelet
extract_hyperkube
run_k8s_kubelet
fi
fi
if [[ "$1" == "stack" && "$2" == "extra" ]]; then
# FIXME(limao): When Kuryr start up, it need to detect if neutron
# support tag plugin.
#
# Kuryr will call neutron extension API to verify if neutron support
# tag. So Kuryr need to start after neutron-server finish load tag
# plugin. The process of devstack is:
# ...
# run_phase "stack" "post-config"
# ...
# start neutron-server
# ...
# run_phase "stack" "extra"
#
# If Kuryr start up in "post-config" phase, there is no way to make
# sure Kuryr can start before neutron-server, so Kuryr start in "extra"
# phase. Bug: https://bugs.launchpad.net/kuryr/+bug/1587522
run_process kuryr-kubernetes \
"python3 ${KURYR_HOME}/scripts/run_server.py \
--config-file $KURYR_CONFIG"
fi
if [[ "$1" == "unstack" ]]; then
stop_process kuryr-kubernetes
docker kill devstack-k8s-setup-files
docker rm devstack-k8s-setup-files
if is_service_enabled kubernetes-controller-manager; then
stop_container kubernetes-controller-manager
fi
if is_service_enabled kubernetes-scheduler; then
stop_container kubernetes-scheduler
fi
if is_service_enabled kubelet; then
stop_process kubelet
fi
if is_service_enabled kubernetes-api; then
stop_container kubernetes-api
fi
if is_service_enabled etcd; then
stop_container etcd
fi
stop_docker
fi
if [[ "$1" == "clean" ]]; then
if is_service_enabled etcd; then
# Cleanup Etcd for the next stacking
sudo rm -rf "$KURYR_ETCD_DATA_DIR"
fi
fi
fi
# Restore xtrace
$XTRACE

31
devstack/settings Normal file
View File

@ -0,0 +1,31 @@
KURYR_HOME=${KURYR_HOME:-$DEST/kuryr-kubernetes}
KURYR_CONFIG_DIR=${KURYR_CONFIG_DIR:-/etc/kuryr}
KURYR_CONFIG=${KURYR_CONFIG:-${KURYR_CONFIG_DIR}/kuryr.conf}
KURYR_AUTH_CACHE_DIR=${KURYR_AUTH_CACHE_DIR:-/var/cache/kuryr}
KURYR_POOL_PREFIX=${KURYR_POOL_PREFIX:-10.10.0.0/16}
KURYR_POOL_PREFIX_LEN=${KURYR_POOL_PREFIX_LEN:-24}
KURYR_DOCKER_ENGINE_PORT=${KURYR_DOCKER_ENGINE_PORT:-2375}
KURYR_DOCKER_ENGINE_SOCKET_FILE=${KURYR_DOCKER_ENGINE_SOCKET_FILE:-/var/run/docker.sock}
# Etcd
KURYR_ETCD_IMAGE=${KURYR_ETCD_IMAGE:-quay.io/coreos/etcd}
KURYR_ETCD_VERSION=${KURYR_ETCD_VERSION:-v3.0.8}
KURYR_ETCD_DATA_DIR=${KURYR_ETCD_DATA_DIR:-/var/lib/etcd}
KURYR_ETCD_ADVERTISE_CLIENT_URL=${KURYR_ETCD_ADVERTISE_CLIENT_URL:-http://${HOST_IP}:2379}
KURYR_ETCD_ADVERTISE_PEER_URL=${KURYR_ETCD_ADVERTISE_PEER_URL:-http://${HOST_IP}:2380}
KURYR_ETCD_LISTEN_CLIENT_URL=${KURYR_ETCD_LISTEN_CLIENT_URL:-http://0.0.0.0:2379}
KURYR_ETCD_LISTEN_PEER_URL=${KURYR_ETCD_LISTEN_PEER_URL:-http://0.0.0.0:2380}
# HYPERKUBE
KURYR_HYPERKUBE_IMAGE=${KURYR_HYPERKUBE_IMAGE:-gcr.io/google_containers/hyperkube-amd64}
KURYR_HYPERKUBE_VERSION=${KURYR_HYPERKUBE_VERSION:-v1.3.7}
KURYR_HYPERKUBE_DATA_DIR=${KURYR_HYPERKUBE_DATA_DIR:-/var/lib/hyperkube}
KURYR_HYPERKUBE_BINARY=${KURYR_HYPERKUBE_BINARY:-/usr/local/bin/hyperkube}
# Kubernetes
KURYR_K8S_CLUSTER_IP_RANGE=${KURYR_K8S_CLUSTER_IP_RANGE:-10.0.0.0/24}
KURYR_K8S_API_PORT=${KURYR_K8S_API_PORT:-8080}
KURYR_K8S_API_URL=${KURYR_K8S_API_URL:-http://${HOST_IP}:${KURYR_K8S_API_PORT}}

17
scripts/run_server.py Normal file
View File

@ -0,0 +1,17 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from kuryr_kubernetes import server
server.start()

View File

@ -0,0 +1,4 @@
{
"name": "kuryr",
"type": "kuryr"
}