Optional tool to live-migrate VMs from K8s node
This little daemon will listen in a pod for unschedulable=true events from Kubernetes and trigger live-migration for all OpenStack instances (VMs) located on this node (preassumbly Compute). It's an independent, PoC feature not deployed by default. Change-Id: I7fd78efc9cfa33b0ab1629541c15af4538c4fe9e
This commit is contained in:
Marek Zawadzki
parent
190866ca94
commit
992e4ec2a0
|
|
@ -0,0 +1,11 @@
|
|||
FROM DOCKER_REGISTRY_HOST_PORT_CHANGE_ME/DOCKER_REGISTRY_NAMESPACE_CHANGE_ME/openstack-base
|
||||
MAINTAINER MOS Microservices <mos-microservices@mirantis.com>
|
||||
|
||||
RUN useradd --user-group --create-home --home-dir /var/lib/k8s-node-evacuator k8s-node-evacuator \
|
||||
&& usermod -a -G microservices k8s-node-evacuator
|
||||
|
||||
COPY k8s-node-evacuator.py /opt/ccp/bin
|
||||
|
||||
USER k8s-node-evacuator
|
||||
|
||||
CMD python /opt/ccp/bin/k8s-node-evacuator.py
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This script builds Docker container and deploys in a Kubernetes pod
|
||||
# a simple daemon to handle Kubernetes node evacuation.
|
||||
# When "unschedulable" event is detected nova-compute service is disabled
|
||||
# on the node and all VMs are live-migrated away.
|
||||
|
||||
set -ex
|
||||
|
||||
function usage {
|
||||
set +x
|
||||
local base_name=$(basename $0)
|
||||
echo "Usage (all parameters are optional):"
|
||||
echo " $base_name -s <k8s_api_server_address>"
|
||||
echo " $base_name -n <k8s namespace to deploy pod in>"
|
||||
echo " $base_name -i <node_to_deploy_on>"
|
||||
echo " $base_name -r <Docker registry host:port>"
|
||||
echo " $base_name -k <Docker registry namsespace>"
|
||||
set -x
|
||||
}
|
||||
|
||||
NAMESPACE_OPT=" --namespace kube-system"
|
||||
DOCKER_REGISTRY_HOST_PORT="127.0.0.1:31500"
|
||||
DOCKER_REGISTRY_NAMESPACE="ccp"
|
||||
|
||||
while getopts "s:n:i:r:k:" opt; do
|
||||
case $opt in
|
||||
"s" )
|
||||
SRV_OPT=" -s ${OPTARG}"
|
||||
;;
|
||||
"n" )
|
||||
NAMESPACE_OPT=" --namespace ${OPTARG}"
|
||||
;;
|
||||
"i" )
|
||||
NODE="${OPTARG}"
|
||||
;;
|
||||
"r" )
|
||||
DOCKER_REGISTRY_HOST_PORT="${OPTARG}"
|
||||
;;
|
||||
"k" )
|
||||
DOCKER_REGISTRY_NAMESPACE="${OPTARG}"
|
||||
;;
|
||||
* )
|
||||
usage
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
which kubectl 1> /dev/null
|
||||
function kube_cmd {
|
||||
kubectl ${SRV_OPT} ${NAMESPACE_OPT} "$@"
|
||||
}
|
||||
|
||||
if [ -z "${NODE}" ]; then
|
||||
NODE=$(kube_cmd get nodes -o template --template="{{ with index .items 0 }}{{ .metadata.name }}{{ end }}")
|
||||
echo "K8S node was not specified, using ${NODE}"
|
||||
fi
|
||||
|
||||
kube_cmd label node "${NODE}" app=k8s-node-evacuator --overwrite
|
||||
|
||||
workdir=$(dirname $0)
|
||||
pushd ${workdir}
|
||||
|
||||
echo "Preparing Dockerfile:"
|
||||
sed -i "s/DOCKER_REGISTRY_HOST_PORT_CHANGE_ME/${DOCKER_REGISTRY_HOST_PORT}/" Dockerfile
|
||||
sed -i "s/DOCKER_REGISTRY_NAMESPACE_CHANGE_ME/${DOCKER_REGISTRY_NAMESPACE}/" Dockerfile
|
||||
|
||||
echo "Building Docker image:"
|
||||
docker build -t k8s-node-evacuator .
|
||||
docker tag k8s-node-evacuator "${DOCKER_REGISTRY_HOST_PORT}"/"${DOCKER_REGISTRY_NAMESPACE}"/k8s-node-evacuator
|
||||
docker push "${DOCKER_REGISTRY_HOST_PORT}"/"${DOCKER_REGISTRY_NAMESPACE}"/k8s-node-evacuator
|
||||
|
||||
echo "Setting variables in pod yaml file:"
|
||||
POD_YAML_FILE="k8s-node-evacuator-pod.yaml"
|
||||
sed -i "s/DOCKER_REGISTRY_HOST_PORT_CHANGE_ME/${DOCKER_REGISTRY_HOST_PORT}/" ${POD_YAML_FILE}
|
||||
sed -i "s/DOCKER_REGISTRY_NAMESPACE_CHANGE_ME/${DOCKER_REGISTRY_NAMESPACE}/" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_USER_DOMAIN_NAME_CHANGE_ME/${OS_USER_DOMAIN_NAME}/" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_PROJECT_NAME_CHANGE_ME/${OS_PROJECT_NAME}/" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_IDENTITY_API_VERSION_CHANGE_ME/\"${OS_IDENTITY_API_VERSION}\"/" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_PASSWORD_CHANGE_ME/${OS_PASSWORD}/" ${POD_YAML_FILE}
|
||||
sed -i "s#OS_AUTH_URL_CHANGE_ME#${OS_AUTH_URL}#" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_USERNAME_CHANGE_ME/${OS_USERNAME}/" ${POD_YAML_FILE}
|
||||
sed -i "s/OS_PROJECT_DOMAIN_NAME_CHANGE_ME/${OS_PROJECT_DOMAIN_NAME}/" ${POD_YAML_FILE}
|
||||
|
||||
echo "Deploying k8s-node-evacuator pod:"
|
||||
kube_cmd create -f k8s-node-evacuator-pod.yaml
|
||||
|
||||
# Waiting for status Running
|
||||
while true; do
|
||||
echo "Waiting for 'Running' state..."
|
||||
cont_running=$(kube_cmd get pod k8s-node-evacuator -o template --template="{{ .status.phase }}")
|
||||
if [ "${cont_running}" == "Running" ]; then
|
||||
break
|
||||
fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
echo "k8s-node-evacuator is ready."
|
||||
|
|
@ -0,0 +1,29 @@
|
|||
apiVersion: v1
|
||||
kind: Pod
|
||||
metadata:
|
||||
name: k8s-node-evacuator
|
||||
labels:
|
||||
app: k8s-node-evacuator
|
||||
spec:
|
||||
hostNetwork: false
|
||||
nodeSelector:
|
||||
app: k8s-node-evacuator
|
||||
containers:
|
||||
- name: k8s-node-evacuator
|
||||
image: DOCKER_REGISTRY_HOST_PORT_CHANGE_ME/DOCKER_REGISTRY_NAMESPACE_CHANGE_ME/k8s-node-evacuator
|
||||
imagePullPolicy: Always
|
||||
env:
|
||||
- name: OS_USER_DOMAIN_NAME
|
||||
value: OS_USER_DOMAIN_NAME_CHANGE_ME
|
||||
- name: OS_PROJECT_NAME
|
||||
value: OS_PROJECT_NAME_CHANGE_ME
|
||||
- name: OS_IDENTITY_API_VERSION
|
||||
value: OS_IDENTITY_API_VERSION_CHANGE_ME
|
||||
- name: OS_PASSWORD
|
||||
value: OS_PASSWORD_CHANGE_ME
|
||||
- name: OS_AUTH_URL
|
||||
value: OS_AUTH_URL_CHANGE_ME
|
||||
- name: OS_USERNAME
|
||||
value: OS_USERNAME_CHANGE_ME
|
||||
- name: OS_PROJECT_DOMAIN_NAME
|
||||
value: OS_PROJECT_DOMAIN_NAME_CHANGE_ME
|
||||
|
|
@ -0,0 +1,81 @@
|
|||
import json
|
||||
import logging
|
||||
import os
|
||||
import requests
|
||||
import time
|
||||
|
||||
from six.moves import _thread as thread
|
||||
|
||||
from keystoneauth1 import loading
|
||||
from keystoneauth1 import session
|
||||
from novaclient import client
|
||||
|
||||
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
LOG = logging.getLogger(__file__)
|
||||
|
||||
|
||||
def nova_live_migrate(node):
|
||||
loader = loading.get_plugin_loader('password')
|
||||
auth = loader.load_from_options(
|
||||
auth_url=os.environ["OS_AUTH_URL"],
|
||||
username=os.environ["OS_USERNAME"],
|
||||
password=os.environ["OS_PASSWORD"],
|
||||
user_domain_name=os.environ["OS_USER_DOMAIN_NAME"],
|
||||
project_domain_name=os.environ["OS_PROJECT_DOMAIN_NAME"],
|
||||
project_name=os.environ["OS_PROJECT_NAME"])
|
||||
OS_COMPUTE_API_VERSION = "2"
|
||||
sess = session.Session(auth=auth)
|
||||
nova = client.Client(OS_COMPUTE_API_VERSION, session=sess)
|
||||
|
||||
LOG.info("Disabling nova-compute service on: %s", node)
|
||||
nova.services.disable(node, "nova-compute")
|
||||
|
||||
for server in nova.servers.list(search_opts={'host': node}):
|
||||
LOG.info("Live-migrating instance: %s from node: %s", server.name,
|
||||
node)
|
||||
server.live_migrate(block_migration=True)
|
||||
thread.start_new_thread(live_migration_watcher_thread, (nova, node))
|
||||
|
||||
|
||||
def get_http_header(TOKEN_FILE):
|
||||
try:
|
||||
token = file(TOKEN_FILE, 'r').read()
|
||||
except IOError:
|
||||
exit('Unable to open token file')
|
||||
header = {'Authorization': "Bearer {}".format(token)}
|
||||
return header
|
||||
|
||||
|
||||
def k8s_watcher():
|
||||
TOKEN_FILE = "/var/run/secrets/kubernetes.io/serviceaccount/token"
|
||||
CA_CERT = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
|
||||
API_URL = "https://" + os.environ["KUBERNETES_SERVICE_HOST"] + ":" + \
|
||||
os.environ["KUBERNETES_PORT_443_TCP_PORT"] + "/api/v1/events"
|
||||
LOG.debug("Listening for events from: %s", API_URL)
|
||||
http_header = get_http_header(TOKEN_FILE)
|
||||
response = requests.get(API_URL, headers=http_header,
|
||||
verify=CA_CERT, params={'watch': 'true'},
|
||||
stream=True)
|
||||
for line in response.iter_lines():
|
||||
event = json.loads(line)
|
||||
reason = event["object"]["reason"]
|
||||
if reason == "NodeNotSchedulable":
|
||||
node = event["object"]["involvedObject"]["name"]
|
||||
LOG.info("Detected event: %s for node: %s", reason, node)
|
||||
nova_live_migrate(node)
|
||||
|
||||
|
||||
def live_migration_watcher_thread(nova, node):
|
||||
while len(nova.servers.list(search_opts={'host': node})):
|
||||
LOG.info("Waiting for instances from %s to live-migrate...", node)
|
||||
time.sleep(5)
|
||||
LOG.info("All instances from node %s has been live-migrated away.", node)
|
||||
|
||||
|
||||
def main():
|
||||
k8s_watcher()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Loading…
Reference in New Issue