openstack
/
magnum
Code Issues Proposed changes

Add TLS support in heat kubernetes 

This patch modifies template to generate certificates and configure TLS
settings for kube-apiserver/kubelet/kube-proxy.

Co-Authored-By: Andrew Melton <andrew.melton@rackspace.com>
Partially-Implements: bp secure-kubernetes
Change-Id: I76b0f91f0c44f9880980e35c6b8856ea48ed3ce1
This commit is contained in:
OTSUKA, Yuanying 2015-09-14 18:50:40 +09:00
parent 3ab11cb16b
commit 30b3d99d5c
21 changed files with 601 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
etc/magnum/magnum.conf.sample
View File

@ -720,22 +720,6 @@
#admin_tenant_name = admin
[kubernetes]
#
# From magnum
#
# Default protocol of k8s master endpoint (http or https). (string
# value)
#k8s_protocol = http
# Default port of the k8s master endpoint. (unknown type)
# Minimum value: 1
# Maximum value: 65535
#k8s_port = 8080
[magnum_client]
#

6
magnum/conductor/handlers/bay_conductor.py
View File

@ -215,9 +215,9 @@ class HeatPoller(object):
self.context = self.openstack_client.context
self.bay = bay
self.attempts = 0
baymodel = conductor_utils.retrieve_baymodel(self.context, bay)
self.baymodel = conductor_utils.retrieve_baymodel(self.context, bay)
self.template_def = TDef.get_template_definition(
'vm', baymodel.cluster_distro, baymodel.coe)
'vm', self.baymodel.cluster_distro, self.baymodel.coe)
def poll_and_check(self):
# TODO(yuanying): temporary implementation to update api_address,
@ -238,7 +238,7 @@ class HeatPoller(object):
raise loopingcall.LoopingCallDone()
if (stack.stack_status in [bay_status.CREATE_COMPLETE,
bay_status.UPDATE_COMPLETE]):
self.template_def.update_outputs(stack, self.bay)
self.template_def.update_outputs(stack, self.baymodel, self.bay)
self.bay.status = stack.stack_status
self.bay.status_reason = stack.stack_status_reason

24
magnum/conductor/k8s_api.py
View File

@ -12,27 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from oslo_config import cfg
from magnum.common import config
from magnum.common.pythonk8sclient.swagger_client import api_client
from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
from magnum.conductor import utils
from magnum.i18n import _
kubernetes_opts = [
cfg.StrOpt('k8s_protocol',
default='http',
help=_('Default protocol of k8s master endpoint '
'(http or https).')),
cfg.Opt('k8s_port',
type=config.PORT_TYPE,
default=8080,
help=_('Default port of the k8s master endpoint.')),
]
cfg.CONF.register_opts(kubernetes_opts, group='kubernetes')
class K8sAPI(apiv_api.ApivApi):
@ -51,11 +33,7 @@ class K8sAPI(apiv_api.ApivApi):
if hasattr(obj, 'bay_uuid'):
obj = utils.retrieve_bay(context, obj)
params = {
'k8s_protocol': cfg.CONF.kubernetes.k8s_protocol,
'api_address': obj.api_address
}
return "%(k8s_protocol)s://%(api_address)s" % params
return obj.api_address
def create_k8s_api(context, obj):

53
magnum/conductor/template_definition.py
View File

@ -29,6 +29,8 @@ from magnum.i18n import _LW
LOG = logging.getLogger(__name__)
KUBE_SECURE_PORT = '6443'
KUBE_INSECURE_PORT = '8080'
template_def_opts = [
cfg.StrOpt('k8s_atomic_template_path',
@ -133,7 +135,7 @@ class OutputMapping(object):
self.bay_attr = bay_attr
self.heat_output = heat_output
def set_output(self, stack, bay):
def set_output(self, stack, baymodel, bay):
if self.bay_attr is None:
return
@ -278,7 +280,8 @@ class TemplateDefinition(object):
self.param_mappings.append(param)
def add_output(self, *args, **kwargs):
output = OutputMapping(*args, **kwargs)
mapping_type = kwargs.pop('mapping_type', OutputMapping)
output = mapping_type(*args, **kwargs)
self.output_mappings.append(output)
def get_output(self, *args, **kwargs):
@ -325,9 +328,9 @@ class TemplateDefinition(object):
return None
def update_outputs(self, stack, bay):
def update_outputs(self, stack, baymodel, bay):
for output in self.output_mappings:
output.set_output(stack, bay)
output.set_output(stack, baymodel, bay)
@abc.abstractproperty
def template_path(self):
@ -362,6 +365,28 @@ class BaseTemplateDefinition(TemplateDefinition):
pass
class K8sApiAddressOutputMapping(OutputMapping):
def set_output(self, stack, baymodel, bay):
# TODO(yuanying): port number is hardcoded, this will be fix
protocol = 'https'
port = KUBE_SECURE_PORT
if baymodel.tls_disabled:
protocol = 'http'
port = KUBE_INSECURE_PORT
output_value = self.get_output_value(stack)
params = {
'protocol': protocol,
'address': output_value,
'port': port,
}
output_value = "%(protocol)s://%(address)s:%(port)s" % params
if output_value is not None:
setattr(bay, self.bay_attr, output_value)
class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
"""Kubernetes template for a Fedora Atomic VM."""
@ -373,6 +398,9 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
def __init__(self):
super(AtomicK8sTemplateDefinition, self).__init__()
self.add_parameter('bay_uuid',
bay_attr='uuid',
param_type=str)
self.add_parameter('master_flavor',
baymodel_attr='master_flavor_id')
self.add_parameter('minion_flavor',
@ -390,13 +418,13 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
required=True)
self.add_parameter('network_driver',
baymodel_attr='network_driver')
# TODO(yuanying): Add below lines if apiserver_port parameter
# is supported
# self.add_parameter('apiserver_port',
# baymodel_attr='apiserver_port')
self.add_parameter('tls_disabled',
baymodel_attr='tls_disabled',
required=True)
self.add_output('api_address',
bay_attr='api_address')
bay_attr='api_address',
mapping_type=K8sApiAddressOutputMapping)
self.add_output('kube_minions',
bay_attr=None)
self.add_output('kube_minions_external',
@ -433,6 +461,13 @@ class AtomicK8sTemplateDefinition(BaseTemplateDefinition):
extra_params['auth_url'] = context.auth_url.replace("v3", "v2")
extra_params['username'] = context.user_name
extra_params['tenant_name'] = context.tenant
extra_params['user_token'] = context.auth_token
osc = clients.OpenStackClients(context)
extra_params['magnum_url'] = osc.magnum_url()
if baymodel.tls_disabled:
extra_params['loadbalancing_protocol'] = 'HTTP'
extra_params['kubernetes_port'] = 8080
for label in label_list:
extra_params[label] = baymodel.labels.get(label)

4
magnum/opts.py
View File

@ -55,7 +55,5 @@ def list_opts():
('certificates',
itertools.chain(magnum.common.cert_manager.cert_manager_opts,
local_cert_manager.local_cert_manager_opts,
)),
('kubernetes',
magnum.conductor.k8s_api.kubernetes_opts),
))
]

22
magnum/templates/heat-kubernetes/fragments/configure-kubernetes-master.sh
View File

@ -11,13 +11,31 @@ sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
' /etc/kubernetes/config
KUBE_API_ARGS="--runtime_config=api/all=true"
if [ "$TLS_DISABLED" == "True" ]; then
KUBE_API_ADDRESS="--insecure-bind-address=0.0.0.0 --insecure-port=$KUBE_API_PORT"
else
KUBE_API_ADDRESS="--bind_address=0.0.0.0 --secure-port=$KUBE_API_PORT"
# insecure port is used internaly
KUBE_API_ADDRESS="$KUBE_API_ADDRESS --insecure-port=8080"
KUBE_API_ARGS="$KUBE_API_ARGS --tls_cert_file=/srv/kubernetes/server.crt"
KUBE_API_ARGS="$KUBE_API_ARGS --tls_private_key_file=/srv/kubernetes/server.key"
KUBE_API_ARGS="$KUBE_API_ARGS --client_ca_file=/srv/kubernetes/ca.crt"
fi
sed -i '
/^KUBE_API_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
/^KUBE_API_ADDRESS=/ s/=.*/='"${KUBE_API_ADDRESS}"'/
/^KUBE_SERVICE_ADDRESSES=/ s|=.*|="--service-cluster-ip-range='"$PORTAL_NETWORK_CIDR"'"|
/^KUBE_API_ARGS=/ s/KUBE_API_ARGS./#Uncomment the following line to disable Load Balancer feature\nKUBE_API_ARGS="--runtime_config=api\/all=true"\n#Uncomment the following line to enable Load Balancer feature\n#KUBE_API_ARGS="--runtime_config=api\/all=true --cloud_config=\/etc\/sysconfig\/kube_openstack_config --cloud_provider=openstack"/
/^KUBE_API_ARGS=/ s/KUBE_API_ARGS.//
/^KUBE_ETCD_SERVERS=/ s/=.*/="--etcd_servers=http:\/\/127.0.0.1:2379"/
/^KUBE_ADMISSION_CONTROL=/ s/=.*/=""/
' /etc/kubernetes/apiserver
cat << _EOC_ >> /etc/kubernetes/apiserver
#Uncomment the following line to disable Load Balancer feature
KUBE_API_ARGS="$KUBE_API_ARGS"
#Uncomment the following line to enable Load Balancer feature
#KUBE_API_ARGS="$KUBE_API_ARGS --cloud_config=/etc/sysconfig/kube_openstack_config --cloud_provider=openstack"
_EOC_
sed -i '
/^KUBELET_ADDRESSES=/ s/=.*/="--machines='""'"/

20
magnum/templates/heat-kubernetes/fragments/configure-kubernetes-minion.sh
View File

@ -5,22 +5,31 @@
echo "configuring kubernetes (minion)"
ETCD_SERVER_IP=${ETCD_SERVER_IP:-$KUBE_MASTER_IP}
KUBE_PROTOCOL="https"
KUBE_CONFIG=""
if [ "$TLS_DISABLED" == "True" ]; then
KUBE_PROTOCOL="http"
else
KUBE_CONFIG="--kubeconfig=/srv/kubernetes/kubeconfig.yaml"
fi
KUBE_MASTER_URI="$KUBE_PROTOCOL://$KUBE_MASTER_IP:$KUBE_API_PORT"
sed -i '
/^KUBE_ALLOW_PRIV=/ s/=.*/="--allow_privileged='"$KUBE_ALLOW_PRIV"'"/
/^KUBE_ETCD_SERVERS=/ s|=.*|="--etcd_servers=http://'"$ETCD_SERVER_IP"':2379"|
/^KUBE_MASTER=/ s|=.*|="--master=http://'"$KUBE_MASTER_IP"':8080"|
/^KUBE_MASTER=/ s|=.*|="--master='"$KUBE_MASTER_URI"'"|
' /etc/kubernetes/config
sed -i '
/^KUBELET_ADDRESS=/ s/=.*/="--address=0.0.0.0"/
/^KUBELET_HOSTNAME=/ s/=.*/=""/
/^KUBELET_API_SERVER=/ s|=.*|="--api_servers=http://'"$KUBE_MASTER_IP"':8080"|
/^KUBELET_API_SERVER=/ s|=.*|="--api_servers='"$KUBE_MASTER_URI"'"|
/^KUBELET_ARGS=/ s|=.*|='"$KUBE_CONFIG"'|
' /etc/kubernetes/kubelet
sed -i '
/^KUBE_MASTER=/ s/=.*/="--master='"$KUBE_MASTER_IP"':8080"/
' /etc/kubernetes/apiserver
/^KUBE_PROXY_ARGS=/ s|=.*|='"$KUBE_CONFIG"'|
' /etc/kubernetes/proxy
if [ "$NETWORK_DRIVER" == "flannel" ]; then
sed -i '
@ -29,10 +38,9 @@ sed -i '
fi
cat >> /etc/environment <<EOF
KUBERNETES_MASTER=http://$KUBE_MASTER_IP:8080
KUBERNETES_MASTER=$KUBE_MASTER_URI
EOF
sed -i '/^DOCKER_STORAGE_OPTIONS=/ s/=.*/=--storage-driver devicemapper --storage-opt dm.fs=xfs --storage-opt dm.thinpooldev=\/dev\/mapper\/docker-docker--pool --storage-opt dm.use_deferred_removal=true/' /etc/sysconfig/docker-storage
hostname `hostname | sed 's/.novalocal//'`

82
magnum/templates/heat-kubernetes/fragments/make-cert-client.sh Normal file
View File

@ -0,0 +1,82 @@
#!/bin/sh
# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
. /etc/sysconfig/heat-params
if [ "$TLS_DISABLED" == "True" ]; then
exit 0
fi
cert_dir=/srv/kubernetes
cert_conf_dir=${cert_dir}/conf
cert_group=root
mkdir -p "$cert_dir"
mkdir -p "$cert_conf_dir"
CA_CERT=$cert_dir/ca.crt
CLIENT_CERT=$cert_dir/client.crt
CLIENT_CSR=$cert_dir/client.csr
CLIENT_KEY=$cert_dir/client.key
# Get CA certificate for this bay
curl -X GET \
-H "X-Auth-Token: $USER_TOKEN" \
$MAGNUM_URL/certificates/$BAY_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > $CA_CERT
# Create config for client's csr
cat > ${cert_conf_dir}/client.conf <<EOF
[req]
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
[req_distinguished_name]
CN = kubernetes.invalid
[req_ext]
keyUsage=critical,digitalSignature,keyEncipherment
extendedKeyUsage=clientAuth
subjectAltName=dirName:kubelet,dirName:kubeproxy
[kubelet]
CN=kubelet
[kubeproxy]
CN=kube-proxy
EOF
# Generate client's private key and csr
openssl genrsa -out "${CLIENT_KEY}" 4096
openssl req -new -days 1000 \
-key "${CLIENT_KEY}" \
-out "${CLIENT_CSR}" \
-reqexts req_ext \
-config "${cert_conf_dir}/client.conf"
# Send csr to Magnum to have it signed
csr_req=$(python -c "import json; fp = open('${CLIENT_CSR}'); print json.dumps({'bay_uuid': '$BAY_UUID', 'csr': fp.read()}); fp.close()")
curl -X POST \
-H "X-Auth-Token: $USER_TOKEN" \
-H "Content-Type: application/json" \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CLIENT_CERT}
sed -i '
s|CA_CERT|'"$CA_CERT"'|
s|CLIENT_CERT|'"$CLIENT_CERT"'|
s|CLIENT_KEY|'"$CLIENT_KEY"'|
' /srv/kubernetes/kubeconfig.yaml

78
magnum/templates/heat-kubernetes/fragments/make-cert.sh Normal file
View File

@ -0,0 +1,78 @@
#!/bin/sh
# Copyright 2014 The Kubernetes Authors All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
. /etc/sysconfig/heat-params
if [ "$TLS_DISABLED" == "True" ]; then
exit 0
fi
cert_ip=$(curl -s http://169.254.169.254/latest/meta-data/public-ipv4)
sans="IP:${cert_ip},IP:${KUBE_API_PUBLIC_ADDRESS},IP:${KUBE_API_PRIVATE_ADDRESS},IP:127.0.0.1"
MASTER_HOSTNAME=${MASTER_HOSTNAME:-}
if [[ -n "${MASTER_HOSTNAME}" ]]; then
sans="${sans},DNS:${MASTER_HOSTNAME}"
fi
cert_dir=/srv/kubernetes
cert_conf_dir=${cert_dir}/conf
cert_group=root
mkdir -p "$cert_dir"
mkdir -p "$cert_conf_dir"
CA_CERT=$cert_dir/ca.crt
SERVER_CERT=$cert_dir/server.crt
SERVER_CSR=$cert_dir/server.csr
SERVER_KEY=$cert_dir/server.key
# Get CA certificate for this bay
curl -X GET \
-H "X-Auth-Token: $USER_TOKEN" \
$MAGNUM_URL/certificates/$BAY_UUID | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${CA_CERT}
# Create config for server's csr
cat > ${cert_conf_dir}/server.conf <<EOF
[req]
distinguished_name = req_distinguished_name
req_extensions = req_ext
prompt = no
[req_distinguished_name]
CN = kubernetes.invalid
[req_ext]
subjectAltName = ${sans}
extendedKeyUsage = clientAuth,serverAuth
EOF
# Generate server's private key and csr
openssl genrsa -out "${SERVER_KEY}" 4096
openssl req -new -days 1000 \
-key "${SERVER_KEY}" \
-out "${SERVER_CSR}" \
-reqexts req_ext \
-config "${cert_conf_dir}/server.conf"
# Send csr to Magnum to have it signed
csr_req=$(python -c "import json; fp = open('${SERVER_CSR}'); print json.dumps({'bay_uuid': '$BAY_UUID', 'csr': fp.read()}); fp.close()")
curl -X POST \
-H "X-Auth-Token: $USER_TOKEN" \
-H "Content-Type: application/json" \
-d "$csr_req" \
$MAGNUM_URL/certificates | python -c 'import sys, json; print json.load(sys.stdin)["pem"]' > ${SERVER_CERT}

7
magnum/templates/heat-kubernetes/fragments/write-heat-params-master.yaml
View File

@ -5,6 +5,9 @@ write_files:
owner: "root:root"
permissions: "0644"
content: |
KUBE_API_PUBLIC_ADDRESS="$KUBE_API_PUBLIC_ADDRESS"
KUBE_API_PRIVATE_ADDRESS="$KUBE_API_PRIVATE_ADDRESS"
KUBE_API_PORT="$KUBE_API_PORT"
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
NETWORK_DRIVER="$NETWORK_DRIVER"
FLANNEL_NETWORK_CIDR="$FLANNEL_NETWORK_CIDR"
@ -17,3 +20,7 @@ write_files:
PASSWORD="$PASSWORD"
TENANT_NAME="$TENANT_NAME"
CLUSTER_SUBNET="$CLUSTER_SUBNET"
TLS_DISABLED="$TLS_DISABLED"
BAY_UUID="$BAY_UUID"
USER_TOKEN="$USER_TOKEN"
MAGNUM_URL="$MAGNUM_URL"

5
magnum/templates/heat-kubernetes/fragments/write-heat-params.yaml
View File

@ -7,6 +7,7 @@ write_files:
content: |
KUBE_ALLOW_PRIV="$KUBE_ALLOW_PRIV"
KUBE_MASTER_IP="$KUBE_MASTER_IP"
KUBE_API_PORT="$KUBE_API_PORT"
ETCD_SERVER_IP="$ETCD_SERVER_IP"
DOCKER_VOLUME="$DOCKER_VOLUME"
NETWORK_DRIVER="$NETWORK_DRIVER"
@ -21,3 +22,7 @@ write_files:
REGISTRY_CONTAINER="$REGISTRY_CONTAINER"
REGISTRY_INSECURE="$REGISTRY_INSECURE"
REGISTRY_CHUNKSIZE="$REGISTRY_CHUNKSIZE"
TLS_DISABLED="$TLS_DISABLED"
BAY_UUID="$BAY_UUID"
USER_TOKEN="$USER_TOKEN"
MAGNUM_URL="$MAGNUM_URL"

24
magnum/templates/heat-kubernetes/fragments/write-kubeconfig.yaml Normal file
View File

@ -0,0 +1,24 @@
#cloud-config
merge_how: dict(recurse_array)+list(append)
write_files:
- path: /srv/kubernetes/kubeconfig.yaml
owner: "root:root"
permissions: "0644"
content: |
apiVersion: v1
kind: Config
users:
- name: kubeclient
user:
client-certificate: CLIENT_CERT
client-key: CLIENT_KEY
clusters:
- name: kubernetes
cluster:
certificate-authority: CA_CERT
contexts:
- context:
cluster: kubernetes
user: kubeclient
name: service-account-context
current-context: service-account-context

51
magnum/templates/heat-kubernetes/kubecluster.yaml
View File

@ -198,6 +198,39 @@ parameters:
description: >
tenant name
loadbalancing_protocol:
type: string
description: >
The protocol which is used for load balancing. If you want to change
tls_disabled option to 'True', please change this to "HTTP".
default: TCP
constraints:
- allowed_values: ["TCP", "HTTP"]
tls_disabled:
type: boolean
description: whether or not to disable TLS
default: False
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
default: 6443
user_token:
type: string
description: token used for communicating back to Magnum for TLS certs
bay_uuid:
type: string
description: identifier for the bay this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
resources:
######################################################################
@ -249,12 +282,12 @@ resources:
api_pool:
type: OS::Neutron::Pool
properties:
protocol: HTTP
protocol: {get_param: loadbalancing_protocol}
monitors: [{get_resource: api_monitor}]
subnet: {get_resource: fixed_subnet}
lb_method: ROUND_ROBIN
vip:
protocol_port: 8080
protocol_port: {get_param: kubernetes_port}
api_pool_floating:
type: OS::Neutron::FloatingIP
@ -295,6 +328,8 @@ resources:
resource_def:
type: kubemaster.yaml
properties:
api_public_address: {get_attr: [api_pool_floating, floating_ip_address]}
api_private_address: {get_attr: [api_pool, vip, address]}
ssh_key_name: {get_param: ssh_key_name}
server_image: {get_param: server_image}
master_flavor: {get_param: master_flavor}
@ -307,6 +342,9 @@ resources:
flannel_use_vxlan: {get_param: flannel_use_vxlan}
portal_network_cidr: {get_param: portal_network_cidr}
discovery_url: {get_param: discovery_url}
user_token: {get_param: user_token}
bay_uuid: {get_param: bay_uuid}
magnum_url: {get_param: magnum_url}
fixed_network: {get_resource: fixed_network}
fixed_subnet: {get_resource: fixed_subnet}
api_pool_id: {get_resource: api_pool}
@ -315,6 +353,8 @@ resources:
username: {get_param: username}
password: {get_param: password}
tenant_name: {get_param: tenant_name}
kubernetes_port: {get_param: kubernetes_port}
tls_disabled: {get_param: tls_disabled}
######################################################################
#
@ -356,13 +396,18 @@ resources:
registry_container: {get_param: registry_container}
registry_insecure: {get_param: registry_insecure}
registry_chunksize: {get_param: registry_chunksize}
user_token: {get_param: user_token}
bay_uuid: {get_param: bay_uuid}
magnum_url: {get_param: magnum_url}
kubernetes_port: {get_param: kubernetes_port}
tls_disabled: {get_param: tls_disabled}
outputs:
api_address:
value:
str_replace:
template: api_ip_address:8080
template: api_ip_address
params:
api_ip_address: {get_attr: [api_pool_floating, floating_ip_address]}
description: >

50
magnum/templates/heat-kubernetes/kubemaster.yaml
View File

@ -63,7 +63,37 @@ parameters:
description: >
Discovery URL used for bootstrapping the etcd cluster.
tls_disabled:
type: boolean
description: whether or not to enable TLS
default: False
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
default: 6443
user_token:
type: string
description: token used for communicating back to Magnum for TLS certs
bay_uuid:
type: string
description: identifier for the bay this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
# The following are all generated in the parent template.
api_public_address:
type: string
description: Public IP address of the Kubernetes master server.
api_private_address:
type: string
description: Private IP address of the Kubernetes master server.
fixed_network:
type: string
description: Network from which to allocate fixed addresses.
@ -135,8 +165,8 @@ resources:
port_range_min: 7080
port_range_max: 7080
- protocol: tcp
port_range_min: 8080
port_range_max: 8080
port_range_min: {get_param: kubernetes_port}
port_range_max: {get_param: kubernetes_port}
- protocol: tcp
port_range_min: 2379
port_range_max: 2379
@ -158,6 +188,9 @@ resources:
str_replace:
template: {get_file: fragments/write-heat-params-master.yaml}
params:
"$KUBE_API_PUBLIC_ADDRESS": {get_param: api_public_address}
"$KUBE_API_PRIVATE_ADDRESS": {get_param: api_private_address}
"$KUBE_API_PORT": {get_param: kubernetes_port}
"$KUBE_ALLOW_PRIV": {get_param: kube_allow_priv}
"$NETWORK_DRIVER": {get_param: network_driver}
"$FLANNEL_NETWORK_CIDR": {get_param: flannel_network_cidr}
@ -170,6 +203,16 @@ resources:
"$PASSWORD": {get_param: password}
"$TENANT_NAME": {get_param: tenant_name}
"$CLUSTER_SUBNET": {get_param: fixed_subnet}
"$TLS_DISABLED": {get_param: tls_disabled}
"$BAY_UUID": {get_param: bay_uuid}
"$USER_TOKEN": {get_param: user_token}
"$MAGNUM_URL": {get_param: magnum_url}
make_cert:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/make-cert.sh}
configure_etcd:
type: OS::Heat::SoftwareConfig
@ -246,6 +289,7 @@ resources:
- config: {get_resource: configure_etcd}
- config: {get_resource: kube_user}
- config: {get_resource: write_kube_os_config}
- config: {get_resource: make_cert}
- config: {get_resource: configure_kubernetes}
- config: {get_resource: enable_services}
- config: {get_resource: write_network_config}
@ -291,7 +335,7 @@ resources:
properties:
pool_id: {get_param: api_pool_id}
address: {get_attr: [kube_master_eth0, fixed_ips, 0, ip_address]}
protocol_port: 8080
protocol_port: {get_param: kubernetes_port}
etcd_pool_member:
type: OS::Neutron::PoolMember

44
magnum/templates/heat-kubernetes/kubeminion.yaml
View File

@ -40,6 +40,31 @@ parameters:
storage
default: 25
tls_disabled:
type: boolean
description: whether or not to enable TLS
default: False
kubernetes_port:
type: number
description: >
The port which are used by kube-apiserver to provide Kubernetes
service.
default: 6443
user_token:
type: string
description: token used for communicating back to Magnum for TLS certs
bay_uuid:
type: string
description: identifier for the bay this template is generating
magnum_url:
type: string
description: endpoint to retrieve TLS certs from
# The following are all generated in the parent template.
kube_master_ip:
type: string
@ -156,6 +181,7 @@ resources:
params:
$KUBE_ALLOW_PRIV: {get_param: kube_allow_priv}
$KUBE_MASTER_IP: {get_param: kube_master_ip}
$KUBE_API_PORT: {get_param: kubernetes_port}
$ETCD_SERVER_IP: {get_param: etcd_server_ip}
$DOCKER_VOLUME: {get_resource: docker_volume}
$NETWORK_DRIVER: {get_param: network_driver}
@ -170,6 +196,22 @@ resources:
$REGISTRY_CONTAINER: {get_param: registry_container}
$REGISTRY_INSECURE: {get_param: registry_insecure}
$REGISTRY_CHUNKSIZE: {get_param: registry_chunksize}
$TLS_DISABLED: {get_param: tls_disabled}
$BAY_UUID: {get_param: bay_uuid}
$USER_TOKEN: {get_param: user_token}
$MAGNUM_URL: {get_param: magnum_url}
write_kubeconfig:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/write-kubeconfig.yaml}
make_cert:
type: OS::Heat::SoftwareConfig
properties:
group: ungrouped
config: {get_file: fragments/make-cert-client.sh}
configure_docker_storage:
type: OS::Heat::SoftwareConfig
@ -250,6 +292,8 @@ resources:
- config: {get_resource: disable_selinux}
- config: {get_resource: write_heat_params}
- config: {get_resource: kube_user}
- config: {get_resource: write_kubeconfig}
- config: {get_resource: make_cert}
- config: {get_resource: kube_examples}
- config: {get_resource: configure_docker_storage}
- config: {get_resource: kube_register}

3
magnum/tests/functional/python_client_base.py
View File

@ -103,6 +103,9 @@ class BaseMagnumClient(base.TestCase):
network_driver='flannel',
coe=coe,
labels={"K1": "V1", "K2": "V2"},
# TODO(yuanying): Change to `tls_disabled=False`
# if k8sclient supports TLS.
tls_disabled=True,
)
return baymodel

3
magnum/tests/functional/test_k8s_python_client.py
View File

@ -38,8 +38,7 @@ class TestKubernetesAPIs(BaseMagnumClient):
super(TestKubernetesAPIs, cls).setUpClass()
cls.baymodel = cls._create_baymodel('testk8sAPI')
cls.bay = cls._create_bay('testk8sAPI', cls.baymodel.uuid)
kube_api_address = cls.cs.bays.get(cls.bay.uuid).api_address
kube_api_url = 'http://%s' % kube_api_address
kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
k8s_client = api_client.ApiClient(kube_api_url)
cls.k8s_api = apiv_api.ApivApi(k8s_client)

33
magnum/tests/unit/conductor/handlers/test_bay_conductor.py
View File

@ -52,9 +52,10 @@ class TestBayConductorWithK8s(base.TestCase):
'labels': {'flannel_network_cidr': '10.101.0.0/16',
'flannel_network_subnetlen': '26',
'flannel_use_vxlan': 'yes'},
'tls_disabled': False,
}
self.bay_dict = {
'uuid': 'bay-xx-xx-xx-xx',
'baymodel_id': 'xx-xx-xx-xx',
'name': 'bay1',
'stack_id': 'xx-xx-xx-xx',
@ -70,6 +71,12 @@ class TestBayConductorWithK8s(base.TestCase):
self.context.auth_url = 'http://192.168.10.10:5000/v3'
self.context.user_name = 'fake_user'
self.context.tenant = 'fake_tenant'
osc_patcher = mock.patch('magnum.common.clients.OpenStackClients')
self.mock_osc_class = osc_patcher.start()
self.addCleanup(osc_patcher.stop)
self.mock_osc = mock.MagicMock()
self.mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
self.mock_osc_class.return_value = self.mock_osc
@patch('magnum.objects.BayModel.get_by_uuid')
def test_extract_template_definition(
@ -112,6 +119,10 @@ class TestBayConductorWithK8s(base.TestCase):
'http_proxy': 'http_proxy',
'https_proxy': 'https_proxy',
'no_proxy': 'no_proxy',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
expected = {
'ssh_key_name': 'keypair_id',
@ -135,6 +146,10 @@ class TestBayConductorWithK8s(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v2',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
if missing_attr is not None:
expected.pop(mapping[missing_attr], None)
@ -186,6 +201,10 @@ class TestBayConductorWithK8s(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v2',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
self.assertEqual(expected, definition)
@ -234,6 +253,10 @@ class TestBayConductorWithK8s(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v2',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
self.assertEqual(expected, definition)
@ -322,6 +345,10 @@ class TestBayConductorWithK8s(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v2',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
self.assertIn('token', definition)
del definition['token']
@ -395,6 +422,10 @@ class TestBayConductorWithK8s(base.TestCase):
'auth_url': 'http://192.168.10.10:5000/v2',
'tenant_name': 'fake_tenant',
'username': 'fake_user',
'user_token': self.context.auth_token,
'bay_uuid': self.bay_dict['uuid'],
'magnum_url': self.mock_osc.magnum_url.return_value,
'tls_disabled': False,
}
self.assertEqual(expected, definition)
reqget.assert_called_once_with('http://etcd/test?size=1')

8
magnum/tests/unit/conductor/handlers/test_k8s_conductor.py
View File

@ -12,8 +12,6 @@
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from magnum.common import exception
from magnum.common.pythonk8sclient.swagger_client import rest
from magnum.conductor.handlers import k8s_conductor
@ -24,12 +22,6 @@ import mock
from mock import patch
cfg.CONF.import_opt('k8s_protocol', 'magnum.conductor.handlers.k8s_conductor',
group='kubernetes')
cfg.CONF.import_opt('k8s_port', 'magnum.conductor.handlers.k8s_conductor',
group='kubernetes')
class TestK8sConductor(base.TestCase):
def setUp(self):
super(TestK8sConductor, self).setUp()

6
magnum/tests/unit/conductor/test_k8s_api.py
View File

@ -13,7 +13,6 @@
# under the License.
from mock import patch
from oslo_config import cfg
from magnum.conductor import k8s_api
from magnum import objects
@ -26,7 +25,6 @@ class TestK8sAPI(base.TestCase):
def test_retrieve_k8s_api_endpoint(self, mock_bay_get_by_uuid):
expected_context = 'context'
expected_api_address = 'api_address'
expected_protocol = cfg.CONF.kubernetes.k8s_protocol
resource = objects.Pod({})
resource.bay_uuid = 'bay_uuid'
@ -37,9 +35,7 @@ class TestK8sAPI(base.TestCase):
actual_api_endpoint = k8s_api.K8sAPI._retrieve_k8s_api_endpoint(
expected_context, resource)
self.assertEqual("%s://%s" % (expected_protocol,
expected_api_address),
actual_api_endpoint)
self.assertEqual(expected_api_address, actual_api_endpoint)
@patch('magnum.conductor.k8s_api.K8sAPI')
def test_create_k8s_api(self, mock_k8s_api_cls):

153
magnum/tests/unit/conductor/test_template_definition.py
View File

@ -136,22 +136,35 @@ class TemplateDefinitionTestCase(base.TestCase):
value = output.get_output_value(mock_stack)
self.assertIsNone(value)
def test_add_output_with_mapping_type(self):
definition = tdef.TemplateDefinition.get_template_definition(
'vm',
'fedora-atomic',
'kubernetes')
mock_args = [1, 3, 4]
mock_kwargs = {'test': 'test'}
mock_mapping_type = mock.MagicMock()
mock_mapping_type.return_value = mock.MagicMock()
definition.add_output(mapping_type=mock_mapping_type, *mock_args,
**mock_kwargs)
mock_mapping_type.assert_called_once_with(*mock_args, **mock_kwargs)
self.assertIn(mock_mapping_type.return_value,
definition.output_mappings)
def test_update_outputs(self):
definition = tdef.TemplateDefinition.get_template_definition(
'vm',
'fedora-atomic',
'kubernetes')
expected_api_address = 'api_address'
expected_node_addresses = ['ex_minion', 'address']
outputs = [
{"output_value": expected_node_addresses,
"description": "No description given",
"output_key": "kube_minions_external"},
{"output_value": expected_api_address,
"description": "No description given",
"output_key": "api_address"},
{"output_value": ['any', 'output'],
"description": "No description given",
"output_key": "kube_minions"}
@ -159,15 +172,17 @@ class TemplateDefinitionTestCase(base.TestCase):
mock_stack = mock.MagicMock()
mock_stack.outputs = outputs
mock_bay = mock.MagicMock()
mock_bay.api_address = None
mock_baymodel = mock.MagicMock()
definition.update_outputs(mock_stack, mock_bay)
definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
self.assertEqual(mock_bay.api_address, expected_api_address)
self.assertEqual(mock_bay.node_addresses, expected_node_addresses)
class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
@mock.patch('magnum.common.clients.OpenStackClients')
@mock.patch('magnum.conductor.template_definition'
'.AtomicK8sTemplateDefinition.get_discovery_url')
@mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
@ -175,11 +190,17 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
@mock.patch('magnum.conductor.template_definition.TemplateDefinition'
'.get_output')
def test_k8s_get_params(self, mock_get_output, mock_get_params,
mock_get_discovery_url):
mock_get_discovery_url, mock_osc_class):
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
mock_baymodel = mock.MagicMock()
mock_baymodel.tls_disabled = False
mock_bay = mock.MagicMock()
mock_bay.uuid = 'bay-xx-xx-xx-xx'
mock_scale_manager = mock.MagicMock()
mock_osc = mock.MagicMock()
mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
mock_osc_class.return_value = mock_osc
removal_nodes = ['node1', 'node2']
mock_scale_manager.get_removal_nodes.return_value = removal_nodes
@ -206,7 +227,62 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
'flannel_network_subnetlen': flannel_vxlan,
'auth_url': 'http://192.168.10.10:5000/v2',
'username': 'fake_user',
'tenant_name': 'fake_tenant'}}
'tenant_name': 'fake_tenant',
'magnum_url': mock_osc.magnum_url.return_value,
'user_token': mock_context.auth_token}}
mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
mock_bay, **expected_kwargs)
@mock.patch('magnum.common.clients.OpenStackClients')
@mock.patch('magnum.conductor.template_definition'
'.AtomicK8sTemplateDefinition.get_discovery_url')
@mock.patch('magnum.conductor.template_definition.BaseTemplateDefinition'
'.get_params')
@mock.patch('magnum.conductor.template_definition.TemplateDefinition'
'.get_output')
def test_k8s_get_params_insecure(self, mock_get_output, mock_get_params,
mock_get_discovery_url, mock_osc_class):
mock_context = mock.MagicMock()
mock_context.auth_token = 'AUTH_TOKEN'
mock_baymodel = mock.MagicMock()
mock_baymodel.tls_disabled = True
mock_bay = mock.MagicMock()
mock_bay.uuid = 'bay-xx-xx-xx-xx'
mock_scale_manager = mock.MagicMock()
mock_osc = mock.MagicMock()
mock_osc.magnum_url.return_value = 'http://127.0.0.1:9511/v1'
mock_osc_class.return_value = mock_osc
removal_nodes = ['node1', 'node2']
mock_scale_manager.get_removal_nodes.return_value = removal_nodes
mock_get_discovery_url.return_value = 'fake_discovery_url'
mock_context.auth_url = 'http://192.168.10.10:5000/v3'
mock_context.user_name = 'fake_user'
mock_context.tenant = 'fake_tenant'
flannel_cidr = mock_baymodel.labels.get('flannel_network_cidr')
flannel_subnet = mock_baymodel.labels.get('flannel_network_subnetlen')
flannel_vxlan = mock_baymodel.labels.get('flannel_use_vxlan')
k8s_def = tdef.AtomicK8sTemplateDefinition()
k8s_def.get_params(mock_context, mock_baymodel, mock_bay,
scale_manager=mock_scale_manager)
expected_kwargs = {'extra_params': {
'minions_to_remove': removal_nodes,
'discovery_url': 'fake_discovery_url',
'flannel_network_cidr': flannel_cidr,
'flannel_use_vxlan': flannel_subnet,
'flannel_network_subnetlen': flannel_vxlan,
'auth_url': 'http://192.168.10.10:5000/v2',
'username': 'fake_user',
'tenant_name': 'fake_tenant',
'magnum_url': mock_osc.magnum_url.return_value,
'user_token': mock_context.auth_token,
'loadbalancing_protocol': 'HTTP',
'kubernetes_port': 8080}}
mock_get_params.assert_called_once_with(mock_context, mock_baymodel,
mock_bay, **expected_kwargs)
@ -249,6 +325,67 @@ class AtomicK8sTemplateDefinitionTestCase(base.TestCase):
tdef.AtomicK8sTemplateDefinition().get_discovery_url,
fake_bay)
def test_update_outputs_api_address(self):
definition = tdef.TemplateDefinition.get_template_definition(
'vm',
'fedora-atomic',
'kubernetes')
address = 'updated_address'
protocol = 'http'
port = '8080'
params = {
'protocol': protocol,
'address': address,
'port': port,
}
expected_api_address = '%(protocol)s://%(address)s:%(port)s' % params
outputs = [
{"output_value": address,
"description": "No description given",
"output_key": "api_address"},
]
mock_stack = mock.MagicMock()
mock_stack.outputs = outputs
mock_bay = mock.MagicMock()
mock_baymodel = mock.MagicMock()
mock_baymodel.tls_disabled = True
definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
self.assertEqual(mock_bay.api_address, expected_api_address)
def test_update_outputs_if_baymodel_is_secure(self):
definition = tdef.TemplateDefinition.get_template_definition(
'vm',
'fedora-atomic',
'kubernetes')
address = 'updated_address'
protocol = 'https'
port = '6443'
params = {
'protocol': protocol,
'address': address,
'port': port,
}
expected_api_address = '%(protocol)s://%(address)s:%(port)s' % params
outputs = [
{"output_value": address,
"description": "No description given",
"output_key": "api_address"},
]
mock_stack = mock.MagicMock()
mock_stack.outputs = outputs
mock_bay = mock.MagicMock()
mock_baymodel = mock.MagicMock()
mock_baymodel.tls_disabled = False
definition.update_outputs(mock_stack, mock_baymodel, mock_bay)
self.assertEqual(mock_bay.api_address, expected_api_address)
class AtomicSwarmTemplateDefinitionTestCase(base.TestCase):