Merge "Allow qinling to connect to k8s API with certificates"
This commit is contained in:
commit
16b5d8adad
|
@ -58,6 +58,26 @@ function mkdir_chown_stack {
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
function configure_k8s_certificates {
|
||||||
|
pushd $QINLING_DIR
|
||||||
|
mkdir_chown_stack "$QINLING_CONF_DIR"/pki/kubernetes
|
||||||
|
|
||||||
|
curl -L https://pkg.cfssl.org/R1.2/cfssl_linux-amd64 -o /tmp/cfssl
|
||||||
|
chmod +x /tmp/cfssl
|
||||||
|
curl -L https://pkg.cfssl.org/R1.2/cfssljson_linux-amd64 -o /tmp/cfssljson
|
||||||
|
chmod +x /tmp/cfssljson
|
||||||
|
|
||||||
|
sudo /tmp/cfssl gencert -ca=/etc/kubernetes/pki/ca.crt -ca-key=/etc/kubernetes/pki/ca.key -config=example/kubernetes/cfssl-ca-config.json -profile=client example/kubernetes/cfssl-client-csr.json | /tmp/cfssljson -bare client
|
||||||
|
# The command above outputs client-key.pem and client.pem
|
||||||
|
mv client-key.pem "$QINLING_CONF_DIR"/pki/kubernetes/qinling.key
|
||||||
|
mv client.pem "$QINLING_CONF_DIR"/pki/kubernetes/qinling.crt
|
||||||
|
|
||||||
|
cp /etc/kubernetes/pki/ca.crt "$QINLING_CONF_DIR"/pki/kubernetes/ca.crt
|
||||||
|
|
||||||
|
popd
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
function configure_qinling {
|
function configure_qinling {
|
||||||
mkdir_chown_stack "$QINLING_AUTH_CACHE_DIR"
|
mkdir_chown_stack "$QINLING_AUTH_CACHE_DIR"
|
||||||
rm -f "$QINLING_AUTH_CACHE_DIR"/*
|
rm -f "$QINLING_AUTH_CACHE_DIR"/*
|
||||||
|
@ -89,6 +109,15 @@ function configure_qinling {
|
||||||
|
|
||||||
# Configure the database.
|
# Configure the database.
|
||||||
iniset $QINLING_CONF_FILE database connection `database_connection_url qinling`
|
iniset $QINLING_CONF_FILE database connection `database_connection_url qinling`
|
||||||
|
|
||||||
|
# Configure Kubernetes API server certificates for qinling if required.
|
||||||
|
if [ "$QINLING_K8S_APISERVER_TLS" == "True" ]; then
|
||||||
|
iniset $QINLING_CONF_FILE kubernetes kube_host https://$(hostname -f):6443
|
||||||
|
configure_k8s_certificates
|
||||||
|
sudo kubectl create -f $QINLING_DIR/example/kubernetes/k8s_qinling_role.yaml
|
||||||
|
else
|
||||||
|
iniset $QINLING_CONF_FILE kubernetes use_api_certificate False
|
||||||
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -24,3 +24,5 @@ QINLING_AUTH_CACHE_DIR=${QINLING_AUTH_CACHE_DIR:-/var/cache/qinling}
|
||||||
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages}
|
QINLING_FUNCTION_STORAGE_DIR=${QINLING_FUNCTION_STORAGE_DIR:-/opt/qinling/funtion/packages}
|
||||||
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime}
|
QINLING_PYTHON_RUNTIME_IMAGE=${QINLING_PYTHON_RUNTIME_IMAGE:-openstackqinling/python-runtime}
|
||||||
QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime}
|
QINLING_NODEJS_RUNTIME_IMAGE=${QINLING_NODEJS_RUNTIME_IMAGE:-openstackqinling/nodejs-runtime}
|
||||||
|
|
||||||
|
QINLING_K8S_APISERVER_TLS=${QINLING_K8S_APISERVER_TLS:-True}
|
||||||
|
|
|
@ -0,0 +1,17 @@
|
||||||
|
{
|
||||||
|
"signing": {
|
||||||
|
"default": {
|
||||||
|
"expiry": "168h"
|
||||||
|
},
|
||||||
|
"profiles": {
|
||||||
|
"client": {
|
||||||
|
"expiry": "8760h",
|
||||||
|
"usages": [
|
||||||
|
"signing",
|
||||||
|
"key encipherment",
|
||||||
|
"client auth"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,7 @@
|
||||||
|
{
|
||||||
|
"CN": "qinling",
|
||||||
|
"key": {
|
||||||
|
"algo": "rsa",
|
||||||
|
"size": 2048
|
||||||
|
}
|
||||||
|
}
|
|
@ -0,0 +1,74 @@
|
||||||
|
---
|
||||||
|
kind: ClusterRole
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: qinling
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["nodes", "namespaces"]
|
||||||
|
verbs: ["list"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["namespaces"]
|
||||||
|
resourceNames: ["qinling"]
|
||||||
|
verbs: ["create"]
|
||||||
|
---
|
||||||
|
kind: ClusterRoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: qinling
|
||||||
|
subjects:
|
||||||
|
- kind: User
|
||||||
|
name: qinling
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
roleRef:
|
||||||
|
kind: ClusterRole
|
||||||
|
name: qinling
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
---
|
||||||
|
# The qinling namespace should be created for the role and rolebinding
|
||||||
|
apiVersion: v1
|
||||||
|
kind: Namespace
|
||||||
|
metadata:
|
||||||
|
name: qinling
|
||||||
|
---
|
||||||
|
kind: Role
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: qinling
|
||||||
|
namespace: qinling
|
||||||
|
rules:
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["services"]
|
||||||
|
verbs: ["list", "get", "create", "delete"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods"]
|
||||||
|
verbs: ["list", "get", "create", "patch", "delete", "deletecollection"]
|
||||||
|
- apiGroups: [""]
|
||||||
|
resources: ["pods/log"]
|
||||||
|
verbs: ["get"]
|
||||||
|
- apiGroups: ["extensions"]
|
||||||
|
resources: ["deployments"]
|
||||||
|
verbs: ["get", "create", "patch", "deletecollection"]
|
||||||
|
- apiGroups: ["extensions"]
|
||||||
|
resources: ["deployments/rollback"]
|
||||||
|
verbs: ["create"]
|
||||||
|
- apiGroups: ["extensions"]
|
||||||
|
resources: ["deployments/status"]
|
||||||
|
verbs: ["get"]
|
||||||
|
- apiGroups: ["extensions"]
|
||||||
|
resources: ["replicasets"]
|
||||||
|
verbs: ["deletecollection"]
|
||||||
|
---
|
||||||
|
kind: RoleBinding
|
||||||
|
apiVersion: rbac.authorization.k8s.io/v1
|
||||||
|
metadata:
|
||||||
|
name: qinling
|
||||||
|
namespace: qinling
|
||||||
|
subjects:
|
||||||
|
- kind: User
|
||||||
|
name: qinling
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
||||||
|
roleRef:
|
||||||
|
kind: Role
|
||||||
|
name: qinling
|
||||||
|
apiGroup: rbac.authorization.k8s.io
|
|
@ -146,6 +146,30 @@ kubernetes_opts = [
|
||||||
help='Kubernetes server address, e.g. you can start a proxy to the '
|
help='Kubernetes server address, e.g. you can start a proxy to the '
|
||||||
'Kubernetes API server by using "kubectl proxy" command.'
|
'Kubernetes API server by using "kubectl proxy" command.'
|
||||||
),
|
),
|
||||||
|
cfg.BoolOpt(
|
||||||
|
'use_api_certificate',
|
||||||
|
default=True,
|
||||||
|
help='Whether to use client certificates to connect to the '
|
||||||
|
'Kubernetes API server.'
|
||||||
|
),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'ssl_ca_cert',
|
||||||
|
default='/etc/qinling/pki/kubernetes/ca.crt',
|
||||||
|
help='Path to the CA certificate for qinling to use to connect to '
|
||||||
|
'the Kubernetes API server.'
|
||||||
|
),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'cert_file',
|
||||||
|
default='/etc/qinling/pki/kubernetes/qinling.crt',
|
||||||
|
help='Path to the client certificate for qinling to use to '
|
||||||
|
'connect to the Kubernetes API server.'
|
||||||
|
),
|
||||||
|
cfg.StrOpt(
|
||||||
|
'key_file',
|
||||||
|
default='/etc/qinling/pki/kubernetes/qinling.key',
|
||||||
|
help='Path to the client certificate key file for qinling to use to '
|
||||||
|
'connect to the Kubernetes API server.'
|
||||||
|
),
|
||||||
cfg.StrOpt(
|
cfg.StrOpt(
|
||||||
'log_devel',
|
'log_devel',
|
||||||
default='INFO',
|
default='INFO',
|
||||||
|
|
|
@ -22,7 +22,12 @@ from kubernetes.client import configuration as k8s_config
|
||||||
def get_k8s_clients(conf):
|
def get_k8s_clients(conf):
|
||||||
config = k8s_config.Configuration()
|
config = k8s_config.Configuration()
|
||||||
config.host = conf.kubernetes.kube_host
|
config.host = conf.kubernetes.kube_host
|
||||||
config.verify_ssl = False
|
if conf.kubernetes.use_api_certificate:
|
||||||
|
config.ssl_ca_cert = conf.kubernetes.ssl_ca_cert
|
||||||
|
config.cert_file = conf.kubernetes.cert_file
|
||||||
|
config.key_file = conf.kubernetes.key_file
|
||||||
|
else:
|
||||||
|
config.verify_ssl = False
|
||||||
client = api_client.ApiClient(configuration=config)
|
client = api_client.ApiClient(configuration=config)
|
||||||
v1 = core_v1_api.CoreV1Api(client)
|
v1 = core_v1_api.CoreV1Api(client)
|
||||||
v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client)
|
v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client)
|
||||||
|
|
|
@ -40,7 +40,4 @@ QinlingGroup = [
|
||||||
choices=['public', 'admin', 'internal',
|
choices=['public', 'admin', 'internal',
|
||||||
'publicURL', 'adminURL', 'internalURL'],
|
'publicURL', 'adminURL', 'internalURL'],
|
||||||
help="The endpoint type to use for the qinling service."),
|
help="The endpoint type to use for the qinling service."),
|
||||||
cfg.StrOpt('kube_host',
|
|
||||||
default='http://127.0.0.1:8001',
|
|
||||||
help="The Kubernetes service address."),
|
|
||||||
]
|
]
|
||||||
|
|
|
@ -43,19 +43,10 @@ class RuntimesTest(base.BaseQinlingTest):
|
||||||
)
|
)
|
||||||
|
|
||||||
# Wait for runtime to be available
|
# Wait for runtime to be available
|
||||||
|
# We don't have to check k8s resource, if runtime's status has changed
|
||||||
|
# to available, then kubernetes deployment is assumed to be ok.
|
||||||
self.await_runtime_available(runtime_id)
|
self.await_runtime_available(runtime_id)
|
||||||
|
|
||||||
# Check k8s resource
|
|
||||||
deploy = self.k8s_v1extention.read_namespaced_deployment(
|
|
||||||
runtime_id,
|
|
||||||
namespace=self.namespace
|
|
||||||
)
|
|
||||||
|
|
||||||
self.assertEqual(runtime_id, deploy.metadata.name)
|
|
||||||
self.assertEqual(
|
|
||||||
deploy.status.replicas, deploy.status.available_replicas
|
|
||||||
)
|
|
||||||
|
|
||||||
# Delete runtime
|
# Delete runtime
|
||||||
resp = self.admin_client.delete_resource('runtimes', runtime_id)
|
resp = self.admin_client.delete_resource('runtimes', runtime_id)
|
||||||
|
|
||||||
|
|
|
@ -21,8 +21,6 @@ from tempest.lib.common.utils import data_utils
|
||||||
from tempest import test
|
from tempest import test
|
||||||
import tenacity
|
import tenacity
|
||||||
|
|
||||||
from qinling_tempest_plugin.tests import utils
|
|
||||||
|
|
||||||
CONF = config.CONF
|
CONF = config.CONF
|
||||||
|
|
||||||
|
|
||||||
|
@ -46,13 +44,6 @@ class BaseQinlingTest(test.BaseTestCase):
|
||||||
cls.alt_client = cls.os_alt.qinling.QinlingClient()
|
cls.alt_client = cls.os_alt.qinling.QinlingClient()
|
||||||
cls.admin_client = cls.os_admin.qinling.QinlingClient()
|
cls.admin_client = cls.os_admin.qinling.QinlingClient()
|
||||||
|
|
||||||
# Initilize k8s client
|
|
||||||
clients = utils.get_k8s_clients(CONF)
|
|
||||||
cls.k8s_v1 = clients['v1']
|
|
||||||
cls.k8s_v1extention = clients['v1extention']
|
|
||||||
# cls.k8s_apps_v1 = clients['apps_v1']
|
|
||||||
cls.namespace = 'qinling'
|
|
||||||
|
|
||||||
@classmethod
|
@classmethod
|
||||||
def resource_setup(cls):
|
def resource_setup(cls):
|
||||||
super(BaseQinlingTest, cls).resource_setup()
|
super(BaseQinlingTest, cls).resource_setup()
|
||||||
|
|
|
@ -14,30 +14,6 @@
|
||||||
|
|
||||||
import hashlib
|
import hashlib
|
||||||
|
|
||||||
from kubernetes.client import api_client
|
|
||||||
# from kubernetes.client.apis import apps_v1_api
|
|
||||||
from kubernetes.client.apis import core_v1_api
|
|
||||||
from kubernetes.client.apis import extensions_v1beta1_api
|
|
||||||
from kubernetes.client import configuration as k8s_config
|
|
||||||
|
|
||||||
|
|
||||||
def get_k8s_clients(conf):
|
|
||||||
config = k8s_config.Configuration()
|
|
||||||
config.host = conf.qinling.kube_host
|
|
||||||
config.verify_ssl = False
|
|
||||||
client = api_client.ApiClient(configuration=config)
|
|
||||||
v1 = core_v1_api.CoreV1Api(client)
|
|
||||||
v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client)
|
|
||||||
# apps_v1 = apps_v1_api.AppsV1Api(client)
|
|
||||||
|
|
||||||
clients = {
|
|
||||||
'v1': v1,
|
|
||||||
'v1extention': v1extention
|
|
||||||
# 'apps_v1': apps_v1
|
|
||||||
}
|
|
||||||
|
|
||||||
return clients
|
|
||||||
|
|
||||||
|
|
||||||
def md5(file=None, content=None):
|
def md5(file=None, content=None):
|
||||||
hash_md5 = hashlib.md5()
|
hash_md5 = hashlib.md5()
|
||||||
|
|
|
@ -40,7 +40,10 @@ while true; do
|
||||||
[ $now -gt $end ] && echo "Failed to setup kubernetes cluster in time" && exit -1
|
[ $now -gt $end ] && echo "Failed to setup kubernetes cluster in time" && exit -1
|
||||||
done
|
done
|
||||||
|
|
||||||
create_k8s_screen
|
if [ "$QINLING_K8S_APISERVER_TLS" != "True" ]; then
|
||||||
|
# Kubernetes proxy is needed if we don't use secure connections.
|
||||||
|
create_k8s_screen
|
||||||
|
fi
|
||||||
|
|
||||||
#net_hosts_post_kube
|
#net_hosts_post_kube
|
||||||
#net_resolv_post_kube
|
#net_resolv_post_kube
|
||||||
|
|
Loading…
Reference in New Issue