Support Kubernetes as VIM in Tacker

This patch add kubernetes_driver in vim nfvo and kubernetes_utils to support
CRUD Kubernetes cluster VIM (register, deregister, delete and update).

Partially Implements: blueprint kubernetes-as-vim

Change-Id: Ib1bf4d78ca4796c4e0297bca6fc7e9f004078242
This commit is contained in:
Cong Phuoc Hoang 2017-10-10 14:30:24 +09:00
parent 8620c28651
commit d6207c6dcd
11 changed files with 520 additions and 17 deletions

View File

@ -143,8 +143,8 @@ kuryr-kubernetes to get more information [#third]_.
5. Register Kubernetes VIM
In vim_config.yaml, project_name is namespace in Kubernetes environment
where user will deploy Pod, Deployment or Horizontal Pod Autoscaling, etc.
In vim_config.yaml, project_name is fixed as "default", that will use to
support multi tenant on Kubernetes in the future.
* Create vim_config.yaml file for Kubernetes VIM as the following examples:
@ -233,6 +233,7 @@ authentication.
username: "admin"
password: "admin"
project_name: "default"
ssl_ca_cert: None
type: "kubernetes"
@ -250,4 +251,3 @@ References
.. [#second] https://github.com/openstack/tacker/blob/master/devstack/local.conf.example
.. [#third] https://github.com/openstack/kuryr-kubernetes/blob/master/doc/source/installation/testing_connectivity.rst
.. [#fourth] https://kubernetes.io/docs/admin/authentication

View File

@ -0,0 +1,5 @@
---
features:
- |
Add support Kubernetes VIM, user can create, update or delete
Kubernetes VIM

View File

@ -43,3 +43,6 @@ paramiko>=2.0.0 # LGPLv2.1+
pyroute2>=0.4.21;sys_platform!='win32' # Apache-2.0 (+ dual licensed GPL2)
python-mistralclient>=3.1.0 # Apache-2.0
python-barbicanclient!=4.5.0,!=4.5.1,>=4.0.0 # Apache-2.0
kubernetes>=1.0.0 # Apache-2.0
setuptools>=16.0,!=24.0.0,!=34.0.0,!=34.0.1,!=34.0.2,!=34.0.3,!=34.1.0,!=34.1.1,!=34.2.0,!=34.3.0,!=34.3.1,!=34.3.2,!=36.2.0 # PSF/ZPL
PyYAML>=3.10 # MIT

View File

@ -47,6 +47,7 @@ tacker.service_plugins =
commonservices = tacker.plugins.common_services.common_services_plugin:CommonServicesPlugin
tacker.nfvo.vim.drivers =
openstack = tacker.nfvo.drivers.vim.openstack_driver:OpenStack_Driver
kubernetes = tacker.nfvo.drivers.vim.kubernetes_driver:Kubernetes_Driver
tacker.openstack.common.cache.backends =
memory = tacker.openstack.common.cache._backends.memory:MemoryBackend
tacker.tacker.vnfm.drivers =
@ -71,6 +72,7 @@ oslo.config.opts =
tacker.service = tacker.service:config_opts
tacker.nfvo.nfvo_plugin = tacker.nfvo.nfvo_plugin:config_opts
tacker.nfvo.drivers.vim.openstack_driver = tacker.nfvo.drivers.vim.openstack_driver:config_opts
tacker.nfvo.drivers.vim.kubernetes_driver = tacker.nfvo.drivers.vim.kubernetes_driver:config_opts
tacker.keymgr = tacker.keymgr:config_opts
tacker.vnfm.monitor = tacker.vnfm.monitor:config_opts
tacker.vnfm.plugin = tacker.vnfm.plugin:config_opts

View File

View File

@ -0,0 +1,92 @@
# All Rights Reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import re
import six
import tempfile
from cryptography import fernet
from kubernetes import client
from kubernetes.client import api_client
from oslo_config import cfg
from oslo_log import log as logging
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class KubernetesHTTPAPI(object):
def get_k8sClient(self, auth_plugin):
config = client.ConfigurationObject()
config.host = auth_plugin['auth_url']
if ('username' in auth_plugin) and ('password' in auth_plugin)\
and (auth_plugin['password'] is not None):
config.username = auth_plugin['username']
config.password = auth_plugin['password']
basic_token = config.get_basic_auth_token()
config.api_key['authorization'] = basic_token
if 'bearer_token' in auth_plugin:
config.api_key_prefix['authorization'] = 'Bearer'
config.api_key['authorization'] = auth_plugin['bearer_token']
ca_cert_file = auth_plugin.get('ca_cert_file')
if ca_cert_file is not None:
config.ssl_ca_cert = ca_cert_file
config.verify_ssl = True
else:
config.verify_ssl = False
k8s_client = api_client.ApiClient(config=config)
return k8s_client
def initialize_ExtensionApiClient(self, auth):
k8s_client = self.get_k8sClient(auth_plugin=auth)
return client.ExtensionsV1beta1Api(api_client=k8s_client)
def initialize_CoreApiV1Client(self, auth):
k8s_client = self.get_k8sClient(auth_plugin=auth)
return client.CoreV1Api(api_client=k8s_client)
def initialize_CoreApiClient(self, auth):
k8s_client = self.get_k8sClient(auth_plugin=auth)
return client.CoreApi(api_client=k8s_client)
@staticmethod
def create_ca_cert_tmp_file(ca_cert):
file_descriptor, file_path = tempfile.mkstemp()
ca_cert = re.sub(r'\s', '\n', ca_cert)
ca_cert = re.sub(r'BEGIN\nCERT', r'BEGIN CERT', ca_cert)
ca_cert = re.sub(r'END\nCERT', r'END CERT', ca_cert)
try:
with open(file_path, 'w') as f:
if six.PY2:
f.write(ca_cert.decode('utf-8'))
else:
f.write(ca_cert)
LOG.debug('ca cert temp file successfully stored in %s',
file_path)
except IOError:
raise Exception('Failed to create %s file', file_path)
return file_descriptor, file_path
@staticmethod
def close_tmp_file(file_descriptor, file_path):
os.close(file_descriptor)
os.remove(file_path)
def create_fernet_key(self):
fernet_key = fernet.Fernet.generate_key()
fernet_obj = fernet.Fernet(fernet_key)
return fernet_key, fernet_obj

View File

@ -59,6 +59,10 @@ class VimKeyNotFoundException(exceptions.TackerException):
message = _("Unable to find key file for VIM %(vim_id)s")
class VimEncryptKeyError(exceptions.TackerException):
message = _("Barbican must be enabled for VIM %(vim_id)s")
class VimUnsupportedResourceTypeException(exceptions.TackerException):
message = _("Resource type %(type)s is unsupported by VIM")

View File

@ -0,0 +1,227 @@
# All Rights Reserved.
#
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_config import cfg
from oslo_log import log as logging
from tacker._i18n import _
from tacker.common.container import kubernetes_utils
from tacker.common import log
from tacker.extensions import nfvo
from tacker.keymgr import API as KEYMGR_API
from tacker.nfvo.drivers.vim import abstract_vim_driver
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
KUBERNETES_OPTS = [
cfg.BoolOpt('use_barbican', default=True,
help=_('Use barbican to encrypt vim password if True'
', save vim credentials in local file system'
'if False'))
]
cfg.CONF.register_opts(KUBERNETES_OPTS, 'k8s_vim')
def config_opts():
return [('k8s_vim', KUBERNETES_OPTS)]
class Kubernetes_Driver(abstract_vim_driver.VimAbstractDriver):
"""Driver for Kubernetes VIM
"""
def __init__(self):
self.kubernetes = kubernetes_utils.KubernetesHTTPAPI()
def get_type(self):
return 'kubernetes'
def get_name(self):
return 'Kubernetes VIM Driver'
def get_description(self):
return 'Kubernetes VIM Driver'
def authenticate_vim(self, vim_obj):
"""Validate VIM auth attributes
"""
auth_cred, file_descriptor = self._get_auth_creds(vim_obj)
self._validate_vim(auth_cred, file_descriptor)
self.clean_authenticate_vim(auth_cred, file_descriptor)
def _get_auth_creds(self, vim_obj):
auth_cred = vim_obj['auth_cred']
file_descriptor = self._create_ssl_ca_file(auth_cred)
auth_cred['auth_url'] = vim_obj['auth_url']
if ('username' not in auth_cred) and ('password' not in auth_cred):
auth_cred['username'] = 'None'
auth_cred['password'] = None
return auth_cred, file_descriptor
def _create_ssl_ca_file(self, auth_cred):
ca_cert = auth_cred['ssl_ca_cert']
if ca_cert is not None:
file_descriptor, file_path = \
self.kubernetes.create_ca_cert_tmp_file(ca_cert)
auth_cred['ca_cert_file'] = file_path
return file_descriptor
else:
return None
def _validate_vim(self, auth, file_descriptor):
# If Tacker can get k8s_info, Kubernetes authentication is valid
# if not, it is invalid
try:
auth_dict = dict(auth)
k8s_coreClient = \
self.kubernetes.initialize_CoreApiClient(auth_dict)
k8s_info = k8s_coreClient.get_api_versions()
LOG.info(k8s_info)
except Exception as e:
LOG.info('VIM Kubernetes authentication is wrong.')
# delete temp file
self.clean_authenticate_vim(auth_dict, file_descriptor)
raise nfvo.VimUnauthorizedException(message=str(e))
def _initialize_k8s_extensionClient(self, auth):
k8s_extensionClient =\
self.kubernetes.initialize_ExtensionApiClient(auth)
return k8s_extensionClient
def _initialize_k8s_coreV1Client(self, auth):
k8s_coreV1Client =\
self.kubernetes.initialize_CoreApiV1Client(auth)
return k8s_coreV1Client
def _find_regions(self, k8s_coreV1Client):
list_namespaces = k8s_coreV1Client.list_namespace()
namespaces = [namespace.metadata.name
for namespace in list_namespaces.items]
return namespaces
def discover_placement_attr(self, vim_obj):
"""Fetch VIM placement information
Attributes can include regions, AZ, namespaces.
"""
# in Kubernetes environment, user can deploy resource
# on specific namespace
auth_cred, file_descriptor = self._get_auth_creds(vim_obj)
k8s_coreV1Client = \
self.kubernetes.initialize_CoreApiV1Client(auth_cred)
namespace_list = self._find_regions(k8s_coreV1Client)
self.clean_authenticate_vim(auth_cred, file_descriptor)
vim_obj['placement_attr'] = {'regions': namespace_list}
return vim_obj
def clean_authenticate_vim(self, vim_auth, file_descriptor):
# remove ca_cert_file from vim_obj if it exists
# close and delete temp ca_cert_file
if file_descriptor is not None:
file_path = vim_auth.pop('ca_cert_file')
self.kubernetes.close_tmp_file(file_descriptor, file_path)
@log.log
def register_vim(self, context, vim_obj):
"""Validate Kubernetes VIM."""
if 'key_type' in vim_obj['auth_cred']:
vim_obj['auth_cred'].pop(u'key_type')
if 'secret_uuid' in vim_obj['auth_cred']:
vim_obj['auth_cred'].pop(u'secret_uuid')
self.authenticate_vim(vim_obj)
self.discover_placement_attr(vim_obj)
self.encode_vim_auth(context, vim_obj['id'],
vim_obj['auth_cred'])
LOG.debug('VIM registration completed for %s', vim_obj)
@log.log
def deregister_vim(self, context, vim_obj):
"""Deregister Kubernetes VIM from NFVO
Delete VIM keys from file system
"""
self.delete_vim_auth(context, vim_obj['id'],
vim_obj['auth_cred'])
@log.log
def delete_vim_auth(self, context, vim_id, auth):
"""Delete kubernetes vim information
Delete vim key stored in file system
"""
if 'secret_uuid' in auth:
# Delete secret id of barbican
LOG.debug('Attempting to delete key for vim id %s',
vim_id)
if auth.get('key_type') == 'barbican_key':
try:
keystone_conf = CONF.keystone_authtoken
secret_uuid = auth['secret_uuid']
keymgr_api = KEYMGR_API(keystone_conf.auth_url)
keymgr_api.delete(context, secret_uuid)
LOG.debug('VIM key deleted successfully for vim %s',
vim_id)
except Exception as exception:
LOG.warning('VIM key deletion failed for vim %s due to %s',
vim_id,
exception)
raise
else:
raise nfvo.VimEncryptKeyError(vim_id=vim_id)
@log.log
def encode_vim_auth(self, context, vim_id, auth):
"""Encode VIM credentials
Store VIM auth using fernet key encryption
"""
fernet_key, fernet_obj = self.kubernetes.create_fernet_key()
if ('password' in auth) and (auth['password'] is not None):
encoded_auth = fernet_obj.encrypt(
auth['password'].encode('utf-8'))
auth['password'] = encoded_auth
if 'bearer_token' in auth:
encoded_auth = fernet_obj.encrypt(
auth['bearer_token'].encode('utf-8'))
auth['bearer_token'] = encoded_auth
if ('ssl_ca_cert' in auth) and (auth['ssl_ca_cert'] is not None):
encoded_auth = fernet_obj.encrypt(
auth['ssl_ca_cert'].encode('utf-8'))
auth['ssl_ca_cert'] = encoded_auth
if CONF.k8s_vim.use_barbican:
try:
keystone_conf = CONF.keystone_authtoken
keymgr_api = KEYMGR_API(keystone_conf.auth_url)
secret_uuid = keymgr_api.store(context, fernet_key)
auth['key_type'] = 'barbican_key'
auth['secret_uuid'] = secret_uuid
LOG.debug('VIM auth successfully stored for vim %s',
vim_id)
except Exception as exception:
LOG.warning('VIM key creation failed for vim %s due to %s',
vim_id,
exception)
raise
else:
raise nfvo.VimEncryptKeyError(vim_id=vim_id)
def get_vim_resource_id(self):
# TODO(phuoc): will update which vim resource need to get
pass

View File

@ -70,7 +70,7 @@ class NfvoPlugin(nfvo_db_plugin.NfvoPluginDb, vnffg_db.VnffgPluginDbMixin,
OPTS = [
cfg.ListOpt(
'vim_drivers', default=['openstack'],
'vim_drivers', default=['openstack', 'kubernetes'],
help=_('VIM driver for launching VNFs')),
cfg.IntOpt(
'monitor_interval', default=30,
@ -140,24 +140,44 @@ class NfvoPlugin(nfvo_db_plugin.NfvoPluginDb, vnffg_db.VnffgPluginDbMixin,
old_auth_need_delete = False
new_auth_created = False
try:
# re-register the VIM only if there is a change in password.
# re-register the VIM only if there is a change in bearer_token,
# username, password or bearer_token.
# auth_url of auth_cred is from vim object which
# is not updatable. so no need to consider it
if 'auth_cred' in update_args:
auth_cred = update_args['auth_cred']
if 'password' in auth_cred:
if ('username' in auth_cred) and ('password' in auth_cred)\
and (auth_cred['password'] is not None):
# update new username and password, remove bearer_token
# if it exists in the old vim
vim_obj['auth_cred']['username'] = auth_cred['username']
vim_obj['auth_cred']['password'] = auth_cred['password']
# Notice: vim_obj may be updated in vim driver's
self._vim_drivers.invoke(vim_type,
'register_vim',
context=context,
vim_obj=vim_obj)
new_auth_created = True
if 'bearer_token' in vim_obj['auth_cred']:
vim_obj['auth_cred'].pop('bearer_token')
elif 'bearer_token' in auth_cred:
# update bearer_token, remove username and password
# if they exist in the old vim
vim_obj['auth_cred']['bearer_token'] =\
auth_cred['bearer_token']
if ('username' in vim_obj['auth_cred']) and\
('password' in vim_obj['auth_cred']):
vim_obj['auth_cred'].pop('username')
vim_obj['auth_cred'].pop('password')
if 'ssl_ca_cert' in auth_cred:
# update new ssl_ca_cert
vim_obj['auth_cred']['ssl_ca_cert'] =\
auth_cred['ssl_ca_cert']
# Notice: vim_obj may be updated in vim driver's
self._vim_drivers.invoke(vim_type,
'register_vim',
context=context,
vim_obj=vim_obj)
new_auth_created = True
# Check whether old vim's auth need to be deleted
old_key_type = old_vim_obj['auth_cred'].get('key_type')
if old_key_type == 'barbican_key':
old_auth_need_delete = True
# Check whether old vim's auth need to be deleted
old_key_type = old_vim_obj['auth_cred'].get('key_type')
if old_key_type == 'barbican_key':
old_auth_need_delete = True
vim_obj = super(NfvoPlugin, self).update_vim(
context, vim_id, vim_obj)

View File

@ -62,7 +62,11 @@ def delete_workflow(mistral_client, vim_id):
def monitor_vim(auth_dict, vim_obj):
mc = get_mistral_client(auth_dict)
auth_url = vim_obj["auth_url"]
vim_ip = auth_url.split("//")[-1].split(":")[0].split("/")[0]
vim_type = vim_obj['type']
if vim_type == 'openstack':
vim_ip = auth_url.split("//")[-1].split(":")[0].split("/")[0]
elif vim_type == 'kubernetes':
vim_ip = auth_url.split("//")[-1].split(":")[0]
workflow_input_dict = {
'vim_id': vim_obj['id'],
'count': cfg.CONF.vim_monitor.count,

View File

@ -0,0 +1,146 @@
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from collections import namedtuple
import mock
from tacker.nfvo.drivers.vim import kubernetes_driver
from tacker.tests.unit import base
class FakeKubernetesAPI(mock.Mock):
pass
class FakeKeymgrAPI(mock.Mock):
pass
class mock_dict(dict):
def __getattr__(self, item):
return self.get(item)
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
class TestKubernetes_Driver(base.TestCase):
def setUp(self):
super(TestKubernetes_Driver, self).setUp()
self._mock_kubernetes()
self.config_fixture.config(group='k8s_vim', use_barbican=True)
self.kubernetes_driver = kubernetes_driver.Kubernetes_Driver()
self.vim_obj = self.get_vim_obj()
self.addCleanup(mock.patch.stopall)
self._mock_keymgr()
def _mock_kubernetes(self):
self.kubernetes_api = mock.Mock(wraps=FakeKubernetesAPI())
fake_kubernetes_api = mock.Mock()
fake_kubernetes_api.return_value = self.kubernetes_api
self._mock('tacker.common.container.kubernetes_utils.'
'KubernetesHTTPAPI', fake_kubernetes_api)
def _mock_keymgr(self):
self.keymgr = mock.Mock(wraps=FakeKeymgrAPI())
fake_keymgr = mock.Mock()
fake_keymgr.return_value = self.keymgr
self._mock(
'tacker.keymgr.barbican_key_manager.BarbicanKeyManager',
fake_keymgr)
def get_vim_obj(self):
return {'id': '647a91c3-d436-43e6-a1e8-71118dde84ce',
'type': 'kubernetes',
'auth_url': 'https://localhost:6443',
'auth_cred': {'username': 'test_user',
'password': 'test_password',
'ssl_ca_cert': None},
'name': 'vim-kubernetes',
'vim_project': {'name': 'default'}}
def get_vim_obj_barbican(self):
return {'id': '647a91c3-d436-43e6-a1e8-71118dde84ce',
'type': 'kubernetes',
'auth_url': 'https://localhost:6443',
'auth_cred': {'username': 'test_user',
'password': 'test_password',
'ssl_ca_cert': 'abcxyz',
'key_type': 'barbican_key',
'secret_uuid': 'fake-secret-uuid'},
'name': 'vim-kubernetes',
'vim_project': {'name': 'default'}}
def test_register_k8sclient(self):
dict = {'name': 'default'}
name = namedtuple("name", dict.keys())(*dict.values())
dict = {'metadata': name}
metadata = namedtuple("metadata", dict.keys())(*dict.values())
dict = {'items': [metadata]}
namespaces = namedtuple("namespace", dict.keys())(*dict.values())
attrs = {'list_namespace.return_value': namespaces}
mock_k8s_client = mock.Mock()
mock_k8s_coreV1Client = mock.Mock(**attrs)
auth_obj = {'username': 'test_user',
'password': 'test_password',
'ssl_ca_cert': None,
'auth_url': 'https://localhost:6443'}
self._test_register_vim(self.vim_obj, mock_k8s_client,
mock_k8s_coreV1Client)
mock_k8s_coreV1Client.list_namespace.assert_called_once_with()
self.kubernetes_api.\
initialize_CoreApiClient.assert_called_once_with(auth_obj)
def _test_register_vim(self, vim_obj, mock_k8s_client,
mock_k8s_coreV1Client):
self.kubernetes_api.\
initialize_CoreApiClient.return_value = mock_k8s_client
self.kubernetes_api.\
initialize_CoreApiV1Client.return_value = mock_k8s_coreV1Client
fernet_attrs = {'encrypt.return_value': 'encrypted_password'}
mock_fernet_obj = mock.Mock(**fernet_attrs)
mock_fernet_key = 'test_fernet_key'
self.kubernetes_api.create_fernet_key.return_value = (mock_fernet_key,
mock_fernet_obj)
self.kubernetes_api.create_ca_cert_tmp_file.\
return_value = ('file_descriptor', 'file_path')
self.kubernetes_driver.register_vim(None, vim_obj)
mock_fernet_obj.encrypt.assert_called_once_with(mock.ANY)
def test_deregister_vim_barbican(self):
self.keymgr.delete.return_value = None
vim_obj = self.get_vim_obj_barbican()
self.kubernetes_driver.deregister_vim(None, vim_obj)
self.keymgr.delete.assert_called_once_with(
None, 'fake-secret-uuid')
def test_encode_vim_auth_barbican(self):
self.config_fixture.config(group='k8s_vim',
use_barbican=True)
fernet_attrs = {'encrypt.return_value': 'encrypted_password'}
mock_fernet_obj = mock.Mock(**fernet_attrs)
mock_fernet_key = 'test_fernet_key'
self.keymgr.store.return_value = 'fake-secret-uuid'
self.kubernetes_api.create_fernet_key.return_value = (mock_fernet_key,
mock_fernet_obj)
vim_obj = self.get_vim_obj()
self.kubernetes_driver.encode_vim_auth(
None, vim_obj['id'], vim_obj['auth_cred'])
self.keymgr.store.assert_called_once_with(
None, 'test_fernet_key')
mock_fernet_obj.encrypt.assert_called_once_with(mock.ANY)
self.assertEqual(vim_obj['auth_cred']['key_type'],
'barbican_key')
self.assertEqual(vim_obj['auth_cred']['secret_uuid'],
'fake-secret-uuid')