Functional: Add testcase of tls_enabled bay creation case.

This patch use tls_enabled mode to test k8sApi.

This patch refactors _create_baymodel, gives _create_baymodel an optional
**kwargs to allow create various baymodel.
This is a preparation for other COE(like swarm, mesos) baymodel testing.

Co-Authored-By: OTSUKA, Yuanying<yuanying@fraction.jp>
Partially implements: blueprint swarm-functional-testin
Change-Id: I7680a7a7dd0f44926741dab8b982bc39a21e11ba

magnum/tests/contrib/post_test_hook.sh

@@ -87,6 +87,10 @@ keypair_id = default
 flavor_id = m1.magnum
 EOF
 
+# Note(eliqiao): Let's keep this only for debugging on gate.
+echo_summary $CREDS_FILE
+cat $CREDS_FILE
+
 # Create a keypair for use in the functional tests.
 echo_summary "Generate a key-pair"
 nova keypair-add default
@@ -95,10 +99,20 @@ nova keypair-add default
 echo_summary "Create a flavor"
 nova flavor-create  m1.magnum 100 2048 8 1
 
-# Run functional tests
+# FIXME(eliqao): workaround for allow 9511 can be accessed from VM.
+# k8s nodes will access m-api (port 9511) to get CA certificate.
+sudo iptables -D openstack-INPUT -j REJECT --reject-with icmp-host-prohibited
+sudo iptables -D openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: "
+
+sudo iptables -A openstack-INPUT -s 172.24.4.0/23 -p tcp -m tcp --dport 9511 -j ACCEPT
+sudo iptables -A openstack-INPUT -m limit --limit 2/min -j LOG --log-prefix "iptables dropped: "
+sudo iptables -A openstack-INPUT -j REJECT --reject-with icmp-host-prohibited
 
+
+# Run functional tests
 # Currently we support functional-api, functional-k8s, will support swarm,
 # mesos later.
+
 echo "Running magnum functional test suite for $1"
 sudo -E -H -u jenkins tox -e functional-$1 -- --concurrency=1
 EXIT_CODE=$?

magnum/tests/functional/k8s/test_k8s_python_client.py

@@ -13,7 +13,7 @@
 
 from magnum.common.pythonk8sclient.swagger_client import api_client
 from magnum.common.pythonk8sclient.swagger_client.apis import apiv_api
-from magnum.tests.functional.python_client_base import BaseMagnumClient
+from magnum.tests.functional.python_client_base import BayAPITLSTest
 from magnum.tests.functional.python_client_base import BayTest
 from magnumclient.openstack.common.apiclient import exceptions
 
@@ -22,24 +22,46 @@ class TestBayModelResource(BayTest):
     coe = 'kubernetes'
 
     def test_baymodel_create_and_delete(self):
-        self._test_baymodel_create_and_delete()
+        self._test_baymodel_create_and_delete('test_k8s_baymodel')
 
 
 class TestBayResource(BayTest):
     coe = 'kubernetes'
 
     def test_bay_create_and_delete(self):
-        self._test_bay_create_and_delete()
+        baymodel_uuid = self._test_baymodel_create_and_delete(
+            'test_k8s_baymodel', delete=False, tls_disabled=True)
+        self._test_bay_create_and_delete('test_k8s_bay', baymodel_uuid)
 
 
-class TestKubernetesAPIs(BaseMagnumClient):
+class TestKubernetesAPIs(BayAPITLSTest):
     @classmethod
     def setUpClass(cls):
         super(TestKubernetesAPIs, cls).setUpClass()
-        cls.baymodel = cls._create_baymodel('testk8sAPI')
+
+        cls.baymodel = cls._create_baymodel('testk8sAPI',
+                                            coe='kubernetes',
+                                            tls_disabled=False,
+                                            network_driver='flannel',
+                                            fixed_network='192.168.0.0/24',
+                                            )
         cls.bay = cls._create_bay('testk8sAPI', cls.baymodel.uuid)
-        kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
-        k8s_client = api_client.ApiClient(kube_api_url)
+
+        config_contents = """[req]
+distinguished_name = req_distinguished_name
+req_extensions     = req_ext
+prompt = no
+[req_distinguished_name]
+CN = Your Name
+[req_ext]
+extendedKeyUsage = clientAuth
+"""
+        cls._create_tls_ca_files(config_contents)
+        cls.kube_api_url = cls.cs.bays.get(cls.bay.uuid).api_address
+        k8s_client = api_client.ApiClient(cls.kube_api_url,
+                                          key_file=cls.key_file,
+                                          cert_file=cls.cert_file,
+                                          ca_certs=cls.ca_file)
         cls.k8s_api = apiv_api.ApivApi(k8s_client)
 
     @classmethod

magnum/tests/functional/python_client_base.py

@@ -19,10 +19,13 @@ Tests for `magnum` module.
 
 import ConfigParser
 import os
+import subprocess
+import tempfile
 import time
 
 import fixtures
 
+from magnum.common.utils import rmtree_without_raise
 from magnum.tests import base
 from magnumclient.openstack.common.apiclient import exceptions
 from magnumclient.openstack.common import cliutils
@@ -92,21 +95,28 @@ class BaseMagnumClient(base.TestCase):
                 raise Exception("Unknown Status : %s" % status)
 
     @classmethod
-    def _create_baymodel(cls, name, coe='kubernetes'):
+    def _create_baymodel(cls, name, **kwargs):
+        # TODO(eliqiao): We don't want these to be have default values,
+        #                just leave them here to make things work.
+        #                Plan is to support other kinds of baymodel creation.
+        coe = kwargs.pop('coe', 'kubernetes')
+        docker_volume_size = kwargs.pop('docker_volume_size', 1)
+        network_driver = kwargs.pop('network_driver', 'flannel')
+        labels = kwargs.pop('labels', {"K1": "V1", "K2": "V2"})
+        tls_disabled = kwargs.pop('tls_disabled', False)
+
         baymodel = cls.cs.baymodels.create(
             name=name,
             keypair_id=cls.keypair_id,
             external_network_id=cls.nic_id,
             image_id=cls.image_id,
             flavor_id=cls.flavor_id,
-            docker_volume_size=1,
-            network_driver='flannel',
+            docker_volume_size=docker_volume_size,
+            network_driver=network_driver,
             coe=coe,
-            labels={"K1": "V1", "K2": "V2"},
-            # TODO(yuanying): Change to `tls_disabled=False`
-            #                 if k8sclient supports TLS.
-            tls_disabled=True,
-        )
+            labels=labels,
+            tls_disabled=tls_disabled,
+            **kwargs)
         return baymodel
 
     @classmethod
@@ -150,8 +160,9 @@ class BayTest(BaseMagnumClient):
         if test_timeout > 0:
             self.useFixture(fixtures.Timeout(test_timeout, gentle=True))
 
-    def _test_baymodel_create_and_delete(self, delete=True):
-        baymodel = self._create_baymodel('testbay', coe=self.coe)
+    def _test_baymodel_create_and_delete(self, baymodel_name,
+                                         delete=True, **kwargs):
+        baymodel = self._create_baymodel(baymodel_name, coe=self.coe, **kwargs)
         list = [item.uuid for item in self.cs.baymodels.list()]
         self.assertIn(baymodel.uuid, list)
 
@@ -162,9 +173,9 @@ class BayTest(BaseMagnumClient):
             list = [item.uuid for item in self.cs.baymodels.list()]
             self.assertNotIn(baymodel.uuid, list)
 
-    def _test_bay_create_and_delete(self):
-        baymodel = self._test_baymodel_create_and_delete(delete=False)
-        bay = self._create_bay('testbay', baymodel.uuid)
+    def _test_bay_create_and_delete(self, bay_name, baymodel):
+        # NOTE(eliqiao): baymodel will be deleted after this testing
+        bay = self._create_bay(bay_name, baymodel.uuid)
         list = [item.uuid for item in self.cs.bays.list()]
         self.assertIn(bay.uuid, list)
 
@@ -189,3 +200,71 @@ class BayTest(BaseMagnumClient):
                 self.cs.baymodels.delete(baymodel.uuid)
             except exceptions.BadRequest:
                 pass
+
+
+class BayAPITLSTest(BaseMagnumClient):
+    """Base class of TLS enabled test case."""
+
+    def setUp(self):
+        super(BayAPITLSTest, self).setUp()
+
+    @classmethod
+    def tearDownClass(cls):
+
+        if cls.ca_dir:
+            rmtree_without_raise(cls.ca_dir)
+
+        cls._delete_bay(cls.bay.uuid)
+        try:
+            cls._wait_on_status(cls.bay,
+                                ["CREATE_COMPLETE",
+                                 "DELETE_IN_PROGRESS", "CREATE_FAILED"],
+                                ["DELETE_FAILED", "DELETE_COMPLETE"])
+        except exceptions.NotFound:
+            pass
+        cls._delete_baymodel(cls.baymodel.uuid)
+
+    @classmethod
+    def _create_tls_ca_files(cls, client_conf_contents):
+        """Creates ca files by client_conf_contents."""
+
+        cls.ca_dir = tempfile.mkdtemp()
+        cls.csr_file = '%s/client.csr' % cls.ca_dir
+        cls.client_config_file = '%s/client.conf' % cls.ca_dir
+
+        cls.key_file = '%s/client.key' % cls.ca_dir
+        cls.cert_file = '%s/client.crt' % cls.ca_dir
+        cls.ca_file = '%s/ca.crt' % cls.ca_dir
+
+        with open(cls.client_config_file, 'w') as f:
+            f.write(client_conf_contents)
+
+        def _write_client_key():
+            subprocess.call(['openssl', 'genrsa',
+                             '-out', cls.key_file,
+                             '4096'])
+
+        def _create_client_csr():
+            subprocess.call(['openssl', 'req', '-new',
+                             '-days', '365',
+                             '-key', cls.key_file,
+                             '-out', cls.csr_file,
+                             '-config', cls.client_config_file])
+
+        _write_client_key()
+        _create_client_csr()
+
+        with open(cls.csr_file, 'r') as f:
+            csr_content = f.read()
+
+        # magnum ca-sign --bay secure-k8sbay --csr client.csr > client.crt
+        resp = cls.cs.certificates.create(bay_uuid=cls.bay.uuid,
+                                          csr=csr_content)
+
+        with open(cls.cert_file, 'w') as f:
+            f.write(resp.pem)
+
+        # magnum ca-show --bay secure-k8sbay > ca.crt
+        resp = cls.cs.certificates.get(cls.bay.uuid)
+        with open(cls.ca_file, 'w') as f:
+            f.write(resp.pem)