diff --git a/devstack/plugin.sh b/devstack/plugin.sh index f3df2949..1b103ba0 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -81,6 +81,23 @@ function configure_k8s_certificates { popd } +function configure_etcd_certificates { + pushd $QINLING_DIR + + mkdir_chown_stack $QINLING_CONF_DIR/pki/etcd + sudo cp /etc/kubernetes/pki/etcd/ca.crt $QINLING_CONF_DIR/pki/etcd/ + + # Re-use k8s api server etcd client cert + sudo cp /etc/kubernetes/pki/apiserver-etcd-client.crt $QINLING_CONF_DIR/pki/etcd/qinling-etcd-client.crt + sudo cp /etc/kubernetes/pki/apiserver-etcd-client.key $QINLING_CONF_DIR/pki/etcd/qinling-etcd-client.key + + mkdir_chown_stack $QINLING_CONF_DIR/pki/etcd + # For the tempest user to read the key file when running tempest + chmod 644 $QINLING_CONF_DIR/pki/etcd/qinling-etcd-client.key + + popd +} + function configure_qinling { mkdir_chown_stack "$QINLING_AUTH_CACHE_DIR" @@ -123,6 +140,9 @@ function configure_qinling { iniset $QINLING_CONF_FILE kubernetes use_api_certificate False fi + # Config etcd TLS certs + configure_etcd_certificates + iniset $QINLING_CONF_FILE kubernetes replicas 5 } diff --git a/lower-constraints.txt b/lower-constraints.txt index 01ef335b..1815e1f1 100644 --- a/lower-constraints.txt +++ b/lower-constraints.txt @@ -20,7 +20,7 @@ debtcollector==1.19.0 decorator==4.2.1 dogpile.cache==0.6.5 enum-compat==0.0.2 -etcd3gw==0.2.2 +etcd3gw==0.2.3 eventlet==0.20.0 extras==1.0.0 fasteners==0.14.1 diff --git a/qinling/config.py b/qinling/config.py index 22c717de..584608c9 100644 --- a/qinling/config.py +++ b/qinling/config.py @@ -186,8 +186,40 @@ kubernetes_opts = [ ETCD_GROUP = 'etcd' etcd_opts = [ - cfg.StrOpt('host', default='127.0.0.1', help='Etcd service host address.'), - cfg.PortOpt('port', default=2379, help='Etcd service port.'), + cfg.StrOpt( + 'host', + default='127.0.0.1', + help='Etcd service host address.' + ), + cfg.PortOpt( + 'port', + default=2379, + help='Etcd service port.' + ), + cfg.StrOpt( + 'protocol', + default='https', + choices=['http', 'https'], + help='Etcd connection protocol.' + ), + cfg.StrOpt( + 'ca_cert', + default='/etc/qinling/pki/etcd/ca.crt', + help='Path to CA certificate file to use to securely ' + 'connect to etcd server.' + ), + cfg.StrOpt( + 'cert_file', + default='/etc/qinling/pki/etcd/qinling-etcd-client.crt', + help='Path to client certificate file to use to securely ' + 'connect to etcd server.' + ), + cfg.StrOpt( + 'cert_key', + default='/etc/qinling/pki/etcd/qinling-etcd-client.key', + help='Path to client certificate key file to use to securely ' + 'connect to etcd server.' + ), ] RLIMITS_GROUP = 'resource_limits' diff --git a/qinling/utils/etcd_util.py b/qinling/utils/etcd_util.py index 9d5c4ec4..586e4592 100644 --- a/qinling/utils/etcd_util.py +++ b/qinling/utils/etcd_util.py @@ -24,7 +24,12 @@ def get_client(conf=None): conf = conf or CONF if not CLIENT: - CLIENT = etcd3gw.client(host=conf.etcd.host, port=conf.etcd.port) + CLIENT = etcd3gw.client(host=conf.etcd.host, + port=conf.etcd.port, + protocol=conf.etcd.protocol, + ca_cert=conf.etcd.ca_cert, + cert_cert=conf.etcd.cert_file, + cert_key=conf.etcd.cert_key) return CLIENT diff --git a/qinling_tempest_plugin/config.py b/qinling_tempest_plugin/config.py index c0e1f8e7..706d8224 100644 --- a/qinling_tempest_plugin/config.py +++ b/qinling_tempest_plugin/config.py @@ -45,10 +45,4 @@ QinlingGroup = [ cfg.StrOpt("nodejs_runtime_image", default="openstackqinling/nodejs-runtime:0.0.1", help="The NodeJS runtime being used in the tests."), - cfg.StrOpt('etcd_host', - default='127.0.0.1', - help='Etcd service host address.'), - cfg.PortOpt('etcd_port', - default=2379, - help='Etcd service port.'), ] diff --git a/qinling_tempest_plugin/tests/api/test_executions.py b/qinling_tempest_plugin/tests/api/test_executions.py index 5a40d226..4db0631e 100644 --- a/qinling_tempest_plugin/tests/api/test_executions.py +++ b/qinling_tempest_plugin/tests/api/test_executions.py @@ -13,20 +13,18 @@ # limitations under the License. from concurrent import futures -import etcd3gw import hashlib import json import requests import futurist from oslo_serialization import jsonutils -from tempest import config from tempest.lib import decorators from tempest.lib import exceptions from qinling_tempest_plugin.tests import base +from qinling_tempest_plugin.tests import utils -CONF = config.CONF INVOKE_ERROR = "Function execution failed because of too much resource " \ "consumption" @@ -83,8 +81,7 @@ class ExecutionsTest(base.BaseQinlingTest): """ function_id = self.create_function() - etcd3_client = etcd3gw.client(host=CONF.qinling.etcd_host, - port=CONF.qinling.etcd_port) + etcd3_client = utils.get_etcd_client() lock_id = "function_worker_%s_%s" % (function_id, 0) with etcd3_client.lock(id=lock_id): resp, body = self.client.create_execution( diff --git a/qinling_tempest_plugin/tests/api/test_function_versions.py b/qinling_tempest_plugin/tests/api/test_function_versions.py index 210ca8e9..9732a984 100644 --- a/qinling_tempest_plugin/tests/api/test_function_versions.py +++ b/qinling_tempest_plugin/tests/api/test_function_versions.py @@ -12,15 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import etcd3gw -from tempest import config from tempest.lib import decorators from tempest.lib import exceptions import tenacity from qinling_tempest_plugin.tests import base - -CONF = config.CONF +from qinling_tempest_plugin.tests import utils class FunctionVersionsTest(base.BaseQinlingTest): @@ -104,8 +101,7 @@ class FunctionVersionsTest(base.BaseQinlingTest): """ function_id = self.create_function() - etcd3_client = etcd3gw.client(host=CONF.qinling.etcd_host, - port=CONF.qinling.etcd_port) + etcd3_client = utils.get_etcd_client() lock_id = "function_version_%s" % function_id with etcd3_client.lock(id=lock_id): self.assertRaises( diff --git a/qinling_tempest_plugin/tests/utils.py b/qinling_tempest_plugin/tests/utils.py index a2acc8ad..1f63ed88 100644 --- a/qinling_tempest_plugin/tests/utils.py +++ b/qinling_tempest_plugin/tests/utils.py @@ -14,6 +14,13 @@ import hashlib +from oslo_config import cfg + +from qinling import config as qinling_config +from qinling.utils import etcd_util + +QINLING_CONF = None + def md5(file=None, content=None): hash_md5 = hashlib.md5() @@ -26,3 +33,16 @@ def md5(file=None, content=None): hash_md5.update(content) return hash_md5.hexdigest() + + +def get_etcd_client(): + """Use qinling's default CONF to connect to etcd.""" + global QINLING_CONF + + if not QINLING_CONF: + QINLING_CONF = cfg.ConfigOpts() + QINLING_CONF(args=[], project='qinling') + QINLING_CONF.register_opts(qinling_config.etcd_opts, + qinling_config.ETCD_GROUP) + + return etcd_util.get_client(conf=QINLING_CONF) diff --git a/releasenotes/notes/secure-connection-etcd-15c6a6c4ea49c33e.yaml b/releasenotes/notes/secure-connection-etcd-15c6a6c4ea49c33e.yaml new file mode 100644 index 00000000..83887345 --- /dev/null +++ b/releasenotes/notes/secure-connection-etcd-15c6a6c4ea49c33e.yaml @@ -0,0 +1,5 @@ +--- +features: + - Qinling can now use TLS certificates to securely connect to the etcd + service. It is enabled by default in Qinling Devstack installation. The + configuration is defined in ``etcd`` section in Qinling config file. diff --git a/requirements.txt b/requirements.txt index c4d242a6..10e34242 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,5 +29,5 @@ croniter>=0.3.4 # MIT License python-dateutil>=2.5.3 # BSD tenacity>=4.4.0 # Apache-2.0 PyMySQL>=0.7.6 # MIT License -etcd3gw>=0.2.2 # Apache-2.0 +etcd3gw>=0.2.3 # Apache-2.0 cotyledon>=1.3.0 # Apache-2.0 diff --git a/tools/gate/kubeadm/funcs/common.sh b/tools/gate/kubeadm/funcs/common.sh index 40f18032..145c9644 100644 --- a/tools/gate/kubeadm/funcs/common.sh +++ b/tools/gate/kubeadm/funcs/common.sh @@ -55,6 +55,6 @@ function gate_base_setup { function create_k8s_screen { # Starts a proxy to the Kubernetes API server in a screen session sudo screen -S kube_proxy -X quit || true - sudo screen -dmS kube_proxy && sudo screen -S kube_proxy -X screen -t kube_proxy - sudo screen -S kube_proxy -p kube_proxy -X stuff 'kubectl proxy --accept-hosts=".*" --address="0.0.0.0"\n' + sudo screen -dmS kube_proxy + sudo screen -S kube_proxy -p bash -X stuff 'kubectl proxy --accept-hosts=".*" --address="0.0.0.0"\n' } \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/k8s_cli/defaults/main.yaml b/tools/gate/kubeadm/playbook/roles/k8s_cli/defaults/main.yaml new file mode 100644 index 00000000..c468dbd4 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/k8s_cli/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +k8s_version: "1.11.0" \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/centos.yml b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/centos.yml index bda39644..46dc2182 100644 --- a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/centos.yml +++ b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/centos.yml @@ -18,9 +18,9 @@ state: present update_cache: yes with_items: - - kubelet-1.9.3 - - kubectl-1.9.3 - - kubeadm-1.9.3 + - kubelet-{{ k8s_version }} + - kubectl-{{ k8s_version }} + - kubeadm-{{ k8s_version }} # On CentOS 7, the kubeadm package adds a configuration file that overrides # kubelet's cgroup driver to systemd, we want to use the default driver diff --git a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/ubuntu.yml b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/ubuntu.yml index 255ef5ca..08c33111 100644 --- a/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/ubuntu.yml +++ b/tools/gate/kubeadm/playbook/roles/k8s_cli/tasks/ubuntu.yml @@ -15,6 +15,6 @@ state: present update_cache: yes with_items: - - kubelet=1.9.3-00 - - kubectl=1.9.3-00 - - kubeadm=1.9.3-00 + - kubelet={{ k8s_version }}-00 + - kubectl={{ k8s_version }}-00 + - kubeadm={{ k8s_version }}-00 diff --git a/tools/gate/kubeadm/playbook/roles/kube_master/defaults/main.yaml b/tools/gate/kubeadm/playbook/roles/kube_master/defaults/main.yaml new file mode 100644 index 00000000..c468dbd4 --- /dev/null +++ b/tools/gate/kubeadm/playbook/roles/kube_master/defaults/main.yaml @@ -0,0 +1,2 @@ +--- +k8s_version: "1.11.0" \ No newline at end of file diff --git a/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml b/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml index 4a1d8be0..38e66bb8 100644 --- a/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml +++ b/tools/gate/kubeadm/playbook/roles/kube_master/tasks/main.yml @@ -17,7 +17,7 @@ - name: Init master vars: extra_opts: "{{ lookup('env', 'EXTRA_KUBEADM_INIT_OPTS') }}" - shell: kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=1.9.3 {{ extra_opts }} + shell: kubeadm init --pod-network-cidr=192.168.0.0/16 --kubernetes-version=v{{ k8s_version }} {{ extra_opts }} register: kubeadm_init - fail: diff --git a/tools/gate/kubeadm/vars.sh b/tools/gate/kubeadm/vars.sh index 94e27ce7..a3529b65 100644 --- a/tools/gate/kubeadm/vars.sh +++ b/tools/gate/kubeadm/vars.sh @@ -12,8 +12,5 @@ source /etc/os-release export HOST_OS=${HOST_OS:="${ID}"} -# Set versions of K8s to use -export KUBE_VERSION=${KUBE_VERSION:-"v1.8.0"} - # Set Upstream DNS -export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"} \ No newline at end of file +export UPSTREAM_DNS=${UPSTREAM_DNS:-"8.8.8.8"}