From 05588c195747427d83508c5ad7b241bfb2a07fe5 Mon Sep 17 00:00:00 2001 From: Lingxian Kong Date: Mon, 5 Mar 2018 11:21:59 +1300 Subject: [PATCH] Add sidecar support for runtime Split function package download functionality into a separate container, so that it will be easy to add additional runtime support. I was planning to use kubernetes new verion apps api in qinling, but failed because the kubernetes client in global requirement is not updated yet, so I comment the code out in case we use them in future. Change-Id: I0709b8ac689638b78e00ca35d0fea2db6cae0d0f Story: 2001580 Task: 6607 --- devstack/plugin.sh | 1 + qinling/orchestrator/kubernetes/manager.py | 1 + .../kubernetes/templates/deployment.j2 | 15 ++++ .../orchestrator/kubernetes/templates/pod.j2 | 1 + .../kubernetes/templates/service.j2 | 1 + qinling/orchestrator/kubernetes/utils.py | 3 + qinling_tempest_plugin/tests/base.py | 1 + qinling_tempest_plugin/tests/utils.py | 3 + runtimes/python2/Dockerfile | 8 +- runtimes/python2/requirements.txt | 1 - runtimes/python2/server.py | 77 ++++++++----------- runtimes/sidecar/Dockerfile | 23 ++++++ runtimes/sidecar/requirements.txt | 3 + runtimes/sidecar/sidecar.py | 75 ++++++++++++++++++ 14 files changed, 164 insertions(+), 49 deletions(-) create mode 100644 runtimes/sidecar/Dockerfile create mode 100644 runtimes/sidecar/requirements.txt create mode 100644 runtimes/sidecar/sidecar.py diff --git a/devstack/plugin.sh b/devstack/plugin.sh index 96fa6e30..814531d9 100755 --- a/devstack/plugin.sh +++ b/devstack/plugin.sh @@ -27,6 +27,7 @@ function install_k8s { # Pre-fetch the default docker image for python runtime and image function # test. sudo docker pull $QINLING_PYTHON_RUNTIME_IMAGE + sudo docker pull openstackqinling/sidecar sudo docker pull openstackqinling/alpine-test } diff --git a/qinling/orchestrator/kubernetes/manager.py b/qinling/orchestrator/kubernetes/manager.py index 0e37d3af..5f336ec5 100644 --- a/qinling/orchestrator/kubernetes/manager.py +++ b/qinling/orchestrator/kubernetes/manager.py @@ -42,6 +42,7 @@ class KubernetesManager(base.OrchestratorBase): clients = k8s_util.get_k8s_clients(self.conf) self.v1 = clients['v1'] self.v1extention = clients['v1extention'] + # self.apps_v1 = clients['apps_v1'] # Create namespace if not exists self._ensure_namespace() diff --git a/qinling/orchestrator/kubernetes/templates/deployment.j2 b/qinling/orchestrator/kubernetes/templates/deployment.j2 index 5be50bf2..8c76dfcf 100644 --- a/qinling/orchestrator/kubernetes/templates/deployment.j2 +++ b/qinling/orchestrator/kubernetes/templates/deployment.j2 @@ -21,12 +21,19 @@ spec: {% endfor %} spec: terminationGracePeriodSeconds: 5 + automountServiceAccountToken: false + volumes: + - name: package-folder + emptyDir: {} containers: - name: {{ container_name }} image: {{ image }} imagePullPolicy: IfNotPresent ports: - containerPort: 9090 + volumeMounts: + - name: package-folder + mountPath: /var/qinling/packages resources: limits: cpu: "0.3" @@ -34,3 +41,11 @@ spec: requests: cpu: "0.1" memory: 32Mi + - name: sidecar + image: openstackqinling/sidecar + imagePullPolicy: IfNotPresent + ports: + - containerPort: 9091 + volumeMounts: + - name: package-folder + mountPath: /var/qinling/packages diff --git a/qinling/orchestrator/kubernetes/templates/pod.j2 b/qinling/orchestrator/kubernetes/templates/pod.j2 index fcb061bb..083f56f6 100644 --- a/qinling/orchestrator/kubernetes/templates/pod.j2 +++ b/qinling/orchestrator/kubernetes/templates/pod.j2 @@ -8,6 +8,7 @@ metadata: {% endfor %} spec: terminationGracePeriodSeconds: 5 + automountServiceAccountToken: false containers: - name: {{ pod_name }} image: {{ pod_image }} diff --git a/qinling/orchestrator/kubernetes/templates/service.j2 b/qinling/orchestrator/kubernetes/templates/service.j2 index 64bdf3ed..84be4094 100644 --- a/qinling/orchestrator/kubernetes/templates/service.j2 +++ b/qinling/orchestrator/kubernetes/templates/service.j2 @@ -15,3 +15,4 @@ spec: ports: - protocol: TCP port: 9090 + targetPort: 9090 diff --git a/qinling/orchestrator/kubernetes/utils.py b/qinling/orchestrator/kubernetes/utils.py index 14e58e8a..f8ef3c1b 100644 --- a/qinling/orchestrator/kubernetes/utils.py +++ b/qinling/orchestrator/kubernetes/utils.py @@ -13,6 +13,7 @@ # limitations under the License. from kubernetes.client import api_client +# from kubernetes.client.apis import apps_v1_api from kubernetes.client.apis import core_v1_api from kubernetes.client.apis import extensions_v1beta1_api from kubernetes.client import configuration as k8s_config @@ -25,9 +26,11 @@ def get_k8s_clients(conf): client = api_client.ApiClient(configuration=config) v1 = core_v1_api.CoreV1Api(client) v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client) + # apps_v1 = apps_v1_api.AppsV1Api(client) clients = { 'v1': v1, + # 'apps_v1': apps_v1 'v1extention': v1extention } diff --git a/qinling_tempest_plugin/tests/base.py b/qinling_tempest_plugin/tests/base.py index 278bcd55..4e616c8c 100644 --- a/qinling_tempest_plugin/tests/base.py +++ b/qinling_tempest_plugin/tests/base.py @@ -50,6 +50,7 @@ class BaseQinlingTest(test.BaseTestCase): clients = utils.get_k8s_clients(CONF) cls.k8s_v1 = clients['v1'] cls.k8s_v1extention = clients['v1extention'] + # cls.k8s_apps_v1 = clients['apps_v1'] cls.namespace = 'qinling' @classmethod diff --git a/qinling_tempest_plugin/tests/utils.py b/qinling_tempest_plugin/tests/utils.py index 9fbe0d56..8a068869 100644 --- a/qinling_tempest_plugin/tests/utils.py +++ b/qinling_tempest_plugin/tests/utils.py @@ -15,6 +15,7 @@ import hashlib from kubernetes.client import api_client +# from kubernetes.client.apis import apps_v1_api from kubernetes.client.apis import core_v1_api from kubernetes.client.apis import extensions_v1beta1_api from kubernetes.client import configuration as k8s_config @@ -27,10 +28,12 @@ def get_k8s_clients(conf): client = api_client.ApiClient(configuration=config) v1 = core_v1_api.CoreV1Api(client) v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client) + # apps_v1 = apps_v1_api.AppsV1Api(client) clients = { 'v1': v1, 'v1extention': v1extention + # 'apps_v1': apps_v1 } return clients diff --git a/runtimes/python2/Dockerfile b/runtimes/python2/Dockerfile index d9a515d9..20a6faea 100644 --- a/runtimes/python2/Dockerfile +++ b/runtimes/python2/Dockerfile @@ -1,6 +1,7 @@ FROM phusion/baseimage:0.9.22 MAINTAINER anlin.kong@gmail.com +# We need to use non-root user because root user is not affected by ulimit. USER root RUN useradd -Ms /bin/bash qinling @@ -10,10 +11,9 @@ RUN apt-get update && \ COPY . /app WORKDIR /app -RUN pip install -r requirements.txt && \ - chmod 0750 custom-entrypoint.sh && \ - mkdir -p /var/lock/qinling && \ - chown -R qinling:qinling /app /var/lock/qinling +RUN pip install --no-cache-dir -r requirements.txt && \ + mkdir -p /var/qinling/packages && \ + chown -R qinling:qinling /app /var/qinling/packages # uwsgi --http :9090 --uid qinling --wsgi-file server.py --callable app --master --processes 5 --threads 1 CMD ["/usr/local/bin/uwsgi", "--http", ":9090", "--uid", "qinling", "--wsgi-file", "server.py", "--callable", "app", "--master", "--processes", "5", "--threads", "1"] diff --git a/runtimes/python2/requirements.txt b/runtimes/python2/requirements.txt index d571a5b5..32dddd28 100644 --- a/runtimes/python2/requirements.txt +++ b/runtimes/python2/requirements.txt @@ -1,5 +1,4 @@ Flask>=0.10,!=0.11,<1.0 # BSD -oslo.concurrency>=3.25.0 # Apache-2.0 python-openstackclient>=3.3.0,!=3.10.0 # Apache-2.0 python-neutronclient>=6.3.0 # Apache-2.0 python-swiftclient>=3.2.0 # Apache-2.0 diff --git a/runtimes/python2/server.py b/runtimes/python2/server.py index e911a0ab..bc230d1f 100644 --- a/runtimes/python2/server.py +++ b/runtimes/python2/server.py @@ -27,7 +27,6 @@ from flask import request from flask import Response from keystoneauth1.identity import generic from keystoneauth1 import session -from oslo_concurrency import lockutils import requests app = Flask(__name__) @@ -73,41 +72,12 @@ def _get_responce(output, duration, logs, success, code): ) -@lockutils.synchronized('download_function', external=True, - lock_path='/var/lock/qinling') -def _download_package(url, zip_file, token=None): - if os.path.isfile(zip_file): - return True, None - - print('Downloading function, download_url:%s' % url) - - headers = {} - if token: - headers = {'X-Auth-Token': token} - - try: - r = requests.get(url, headers=headers, stream=True, timeout=5, - verify=False) - if r.status_code != 200: - return False, _get_responce( - DOWNLOAD_ERROR % (url, r.content), 0, '', False, 500 - ) - - with open(zip_file, 'wb') as fd: - for chunk in r.iter_content(chunk_size=65535): - fd.write(chunk) - except Exception as e: - return False, _get_responce( - DOWNLOAD_ERROR % (url, str(e)), 0, '', False, 500 - ) - - print('Downloaded function package to %s' % zip_file) - return True, None - - def _invoke_function(execution_id, zip_file, module_name, method, arg, input, return_dict): """Thie function is supposed to be running in a child process.""" + # Set resource limit for current sub-process + _set_ulimit() + sys.path.insert(0, zip_file) sys.stdout = open("%s.out" % execution_id, "w", 0) @@ -152,7 +122,7 @@ def execute(): auth_url = params.get('auth_url') username = params.get('username') password = params.get('password') - zip_file = '%s.zip' % function_id + zip_file = '/var/qinling/packages/%s.zip' % function_id function_module, function_method = 'main', 'main' if entry: @@ -164,16 +134,28 @@ def execute(): (request_id, execution_id, input, auth_url) ) - # Download function package if needed. - ret, resp = _download_package( - download_url, - zip_file, - params.get('token') + #################################################################### + # + # Download function package by calling sidecar service. We don't check the + # zip file existence here to avoid using partial file during downloading. + # + #################################################################### + resp = requests.post( + 'http://localhost:9091/download', + json={ + 'download_url': download_url, + 'function_id': function_id, + 'token': params.get('token') + } ) - if not ret: - return resp + if not resp.ok: + return _get_responce(resp.content, 0, '', False, 500) + #################################################################### + # # Provide an openstack session to user's function + # + #################################################################### os_session = None if auth_url: auth = generic.Password( @@ -186,9 +168,11 @@ def execute(): os_session = session.Session(auth=auth, verify=False) input.update({'context': {'os_session': os_session}}) - # Set resource limit - _set_ulimit() - + #################################################################### + # + # Create a new process to run user's function + # + #################################################################### manager = Manager() return_dict = manager.dict() return_dict['success'] = False @@ -203,6 +187,11 @@ def execute(): p.start() p.join() + #################################################################### + # + # Get execution output(log, duration, etc.) + # + #################################################################### duration = round(time.time() - start, 3) # Process was killed unexpectedly or finished with error. diff --git a/runtimes/sidecar/Dockerfile b/runtimes/sidecar/Dockerfile new file mode 100644 index 00000000..8d2b11c1 --- /dev/null +++ b/runtimes/sidecar/Dockerfile @@ -0,0 +1,23 @@ +FROM alpine:3.7 +MAINTAINER lingxian.kong@gmail.com + +# We need to use qinling user to keep consistent with server. +USER root +RUN adduser -HDs /bin/sh qinling + +RUN apk update && \ + apk add --no-cache linux-headers build-base python2 python2-dev py2-pip uwsgi-python uwsgi-http && \ + pip install --upgrade pip && \ + rm -r /root/.cache + +COPY . /sidecar +WORKDIR /sidecar +RUN pip install --no-cache-dir -r requirements.txt && \ + mkdir -p /var/lock/qinling && \ + mkdir -p /var/qinling/packages && \ + chown -R qinling:qinling /sidecar /var/lock/qinling /var/qinling/packages + +EXPOSE 9091 + +# uwsgi --plugin http,python --http :9091 --uid qinling --wsgi-file sidecar.py --callable app --master --processes 1 --threads 1 +CMD ["/usr/sbin/uwsgi", "--plugin", "http,python", "--http", "127.0.0.1:9091", "--uid", "qinling", "--wsgi-file", "sidecar.py", "--callable", "app", "--master", "--processes", "1", "--threads", "1"] \ No newline at end of file diff --git a/runtimes/sidecar/requirements.txt b/runtimes/sidecar/requirements.txt new file mode 100644 index 00000000..b318ae73 --- /dev/null +++ b/runtimes/sidecar/requirements.txt @@ -0,0 +1,3 @@ +Flask>=0.10,!=0.11,<1.0 # BSD +oslo.concurrency>=3.25.0 # Apache-2.0 +requests>=2.18.4 \ No newline at end of file diff --git a/runtimes/sidecar/sidecar.py b/runtimes/sidecar/sidecar.py new file mode 100644 index 00000000..f7f766b5 --- /dev/null +++ b/runtimes/sidecar/sidecar.py @@ -0,0 +1,75 @@ +# Copyright 2018 Catalyst IT Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os + +from flask import Flask +from flask import make_response +from flask import request +from oslo_concurrency import lockutils +import requests + +app = Flask(__name__) + +DOWNLOAD_ERROR = "Failed to download function package from %s, error: %s" + + +@lockutils.synchronized('download_function', external=True, + lock_path='/var/lock/qinling') +def _download_package(url, zip_file, token=None): + """Download package as needed. + + Return None if successful otherwise a Flask.Response object. + """ + if os.path.isfile(zip_file): + return None + + print('Downloading function, download_url:%s' % url) + + headers = {} + if token: + headers = {'X-Auth-Token': token} + + try: + r = requests.get(url, headers=headers, stream=True, timeout=5, + verify=False) + if r.status_code != 200: + return make_response(DOWNLOAD_ERROR % (url, r.content), 500) + + with open(zip_file, 'wb') as fd: + for chunk in r.iter_content(chunk_size=65535): + fd.write(chunk) + except Exception as e: + return make_response(DOWNLOAD_ERROR % (url, str(e)), 500) + + print('Downloaded function package to %s' % zip_file) + + +@app.route('/download', methods=['POST']) +def download(): + """Download function package to a shared folder. + + The parameters 'download_url' and 'function_id' need to be specified + explicitly. It's guaranteed in the server side. + """ + params = request.get_json() + zip_file = '/var/qinling/packages/%s.zip' % params['function_id'] + + resp = _download_package( + params['download_url'], + zip_file, + params.get('token') + ) + + return resp if resp else 'downloaded'