Add sidecar support for runtime

Split function package download functionality into a separate
container, so that it will be easy to add additional runtime support.

I was planning to use kubernetes new verion apps api in qinling, but
failed because the kubernetes client in global requirement is not
updated yet, so I comment the code out in case we use them in future.

Change-Id: I0709b8ac689638b78e00ca35d0fea2db6cae0d0f
Story: 2001580
Task: 6607

devstack/plugin.sh

@@ -27,6 +27,7 @@ function install_k8s {
     # Pre-fetch the default docker image for python runtime and image function
     # test.
     sudo docker pull $QINLING_PYTHON_RUNTIME_IMAGE
+    sudo docker pull openstackqinling/sidecar
     sudo docker pull openstackqinling/alpine-test
 }
 

qinling/orchestrator/kubernetes/manager.py

@@ -42,6 +42,7 @@ class KubernetesManager(base.OrchestratorBase):
         clients = k8s_util.get_k8s_clients(self.conf)
         self.v1 = clients['v1']
         self.v1extention = clients['v1extention']
+        # self.apps_v1 = clients['apps_v1']
 
         # Create namespace if not exists
         self._ensure_namespace()

qinling/orchestrator/kubernetes/templates/deployment.j2

@@ -21,12 +21,19 @@ spec:
       {% endfor %}
     spec:
       terminationGracePeriodSeconds: 5
+      automountServiceAccountToken: false
+      volumes:
+      - name: package-folder
+        emptyDir: {}
       containers:
       - name: {{ container_name }}
         image: {{ image }}
         imagePullPolicy: IfNotPresent
         ports:
         - containerPort: 9090
+        volumeMounts:
+          - name: package-folder
+            mountPath: /var/qinling/packages
         resources:
           limits:
             cpu: "0.3"
@@ -34,3 +41,11 @@ spec:
           requests:
             cpu: "0.1"
             memory: 32Mi
+      - name: sidecar
+        image: openstackqinling/sidecar
+        imagePullPolicy: IfNotPresent
+        ports:
+          - containerPort: 9091
+        volumeMounts:
+          - name: package-folder
+            mountPath: /var/qinling/packages

qinling/orchestrator/kubernetes/templates/pod.j2

@@ -8,6 +8,7 @@ metadata:
   {% endfor %}
 spec:
   terminationGracePeriodSeconds: 5
+  automountServiceAccountToken: false
   containers:
     - name: {{ pod_name }}
       image: {{ pod_image }}

qinling/orchestrator/kubernetes/templates/service.j2

@@ -15,3 +15,4 @@ spec:
   ports:
     - protocol: TCP
       port: 9090
+      targetPort: 9090

qinling/orchestrator/kubernetes/utils.py

@@ -13,6 +13,7 @@
 #    limitations under the License.
 
 from kubernetes.client import api_client
+# from kubernetes.client.apis import apps_v1_api
 from kubernetes.client.apis import core_v1_api
 from kubernetes.client.apis import extensions_v1beta1_api
 from kubernetes.client import configuration as k8s_config
@@ -25,9 +26,11 @@ def get_k8s_clients(conf):
     client = api_client.ApiClient(configuration=config)
     v1 = core_v1_api.CoreV1Api(client)
     v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client)
+    # apps_v1 = apps_v1_api.AppsV1Api(client)
 
     clients = {
         'v1': v1,
+        # 'apps_v1': apps_v1
         'v1extention': v1extention
     }
 

qinling_tempest_plugin/tests/base.py

@@ -50,6 +50,7 @@ class BaseQinlingTest(test.BaseTestCase):
         clients = utils.get_k8s_clients(CONF)
         cls.k8s_v1 = clients['v1']
         cls.k8s_v1extention = clients['v1extention']
+        # cls.k8s_apps_v1 = clients['apps_v1']
         cls.namespace = 'qinling'
 
     @classmethod

qinling_tempest_plugin/tests/utils.py

@@ -15,6 +15,7 @@
 import hashlib
 
 from kubernetes.client import api_client
+# from kubernetes.client.apis import apps_v1_api
 from kubernetes.client.apis import core_v1_api
 from kubernetes.client.apis import extensions_v1beta1_api
 from kubernetes.client import configuration as k8s_config
@@ -27,10 +28,12 @@ def get_k8s_clients(conf):
     client = api_client.ApiClient(configuration=config)
     v1 = core_v1_api.CoreV1Api(client)
     v1extention = extensions_v1beta1_api.ExtensionsV1beta1Api(client)
+    # apps_v1 = apps_v1_api.AppsV1Api(client)
 
     clients = {
         'v1': v1,
         'v1extention': v1extention
+        # 'apps_v1': apps_v1
     }
 
     return clients

runtimes/python2/Dockerfile

@@ -1,6 +1,7 @@
 FROM phusion/baseimage:0.9.22
 MAINTAINER anlin.kong@gmail.com
 
+# We need to use non-root user because root user is not affected by ulimit.
 USER root
 RUN useradd -Ms /bin/bash qinling
 
@@ -10,10 +11,9 @@ RUN apt-get update && \
 
 COPY . /app
 WORKDIR /app
-RUN pip install -r requirements.txt && \
+RUN pip install --no-cache-dir -r requirements.txt && \
-    chmod 0750 custom-entrypoint.sh && \
+    mkdir -p /var/qinling/packages && \
-    mkdir -p /var/lock/qinling && \
+    chown -R qinling:qinling /app /var/qinling/packages
-    chown -R qinling:qinling /app /var/lock/qinling
 
 # uwsgi --http :9090 --uid qinling --wsgi-file server.py --callable app --master --processes 5 --threads 1
 CMD ["/usr/local/bin/uwsgi", "--http", ":9090", "--uid", "qinling", "--wsgi-file", "server.py", "--callable", "app", "--master", "--processes", "5", "--threads", "1"]

runtimes/python2/requirements.txt

@@ -1,5 +1,4 @@
 Flask>=0.10,!=0.11,<1.0  # BSD
-oslo.concurrency>=3.25.0  # Apache-2.0
 python-openstackclient>=3.3.0,!=3.10.0  # Apache-2.0
 python-neutronclient>=6.3.0  # Apache-2.0
 python-swiftclient>=3.2.0  # Apache-2.0

runtimes/python2/server.py

@@ -27,7 +27,6 @@ from flask import request
 from flask import Response
 from keystoneauth1.identity import generic
 from keystoneauth1 import session
-from oslo_concurrency import lockutils
 import requests
 
 app = Flask(__name__)
@@ -73,41 +72,12 @@ def _get_responce(output, duration, logs, success, code):
     )
 
 
-@lockutils.synchronized('download_function', external=True,
-                        lock_path='/var/lock/qinling')
-def _download_package(url, zip_file, token=None):
-    if os.path.isfile(zip_file):
-        return True, None
-
-    print('Downloading function, download_url:%s' % url)
-
-    headers = {}
-    if token:
-        headers = {'X-Auth-Token': token}
-
-    try:
-        r = requests.get(url, headers=headers, stream=True, timeout=5,
-                         verify=False)
-        if r.status_code != 200:
-            return False, _get_responce(
-                DOWNLOAD_ERROR % (url, r.content), 0, '', False, 500
-            )
-
-        with open(zip_file, 'wb') as fd:
-            for chunk in r.iter_content(chunk_size=65535):
-                fd.write(chunk)
-    except Exception as e:
-        return False, _get_responce(
-            DOWNLOAD_ERROR % (url, str(e)), 0, '', False, 500
-        )
-
-    print('Downloaded function package to %s' % zip_file)
-    return True, None
-
-
 def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
                      return_dict):
     """Thie function is supposed to be running in a child process."""
+    # Set resource limit for current sub-process
+    _set_ulimit()
+
     sys.path.insert(0, zip_file)
     sys.stdout = open("%s.out" % execution_id, "w", 0)
 
@@ -152,7 +122,7 @@ def execute():
     auth_url = params.get('auth_url')
     username = params.get('username')
     password = params.get('password')
-    zip_file = '%s.zip' % function_id
+    zip_file = '/var/qinling/packages/%s.zip' % function_id
 
     function_module, function_method = 'main', 'main'
     if entry:
@@ -164,16 +134,28 @@ def execute():
         (request_id, execution_id, input, auth_url)
     )
 
-    # Download function package if needed.
+    ####################################################################
-    ret, resp = _download_package(
+    #
-        download_url,
+    # Download function package by calling sidecar service. We don't check the
-        zip_file,
+    # zip file existence here to avoid using partial file during downloading.
-        params.get('token')
+    #
+    ####################################################################
+    resp = requests.post(
+        'http://localhost:9091/download',
+        json={
+            'download_url': download_url,
+            'function_id': function_id,
+            'token': params.get('token')
+        }
     )
-    if not ret:
+    if not resp.ok:
-        return resp
+        return _get_responce(resp.content, 0, '', False, 500)
 
+    ####################################################################
+    #
     # Provide an openstack session to user's function
+    #
+    ####################################################################
     os_session = None
     if auth_url:
         auth = generic.Password(
@@ -186,9 +168,11 @@ def execute():
         os_session = session.Session(auth=auth, verify=False)
     input.update({'context': {'os_session': os_session}})
 
-    # Set resource limit
+    ####################################################################
-    _set_ulimit()
+    #
-
+    # Create a new process to run user's function
+    #
+    ####################################################################
     manager = Manager()
     return_dict = manager.dict()
     return_dict['success'] = False
@@ -203,6 +187,11 @@ def execute():
     p.start()
     p.join()
 
+    ####################################################################
+    #
+    # Get execution output(log, duration, etc.)
+    #
+    ####################################################################
     duration = round(time.time() - start, 3)
 
     # Process was killed unexpectedly or finished with error.

runtimes/sidecar/Dockerfile

@@ -0,0 +1,23 @@
+FROM alpine:3.7
+MAINTAINER lingxian.kong@gmail.com
+
+# We need to use qinling user to keep consistent with server.
+USER root
+RUN adduser -HDs /bin/sh qinling
+
+RUN apk update && \
+    apk add --no-cache linux-headers build-base python2 python2-dev py2-pip uwsgi-python uwsgi-http && \
+    pip install --upgrade pip && \
+    rm -r /root/.cache
+
+COPY . /sidecar
+WORKDIR /sidecar
+RUN pip install --no-cache-dir -r requirements.txt && \
+    mkdir -p /var/lock/qinling && \
+    mkdir -p /var/qinling/packages && \
+    chown -R qinling:qinling /sidecar /var/lock/qinling /var/qinling/packages
+
+EXPOSE 9091
+
+# uwsgi --plugin http,python --http :9091 --uid qinling --wsgi-file sidecar.py --callable app --master --processes 1 --threads 1
+CMD ["/usr/sbin/uwsgi", "--plugin", "http,python", "--http", "127.0.0.1:9091", "--uid", "qinling", "--wsgi-file", "sidecar.py", "--callable", "app", "--master", "--processes", "1", "--threads", "1"]

runtimes/sidecar/requirements.txt

@@ -0,0 +1,3 @@
+Flask>=0.10,!=0.11,<1.0  # BSD
+oslo.concurrency>=3.25.0  # Apache-2.0
+requests>=2.18.4

runtimes/sidecar/sidecar.py

@@ -0,0 +1,75 @@
+# Copyright 2018 Catalyst IT Limited
+#
+#    Licensed under the Apache License, Version 2.0 (the "License");
+#    you may not use this file except in compliance with the License.
+#    You may obtain a copy of the License at
+#
+#        http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS,
+#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#    See the License for the specific language governing permissions and
+#    limitations under the License.
+
+import os
+
+from flask import Flask
+from flask import make_response
+from flask import request
+from oslo_concurrency import lockutils
+import requests
+
+app = Flask(__name__)
+
+DOWNLOAD_ERROR = "Failed to download function package from %s, error: %s"
+
+
+@lockutils.synchronized('download_function', external=True,
+                        lock_path='/var/lock/qinling')
+def _download_package(url, zip_file, token=None):
+    """Download package as needed.
+
+    Return None if successful otherwise a Flask.Response object.
+    """
+    if os.path.isfile(zip_file):
+        return None
+
+    print('Downloading function, download_url:%s' % url)
+
+    headers = {}
+    if token:
+        headers = {'X-Auth-Token': token}
+
+    try:
+        r = requests.get(url, headers=headers, stream=True, timeout=5,
+                         verify=False)
+        if r.status_code != 200:
+            return make_response(DOWNLOAD_ERROR % (url, r.content), 500)
+
+        with open(zip_file, 'wb') as fd:
+            for chunk in r.iter_content(chunk_size=65535):
+                fd.write(chunk)
+    except Exception as e:
+        return make_response(DOWNLOAD_ERROR % (url, str(e)), 500)
+
+    print('Downloaded function package to %s' % zip_file)
+
+
+@app.route('/download', methods=['POST'])
+def download():
+    """Download function package to a shared folder.
+
+    The parameters 'download_url' and 'function_id' need to be specified
+    explicitly. It's guaranteed in the server side.
+    """
+    params = request.get_json()
+    zip_file = '/var/qinling/packages/%s.zip' % params['function_id']
+
+    resp = _download_package(
+        params['download_url'],
+        zip_file,
+        params.get('token')
+    )
+
+    return resp if resp else 'downloaded'