RETIRED, Function as a Service for OpenStack
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 
 
 
 

486 lines
20 KiB

# Copyright 2017 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from concurrent import futures
import etcd3gw
import hashlib
import json
import requests
import futurist
from oslo_serialization import jsonutils
from tempest import config
from tempest.lib import decorators
from tempest.lib import exceptions
from qinling_tempest_plugin.tests import base
CONF = config.CONF
INVOKE_ERROR = "Function execution failed because of too much resource " \
"consumption"
class ExecutionsTest(base.BaseQinlingTest):
name_prefix = 'ExecutionsTest'
def setUp(self):
super(ExecutionsTest, self).setUp()
self.wait_runtime_available(self.runtime_id)
@decorators.idempotent_id('2a93fab0-2dae-4748-b0d4-f06b735ff451')
def test_crud_execution(self):
function_id = self.create_function()
resp, body = self.client.create_execution(function_id,
input='{"name": "Qinling"}')
self.assertEqual(201, resp.status)
execution_id_1 = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id_1, ignore_notfound=True)
self.assertEqual('success', body['status'])
# Create another execution without input
resp, body = self.client.create_execution(function_id)
self.assertEqual(201, resp.status)
execution_id_2 = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id_2, ignore_notfound=True)
self.assertEqual('success', body['status'])
# Get executions
resp, body = self.client.get_resources('executions')
self.assertEqual(200, resp.status)
expected = {execution_id_1, execution_id_2}
actual = set([execution['id'] for execution in body['executions']])
self.assertTrue(expected.issubset(actual))
# Delete executions
resp = self.client.delete_resource('executions', execution_id_1)
self.assertEqual(204, resp.status)
resp = self.client.delete_resource('executions', execution_id_2)
self.assertEqual(204, resp.status)
@decorators.idempotent_id('6a388918-86eb-4e10-88e2-0032a7df38e9')
def test_create_execution_worker_lock_failed(self):
"""test_create_execution_worker_lock_failed
When creating an execution, the qinling-engine will check the load
and try to scaleup the function if needed. A lock is required when
doing this check.
In this test we acquire the lock manually, so that qinling will fail
to acquire the lock.
"""
function_id = self.create_function()
etcd3_client = etcd3gw.client(host=CONF.qinling.etcd_host,
port=CONF.qinling.etcd_port)
lock_id = "function_worker_%s_%s" % (function_id, 0)
with etcd3_client.lock(id=lock_id):
resp, body = self.client.create_execution(
function_id, input='{"name": "Qinling"}'
)
self.assertEqual(201, resp.status)
self.assertEqual('error', body['status'])
result = jsonutils.loads(body['result'])
self.assertEqual('Function execution failed.', result['error'])
@decorators.idempotent_id('2199d1e6-de7d-4345-8745-a8184d6022b1')
def test_get_all_admin(self):
"""Admin user can get executions of other projects"""
function_id = self.create_function()
resp, body = self.client.create_execution(
function_id, input='{"name": "Qinling"}'
)
self.assertEqual(201, resp.status)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
resp, body = self.admin_client.get_resources(
'executions?all_projects=true'
)
self.assertEqual(200, resp.status)
self.assertIn(
execution_id,
[execution['id'] for execution in body['executions']]
)
@decorators.idempotent_id('009fba47-957e-4de5-82e8-a032386d3ac0')
def test_get_all_not_allowed(self):
# Get other projects functions by normal user
context = self.assertRaises(
exceptions.Forbidden,
self.client.get_resources,
'executions?all_projects=true'
)
self.assertIn(
'Operation not allowed',
context.resp_body.get('faultstring')
)
@decorators.idempotent_id('794cdfb2-0a27-4e56-86e8-be18eee9400f')
def test_create_with_function_version(self):
function_id = self.create_function()
execution_id = self.create_execution(function_id)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, World', body)
version_1 = self.create_function_version(function_id)
execution_id = self.create_execution(function_id, version=version_1)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, World', body)
self.update_function_package(function_id,
"python/test_python_sleep.py")
version_2 = self.create_function_version(function_id)
execution_id = self.create_execution(function_id, version=version_2)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertNotIn('Hello, World', body)
@decorators.idempotent_id('dbf4bd84-bde3-4d1d-8dec-93aaf18b4b5f')
def test_create_with_function_alias(self):
function_id = self.create_function()
alias_name = self.create_function_alias(function_id)
execution_id = self.create_execution(alias_name=alias_name)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, World', body)
version_1 = self.create_function_version(function_id)
alias_name_1 = self.create_function_alias(function_id, version_1)
execution_id = self.create_execution(alias_name=alias_name_1)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, World', body)
self.update_function_package(function_id,
"python/test_python_sleep.py")
version_2 = self.create_function_version(function_id)
alias_name_2 = self.create_function_alias(function_id, version_2)
execution_id = self.create_execution(alias_name=alias_name_2)
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertNotIn('Hello, World', body)
@decorators.idempotent_id('8096cc52-64d2-4660-a657-9ac0bdd743ae')
def test_execution_async(self):
function_id = self.create_function()
resp, body = self.client.create_execution(function_id, sync=False)
self.assertEqual(201, resp.status)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
self.assertEqual('running', body['status'])
self.wait_execution_success(execution_id)
@decorators.idempotent_id('6cb47b1d-a8c6-48f2-a92f-c4f613c33d1c')
def test_execution_log(self):
function_id = self.create_function()
resp, body = self.client.create_execution(
function_id, input='{"name": "OpenStack"}'
)
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('success', body['status'])
execution_id = body['id']
# Get execution log
resp, body = self.client.get_execution_log(execution_id)
self.assertEqual(200, resp.status)
self.assertIn('Hello, OpenStack', body)
@decorators.idempotent_id('f22097dc-37db-484d-83d3-3a97e72ec576')
def test_execution_concurrency_no_scale(self):
package = self.create_package(name='python/test_python_sleep.py')
function_id = self.create_function(package_path=package)
def _create_execution():
resp, body = self.client.create_execution(function_id)
return resp, body
futs = []
with futurist.ThreadPoolExecutor(max_workers=10) as executor:
for _ in range(3):
fut = executor.submit(_create_execution)
futs.append(fut)
for f in futures.as_completed(futs):
# Wait until we get the response
resp, body = f.result()
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('success', body['status'])
resp, body = self.admin_client.get_function_workers(function_id)
self.assertEqual(200, resp.status)
self.assertEqual(1, len(body['workers']))
@decorators.idempotent_id('a5ed173a-19b7-4c92-ac78-c8862ad1d1d2')
def test_execution_concurrency_scale_up(self):
package = self.create_package(name='python/test_python_sleep.py')
function_id = self.create_function(package_path=package)
def _create_execution():
resp, body = self.client.create_execution(function_id)
return resp, body
futs = []
with futurist.ThreadPoolExecutor(max_workers=10) as executor:
for _ in range(6):
fut = executor.submit(_create_execution)
futs.append(fut)
for f in futures.as_completed(futs):
# Wait until we get the response
resp, body = f.result()
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('success', body['status'])
resp, body = self.admin_client.get_function_workers(function_id)
self.assertEqual(200, resp.status)
self.assertEqual(2, len(body['workers']))
@decorators.idempotent_id('ccfe67ce-e467-11e7-916c-00224d6b7bc1')
def test_python_execution_positional_args(self):
package = self.create_package(
name='python/test_python_positional_args.py'
)
function_id = self.create_function(package_path=package)
resp, body = self.client.create_execution(function_id,
input='Qinling')
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('success', body['status'])
result = jsonutils.loads(body['result'])
self.assertIn('Qinling', result['output'])
@decorators.idempotent_id('a948382a-84af-4f0e-ad08-4297345e302c')
def test_python_execution_file_limit(self):
package = self.create_package(name='python/test_python_file_limit.py')
function_id = self.create_function(package_path=package)
resp, body = self.client.create_execution(function_id)
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('failed', body['status'])
result = jsonutils.loads(body['result'])
self.assertNotIn('error', result)
self.assertIn(
'Too many open files', result['output']
)
@decorators.idempotent_id('bf6f8f35-fa88-469b-8878-7aa85a8ce5ab')
def test_python_execution_process_number(self):
package = self.create_package(
name='python/test_python_process_limit.py'
)
function_id = self.create_function(package_path=package)
resp, body = self.client.create_execution(function_id)
self.assertEqual(201, resp.status)
self.addCleanup(self.client.delete_resource, 'executions',
body['id'], ignore_notfound=True)
self.assertEqual('failed', body['status'])
result = jsonutils.loads(body['result'])
self.assertNotIn('error', result)
self.assertIn(
'too much resource consumption', result['output']
)
@decorators.idempotent_id('d0598868-e45d-11e7-9125-00224d6b7bc1')
def test_execution_image_function(self):
function_id = self.create_function(image=True)
resp, body = self.client.create_execution(function_id,
input='Qinling')
self.assertEqual(201, resp.status)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
self.assertEqual('success', body['status'])
self.assertIn('Qinling', jsonutils.loads(body['result'])['output'])
@decorators.idempotent_id('2b5f0787-b82d-4fc4-af76-cf86d389a76b')
def test_python_execution_memory_limit_non_image(self):
"""In this case, the following steps are taken:
1. Create a function that requires ~80M memory to run.
2. Create an execution using the function.
3. Verify that the execution is killed by the OOM-killer
because the function memory limit is only 32M(default).
4. Increase the function memory limit to 96M.
5. Create another execution.
6. Check the execution finished normally.
"""
# Create function
package = self.create_package(
name='python/test_python_memory_limit.py'
)
function_id = self.create_function(package_path=package)
# Invoke function
resp, body = self.client.create_execution(function_id)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
# Check the process is killed
self.assertEqual(201, resp.status)
result = json.loads(body['result'])
output = result.get('output')
self.assertEqual(INVOKE_ERROR, output)
# Increase the memory limit to 100663296(96M).
resp, body = self.client.update_function(
function_id, memory_size=100663296)
self.assertEqual(200, resp.status_code)
# Invoke the function again
resp, body = self.client.create_execution(function_id)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
# Check the process exited normally
self.assertEqual(201, resp.status)
result = json.loads(body['result'])
output = result.get('output')
# The function returns the length of a list containing 4 long strings.
self.assertEqual(4, output)
@decorators.idempotent_id('ed714f98-29fe-4e8d-b6ee-9730f92bddea')
def test_python_execution_cpu_limit_non_image(self):
"""In this case, the following steps are taken:
1. Create a function that takes some time to finish (calculating the
first 50000 digits of PI)
2. Create an execution using the function.
3. Store the duration of the first execution.
4. Increase the function cpu limit from 100(default) to 200 millicpu.
5. Create another execution.
6. Check whether the duration of the first execution is approximately
the double of the duration of the second one as its cpu resource is
half of the second run.
"""
# Create function
package = self.create_package(
name='python/test_python_cpu_limit.py'
)
function_id = self.create_function(package_path=package)
# Invoke function
resp, body = self.client.create_execution(function_id)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
# Record the duration, check whether the result is correct.
self.assertEqual(201, resp.status)
result = json.loads(body['result'])
output = result.get('output')
# Only the first 15 digits are returned.
self.assertEqual('314159265358979', output)
first_duration = result.get('duration', 0)
# Increase the cpu limit
resp, body = self.client.update_function(function_id, cpu=200)
self.assertEqual(200, resp.status_code)
# Invoke the function again
resp, body = self.client.create_execution(function_id)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
# Record the second duration, check whether the result is correct.
self.assertEqual(201, resp.status)
result = json.loads(body['result'])
output = result.get('output')
# Only the first 15 digits are returned.
self.assertEqual('314159265358979', output)
second_duration = result.get('duration', 0)
# Check whether the duration of the first execution is approximately
# the double (1.8x ~ 2.2x) of the duration of the second one.
# NOTE(huntxu): on my testbed, the result is quite near 2x. However
# it may vary in different environments, so we give a wider range
# here.
self.assertNotEqual(0, first_duration)
self.assertNotEqual(0, second_duration)
upper = second_duration * 2.2
lower = second_duration * 1.8
self.assertGreaterEqual(upper, first_duration)
self.assertLessEqual(lower, first_duration)
@decorators.idempotent_id('07edf2ff-7544-4f30-b006-fd5302a2a9cc')
def test_python_execution_public_connection(self):
"""Test connections from k8s pod to the outside.
Create a function that reads a webpage on the Internet, to
verify that pods in Kubernetes can connect to the outside.
"""
# Create function
package = self.create_package(name='python/test_python_http_get.py')
function_id = self.create_function(package_path=package)
url = 'https://docs.openstack.org/qinling/latest'
# Gets the page's sha256 outside Qinling
response = requests.get(url, timeout=10)
page_sha256 = hashlib.sha256(response.text.encode('utf-8')).hexdigest()
# Create an execution to get the page's sha256 with Qinling
resp, body = self.client.create_execution(
function_id, input='{"url": "%s"}' % url
)
execution_id = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
self.assertEqual(201, resp.status)
self.assertEqual('success', body['status'])
result = json.loads(body['result'])
self.assertEqual(page_sha256, result['output'])