Browse Source

Support webhook

- Allow user create webhook based on the function.
- Webhook can be invoked without authentication.
- Can not delete function associated with webhook.

Another big change is, we are going to use minikube instead of
kubernetes-aio scripts from openstack-helm project.

Implements: blueprint qinling-function-webhook
Change-Id: I85e0b0f999f0d820bfacca9ac3b9af04e80df0d7
changes/63/532363/10
Lingxian Kong 4 years ago
parent
commit
3e35a4b7d5
59 changed files with 753 additions and 1979 deletions
  1. +2
    -2
      devstack/plugin.sh
  2. +4
    -2
      qinling/api/controllers/v1/execution.py
  3. +5
    -2
      qinling/api/controllers/v1/function.py
  4. +20
    -1
      qinling/api/controllers/v1/resources.py
  5. +2
    -0
      qinling/api/controllers/v1/root.py
  6. +175
    -0
      qinling/api/controllers/v1/webhook.py
  7. +9
    -6
      qinling/config.py
  8. +5
    -2
      qinling/context.py
  9. +26
    -1
      qinling/db/api.py
  10. +51
    -1
      qinling/db/sqlalchemy/api.py
  11. +12
    -0
      qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py
  12. +12
    -1
      qinling/db/sqlalchemy/models.py
  13. +4
    -2
      qinling/engine/default_engine.py
  14. +4
    -2
      qinling/engine/service.py
  15. +22
    -13
      qinling/engine/utils.py
  16. +2
    -2
      qinling/orchestrator/base.py
  17. +4
    -7
      qinling/orchestrator/kubernetes/manager.py
  18. +1
    -0
      qinling/orchestrator/kubernetes/templates/deployment.j2
  19. +1
    -0
      qinling/orchestrator/kubernetes/templates/pod.j2
  20. +13
    -0
      qinling/tests/unit/api/controllers/v1/test_function.py
  21. +69
    -0
      qinling/tests/unit/api/controllers/v1/test_webhook.py
  22. +19
    -3
      qinling/tests/unit/base.py
  23. +1
    -1
      qinling/tests/unit/services/test_periodics.py
  24. +18
    -0
      qinling/utils/common.py
  25. +2
    -0
      qinling/utils/constants.py
  26. +22
    -0
      qinling/utils/openstack/keystone.py
  27. +10
    -2
      qinling_tempest_plugin/services/base.py
  28. +7
    -2
      qinling_tempest_plugin/services/qinling_client.py
  29. +16
    -12
      qinling_tempest_plugin/tests/api/test_executions.py
  30. +101
    -0
      qinling_tempest_plugin/tests/api/test_webhooks.py
  31. +9
    -0
      qinling_tempest_plugin/tests/base.py
  32. +48
    -48
      runtimes/python2/server.py
  33. +7
    -0
      tools/clear_resources.sh
  34. +0
    -97
      tools/gate/dump_logs.sh
  35. +7
    -81
      tools/gate/funcs/common.sh
  36. +0
    -148
      tools/gate/funcs/kube.sh
  37. +2
    -40
      tools/gate/funcs/network.sh
  38. +0
    -25
      tools/gate/kubeadm_aio.sh
  39. +5
    -17
      tools/gate/setup_gate.sh
  40. +35
    -0
      tools/gate/setup_minikube.sh
  41. +1
    -25
      tools/gate/vars.sh
  42. +0
    -88
      tools/kubeadm-aio/Dockerfile
  43. +0
    -110
      tools/kubeadm-aio/README.rst
  44. +0
    -2
      tools/kubeadm-aio/assets/etc/kube-cni
  45. +0
    -3
      tools/kubeadm-aio/assets/etc/kube-role
  46. +0
    -3
      tools/kubeadm-aio/assets/etc/kube-version
  47. +0
    -1
      tools/kubeadm-aio/assets/etc/kubeadm-join-command-args
  48. +0
    -4
      tools/kubeadm-aio/assets/etc/kubeadm.conf
  49. +0
    -3
      tools/kubeadm-aio/assets/etc/kubeapi-device
  50. +0
    -3
      tools/kubeadm-aio/assets/etc/kubelet-container
  51. +0
    -54
      tools/kubeadm-aio/assets/kubeadm-aio
  52. +0
    -365
      tools/kubeadm-aio/assets/opt/cni-manifests/calico.yaml
  53. +0
    -329
      tools/kubeadm-aio/assets/opt/cni-manifests/canal.yaml
  54. +0
    -94
      tools/kubeadm-aio/assets/opt/cni-manifests/flannel.yaml
  55. +0
    -187
      tools/kubeadm-aio/assets/opt/cni-manifests/weave.yaml
  56. +0
    -73
      tools/kubeadm-aio/assets/opt/nfs-provisioner/deployment.yaml
  57. +0
    -5
      tools/kubeadm-aio/assets/opt/nfs-provisioner/storageclass.yaml
  58. +0
    -15
      tools/kubeadm-aio/assets/opt/rbac/dev.yaml
  59. +0
    -95
      tools/kubeadm-aio/kubeadm-aio-launcher.sh

+ 2
- 2
devstack/plugin.sh View File

@ -24,7 +24,7 @@ function install_k8s {
source tools/gate/setup_gate.sh
popd
# Pre-pull the default docker image for python runtime and image function
# Pre-fetch the default docker image for python runtime and image function
# test.
sudo docker pull $QINLING_PYTHON_RUNTIME_IMAGE
sudo docker pull openstackqinling/alpine-test
@ -76,11 +76,11 @@ function configure_qinling {
iniset $QINLING_CONF_FILE DEFAULT server all
iniset $QINLING_CONF_FILE DEFAULT logging_context_format_string "%(asctime)s %(process)d %(color)s %(levelname)s [%(request_id)s] %(message)s %(resource)s (%(name)s)"
iniset $QINLING_CONF_FILE storage file_system_dir $QINLING_FUNCTION_STORAGE_DIR
iniset $QINLING_CONF_FILE kubernetes qinling_service_address $DEFAULT_HOST_IP
# Setup keystone_authtoken section
configure_auth_token_middleware $QINLING_CONF_FILE qinling $QINLING_AUTH_CACHE_DIR
iniset $QINLING_CONF_FILE keystone_authtoken www_authenticate_uri $KEYSTONE_AUTH_URI_V3
iniset $QINLING_CONF_FILE keystone_authtoken region_name "$REGION_NAME"
# Setup RabbitMQ credentials
iniset_rpc_backend qinling $QINLING_CONF_FILE


+ 4
- 2
qinling/api/controllers/v1/execution.py View File

@ -65,9 +65,9 @@ class ExecutionsController(rest.RestController):
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(resources.Executions, wtypes.text, bool, wtypes.text,
wtypes.text)
wtypes.text, wtypes.text)
def get_all(self, function_id=None, all_projects=False, project_id=None,
status=None):
status=None, description=None):
"""Return a list of executions.
:param function_id: Optional. Filtering executions by function_id.
@ -75,6 +75,7 @@ class ExecutionsController(rest.RestController):
resources, the param is ignored for normal user.
:param all_projects: Optional. Get resources of all projects.
:param status: Optional. Filter by execution status.
:param description: Optional. Filter by description.
"""
ctx = context.get_ctx()
if project_id and not ctx.is_admin:
@ -89,6 +90,7 @@ class ExecutionsController(rest.RestController):
function_id=function_id,
project_id=project_id,
status=status,
description=description
)
LOG.info("Get all %ss. filters=%s", self.type, filters)


+ 5
- 2
qinling/api/controllers/v1/function.py View File

@ -131,14 +131,13 @@ class FunctionsController(rest.RestController):
@rest_utils.wrap_pecan_controller_exception
@pecan.expose('json')
def post(self, **kwargs):
LOG.info("Creating function, params: %s", kwargs)
# When using image to create function, runtime_id is not a required
# param.
if not POST_REQUIRED.issubset(set(kwargs.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED
)
LOG.info("Creating function, params: %s", kwargs)
values = {
'name': kwargs.get('name'),
@ -241,6 +240,10 @@ class FunctionsController(rest.RestController):
raise exc.NotAllowedException(
'The function is still associated with running job(s).'
)
if func_db.webhook:
raise exc.NotAllowedException(
'The function is still associated with webhook.'
)
# Even admin user can not delete other project's function because
# the trust associated can only be removed by function owner.


+ 20
- 1
qinling/api/controllers/v1/resources.py View File

@ -290,7 +290,7 @@ class Execution(Resource):
obj = cls()
for key, val in d.items():
if key == 'input' and val:
if key == 'input' and val is not None:
if val.get('__function_input'):
setattr(obj, key, val.get('__function_input'))
else:
@ -393,3 +393,22 @@ class Jobs(ResourceList):
class ScaleInfo(Resource):
count = wtypes.IntegerType(minimum=1)
class Webhook(Resource):
id = types.uuid
function_id = types.uuid
description = wtypes.text
project_id = wsme.wsattr(wtypes.text, readonly=True)
created_at = wsme.wsattr(wtypes.text, readonly=True)
updated_at = wsme.wsattr(wtypes.text, readonly=True)
webhook_url = wsme.wsattr(wtypes.text, readonly=True)
class Webhooks(ResourceList):
webhooks = [Webhook]
def __init__(self, **kwargs):
self._type = 'webhooks'
super(Webhooks, self).__init__(**kwargs)

+ 2
- 0
qinling/api/controllers/v1/root.py View File

@ -20,6 +20,7 @@ from qinling.api.controllers.v1 import function
from qinling.api.controllers.v1 import job
from qinling.api.controllers.v1 import resources
from qinling.api.controllers.v1 import runtime
from qinling.api.controllers.v1 import webhook
class RootResource(resources.Resource):
@ -36,6 +37,7 @@ class Controller(object):
runtimes = runtime.RuntimesController()
executions = execution.ExecutionsController()
jobs = job.JobsController()
webhooks = webhook.WebhooksController()
@wsme_pecan.wsexpose(RootResource)
def index(self):


+ 175
- 0
qinling/api/controllers/v1/webhook.py View File

@ -0,0 +1,175 @@
# Copyright 2018 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import copy
import json
from oslo_log import log as logging
import pecan
from pecan import rest
import wsmeext.pecan as wsme_pecan
from qinling.api import access_control as acl
from qinling.api.controllers.v1 import resources
from qinling.api.controllers.v1 import types
from qinling import context
from qinling.db import api as db_api
from qinling import exceptions as exc
from qinling import rpc
from qinling.utils import constants
from qinling.utils import executions
from qinling.utils.openstack import keystone as keystone_utils
from qinling.utils import rest_utils
LOG = logging.getLogger(__name__)
POST_REQUIRED = set(['function_id'])
UPDATE_ALLOWED = set(['function_id', 'description'])
class WebhooksController(rest.RestController):
_custom_actions = {
'invoke': ['POST'],
}
def __init__(self, *args, **kwargs):
self.type = 'webhook'
self.engine_client = rpc.get_engine_client()
self.qinling_endpoint = keystone_utils.get_qinling_endpoint()
super(WebhooksController, self).__init__(*args, **kwargs)
def _add_webhook_url(self, id, webhook):
"""Add webhook_url attribute for webhook.
We generate the url dynamically in case the service url is changing.
"""
res = copy.deepcopy(webhook)
url = '/'.join(
[self.qinling_endpoint.strip('/'), constants.CURRENT_VERSION,
'webhooks', id, 'invoke']
)
res.update({'webhook_url': url})
return res
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(resources.Webhook, types.uuid)
def get(self, id):
LOG.info("Get %s %s.", self.type, id)
webhook = db_api.get_webhook(id).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(resources.Webhooks)
def get_all(self):
LOG.info("Get all %ss.", self.type)
webhooks = []
for i in db_api.get_webhooks():
webhooks.append(
resources.Webhook.from_dict(
self._add_webhook_url(i.id, i.to_dict())
)
)
return resources.Webhooks(webhooks=webhooks)
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(
resources.Webhook,
body=resources.Webhook,
status_code=201
)
def post(self, webhook):
acl.enforce('webhook:create', context.get_ctx())
params = webhook.to_dict()
if not POST_REQUIRED.issubset(set(params.keys())):
raise exc.InputException(
'Required param is missing. Required: %s' % POST_REQUIRED
)
LOG.info("Creating %s, params: %s", self.type, params)
# Even admin user can not expose normal user's function
db_api.get_function(params['function_id'], insecure=False)
webhook_d = db_api.create_webhook(params).to_dict()
return resources.Webhook.from_dict(
self._add_webhook_url(webhook_d['id'], webhook_d)
)
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(None, types.uuid, status_code=204)
def delete(self, id):
acl.enforce('webhook:delete', context.get_ctx())
LOG.info("Delete %s %s.", self.type, id)
db_api.delete_webhook(id)
@rest_utils.wrap_wsme_controller_exception
@wsme_pecan.wsexpose(
resources.Webhook,
types.uuid,
body=resources.Webhook
)
def put(self, id, webhook):
"""Update webhook.
Currently, we only support update function_id.
"""
acl.enforce('webhook:update', context.get_ctx())
values = {}
for key in UPDATE_ALLOWED:
if webhook.to_dict().get(key) is not None:
values.update({key: webhook.to_dict()[key]})
LOG.info('Update %s %s, params: %s', self.type, id, values)
if 'function_id' in values:
# Even admin user can not expose normal user's function
db_api.get_function(values['function_id'], insecure=False)
webhook = db_api.update_webhook(id, values).to_dict()
return resources.Webhook.from_dict(self._add_webhook_url(id, webhook))
@rest_utils.wrap_pecan_controller_exception
@pecan.expose('json')
def invoke(self, id, **kwargs):
with db_api.transaction():
# The webhook url can be accessed without authentication, so
# insecure is used here
webhook_db = db_api.get_webhook(id, insecure=True)
function_db = webhook_db.function
trust_id = function_db.trust_id
project_id = function_db.project_id
LOG.info(
'Invoking function %s by webhook %s',
webhook_db.function_id, id
)
# Setup user context
ctx = keystone_utils.create_trust_context(trust_id, project_id)
context.set_ctx(ctx)
params = {
'function_id': webhook_db.function_id,
'sync': False,
'input': json.dumps(kwargs),
'description': constants.EXECUTION_BY_WEBHOOK % id
}
execution = executions.create_execution(self.engine_client, params)
pecan.response.status = 202
return {'execution_id': execution.id}

+ 9
- 6
qinling/config.py View File

@ -26,6 +26,13 @@ launch_opt = cfg.ListOpt(
help='Specifies which qinling server to start by the launch script.'
)
default_opts = [
cfg.StrOpt(
'qinling_endpoint',
help='Qinling service endpoint.'
),
]
API_GROUP = 'api'
api_opts = [
cfg.StrOpt('host', default='0.0.0.0', help='Qinling API server host.'),
@ -139,11 +146,6 @@ kubernetes_opts = [
help='Kubernetes server address, e.g. you can start a proxy to the '
'Kubernetes API server by using "kubectl proxy" command.'
),
cfg.IPOpt(
'qinling_service_address',
default='127.0.0.1',
help='Qinling API service ip address.'
),
cfg.StrOpt(
'log_devel',
default='INFO',
@ -172,7 +174,8 @@ def list_opts():
(STORAGE_GROUP, storage_opts),
(KUBERNETES_GROUP, kubernetes_opts),
(ETCD_GROUP, etcd_opts),
(None, [launch_opt])
(None, [launch_opt]),
(None, default_opts),
]
return keystone_middleware_opts + keystone_loading_opts + qinling_opts


+ 5
- 2
qinling/context.py View File

@ -11,6 +11,7 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re
from oslo_config import cfg
from oslo_context import context as oslo_context
@ -22,6 +23,7 @@ from qinling.utils import thread_local
CONF = cfg.CONF
ALLOWED_WITHOUT_AUTH = ['/', '/v1/']
WEBHOOK_REG = '^/v1/webhooks/[a-f0-9-]+/invoke$'
CTX_THREAD_LOCAL_NAME = "QINLING_APP_CTX_THREAD_LOCAL"
DEFAULT_PROJECT_ID = "default"
@ -46,10 +48,11 @@ def authenticate(req):
class AuthHook(hooks.PecanHook):
def before(self, state):
if not CONF.pecan.auth_enable:
return
if state.request.path in ALLOWED_WITHOUT_AUTH:
return
if not CONF.pecan.auth_enable:
if re.search(WEBHOOK_REG, state.request.path):
return
try:


+ 26
- 1
qinling/db/api.py View File

@ -57,6 +57,7 @@ def transaction():
def delete_all():
"""A helper function for testing."""
delete_jobs(insecure=True)
delete_webhooks(insecure=True)
delete_executions(insecure=True)
delete_functions(insecure=True)
delete_runtimes(insecure=True)
@ -69,7 +70,7 @@ def conditional_update(model, values, expected_values, **kwargs):
def get_function(id, insecure=None):
"""Get function from db.
'insecure' param is needed for job handler.
'insecure' param is needed for job handler and webhook.
"""
return IMPL.get_function(id, insecure=insecure)
@ -176,3 +177,27 @@ def get_jobs():
def delete_jobs(**kwargs):
return IMPL.delete_jobs(**kwargs)
def create_webhook(values):
return IMPL.create_webhook(values)
def get_webhook(id, insecure=None):
return IMPL.get_webhook(id, insecure=insecure)
def get_webhooks():
return IMPL.get_webhooks()
def delete_webhook(id):
return IMPL.delete_webhook(id)
def update_webhook(id, values):
return IMPL.update_webhook(id, values)
def delete_webhooks(**kwargs):
return IMPL.delete_webhooks(**kwargs)

+ 51
- 1
qinling/db/sqlalchemy/api.py View File

@ -155,7 +155,6 @@ def _get_collection(model, insecure=False, limit=None, marker=None,
query = (db_base.model_query(model, columns) if insecure
else _secure_query(model, *columns))
query = db_filters.apply_filters(query, model, **filters)
query = _paginate_query(
model,
limit,
@ -441,3 +440,54 @@ def get_jobs(session=None, **kwargs):
@db_base.session_aware()
def delete_jobs(session=None, insecure=None, **kwargs):
return _delete_all(models.Job, insecure=insecure, **kwargs)
@db_base.session_aware()
def create_webhook(values, session=None):
webhook = models.Webhook()
webhook.update(values.copy())
try:
webhook.save(session=session)
except oslo_db_exc.DBDuplicateEntry as e:
raise exc.DBError(
"Duplicate entry for webhook: %s" % e.columns
)
return webhook
@db_base.insecure_aware()
@db_base.session_aware()
def get_webhook(id, insecure=None, session=None):
webhook = _get_db_object_by_id(models.Webhook, id, insecure=insecure)
if not webhook:
raise exc.DBEntityNotFoundError("Webhook not found [id=%s]" % id)
return webhook
@db_base.session_aware()
def get_webhooks(session=None, **kwargs):
return _get_collection_sorted_by_time(models.Webhook, **kwargs)
@db_base.session_aware()
def delete_webhook(id, session=None):
webhook = get_webhook(id)
session.delete(webhook)
@db_base.session_aware()
def update_webhook(id, values, session=None):
webhook = get_webhook(id)
webhook.update(values.copy())
return webhook
@db_base.insecure_aware()
@db_base.session_aware()
def delete_webhooks(session=None, insecure=None, **kwargs):
return _delete_all(models.Webhook, insecure=insecure, **kwargs)

+ 12
- 0
qinling/db/sqlalchemy/migration/alembic_migrations/versions/001_pike.py View File

@ -114,3 +114,15 @@ def upgrade():
sa.ForeignKeyConstraint(['function_id'], [u'functions.id']),
info={"check_ifexists": True}
)
op.create_table(
'webhooks',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('project_id', sa.String(length=80), nullable=False),
sa.Column('id', sa.String(length=36), nullable=False),
sa.Column('description', sa.String(length=255), nullable=True),
sa.Column('function_id', sa.String(length=36), nullable=False),
sa.PrimaryKeyConstraint('id'),
info={"check_ifexists": True}
)

+ 12
- 1
qinling/db/sqlalchemy/models.py View File

@ -85,9 +85,19 @@ class Job(model_base.QinlingSecureModelBase):
return d
class Webhook(model_base.QinlingSecureModelBase):
__tablename__ = 'webhooks'
function_id = sa.Column(
sa.String(36),
sa.ForeignKey(Function.id)
)
description = sa.Column(sa.String(255))
Runtime.functions = relationship("Function", back_populates="runtime")
# Only get jobs
# Only get running jobs
Function.jobs = relationship(
"Job",
back_populates="function",
@ -96,3 +106,4 @@ Function.jobs = relationship(
"~Job.status.in_(['done', 'cancelled']))"
)
)
Function.webhook = relationship("Webhook", uselist=False, backref="function")

+ 4
- 2
qinling/engine/default_engine.py View File

@ -28,8 +28,9 @@ CONF = cfg.CONF
class DefaultEngine(object):
def __init__(self, orchestrator):
def __init__(self, orchestrator, qinling_endpoint):
self.orchestrator = orchestrator
self.qinling_endpoint = qinling_endpoint
self.session = requests.Session()
def create_runtime(self, ctx, runtime_id):
@ -142,7 +143,8 @@ class DefaultEngine(object):
data = utils.get_request_data(
CONF, function_id, execution_id,
input, function.entry, function.trust_id
input, function.entry, function.trust_id,
self.qinling_endpoint
)
success, res = utils.url_request(
self.session, func_url, body=data


+ 4
- 2
qinling/engine/service.py View File

@ -23,6 +23,7 @@ from qinling.engine import default_engine as engine
from qinling.orchestrator import base as orchestra_base
from qinling import rpc
from qinling.services import periodics
from qinling.utils.openstack import keystone as keystone_utils
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
@ -34,14 +35,15 @@ class EngineService(cotyledon.Service):
self.server = None
def run(self):
orchestrator = orchestra_base.load_orchestrator(CONF)
qinling_endpoint = keystone_utils.get_qinling_endpoint()
orchestrator = orchestra_base.load_orchestrator(CONF, qinling_endpoint)
db_api.setup_db()
topic = CONF.engine.topic
server = CONF.engine.host
transport = messaging.get_rpc_transport(CONF)
target = messaging.Target(topic=topic, server=server, fanout=False)
endpoint = engine.DefaultEngine(orchestrator)
endpoint = engine.DefaultEngine(orchestrator, qinling_endpoint)
access_policy = dispatcher.DefaultRPCAccessPolicy
self.server = messaging.get_rpc_server(
transport,


+ 22
- 13
qinling/engine/utils.py View File

@ -19,6 +19,7 @@ import six
import tenacity
from qinling import context
from qinling.utils import constants
LOG = logging.getLogger(__name__)
@ -33,10 +34,11 @@ def url_request(request_session, url, body=None):
temp[-1] = 'ping'
ping_url = '/'.join(temp)
r = tenacity.Retrying(
wait=tenacity.wait_fixed(0.5),
stop=tenacity.stop_after_attempt(5),
retry=tenacity.retry_if_exception_type(IOError))
r.call(request_session.get, ping_url, timeout=(3, 3))
wait=tenacity.wait_fixed(1),
stop=tenacity.stop_after_attempt(30),
retry=tenacity.retry_if_exception_type(IOError)
)
r.call(request_session.get, ping_url, timeout=(3, 3), verify=False)
except Exception as e:
LOG.exception(
"Failed to request url %s, error: %s", ping_url, str(e)
@ -44,16 +46,24 @@ def url_request(request_session, url, body=None):
return False, {'error': 'Function execution failed.'}
for a in six.moves.xrange(10):
res = None
try:
# Default execution max duration is 3min, could be configurable
r = request_session.post(url, json=body, timeout=(3, 180))
return True, r.json()
res = request_session.post(
url, json=body, timeout=(3, 180), verify=False
)
return True, res.json()
except requests.ConnectionError as e:
exception = e
# NOTE(kong): Could be configurable
time.sleep(1)
except Exception as e:
LOG.exception("Failed to request url %s, error: %s", url, str(e))
LOG.exception(
"Failed to request url %s, error: %s", url, str(e)
)
if res:
LOG.error("Response status: %s, content: %s",
res.status_code, res.content)
return False, {'error': 'Function execution timeout.'}
LOG.exception("Could not connect to function service. Reason: %s",
@ -62,13 +72,12 @@ def url_request(request_session, url, body=None):
return False, {'error': 'Internal service error.'}
def get_request_data(conf, function_id, execution_id, input, entry, trust_id):
def get_request_data(conf, function_id, execution_id, input, entry, trust_id,
qinling_endpoint):
ctx = context.get_ctx()
download_url = (
'http://%s:%s/v1/functions/%s?download=true' %
(conf.kubernetes.qinling_service_address,
conf.api.port, function_id)
'%s/%s/functions/%s?download=true' %
(qinling_endpoint.strip('/'), constants.CURRENT_VERSION, function_id)
)
data = {
'execution_id': execution_id,


+ 2
- 2
qinling/orchestrator/base.py View File

@ -59,7 +59,7 @@ class OrchestratorBase(object):
raise NotImplementedError
def load_orchestrator(conf):
def load_orchestrator(conf, qinling_endpoint):
global ORCHESTRATOR
if not ORCHESTRATOR:
@ -67,7 +67,7 @@ def load_orchestrator(conf):
mgr = driver.DriverManager('qinling.orchestrator',
conf.engine.orchestrator,
invoke_on_load=True,
invoke_args=[conf])
invoke_args=[conf, qinling_endpoint])
ORCHESTRATOR = mgr.driver
except Exception as e:


+ 4
- 7
qinling/orchestrator/kubernetes/manager.py View File

@ -35,8 +35,9 @@ TEMPLATES_DIR = (os.path.dirname(os.path.realpath(__file__)) + '/templates/')
class KubernetesManager(base.OrchestratorBase):
def __init__(self, conf):
def __init__(self, conf, qinling_endpoint):
self.conf = conf
self.qinling_endpoint = qinling_endpoint
clients = k8s_util.get_k8s_clients(self.conf)
self.v1 = clients['v1']
@ -125,7 +126,6 @@ class KubernetesManager(base.OrchestratorBase):
def delete_pool(self, name, labels=None):
"""Delete all resources belong to the deployment."""
LOG.info("Deleting deployment %s", name)
selector = common.convert_dict_to_string(labels)
@ -134,7 +134,6 @@ class KubernetesManager(base.OrchestratorBase):
self.conf.kubernetes.namespace,
label_selector=selector
)
LOG.info("ReplicaSets in deployment %s deleted.", name)
ret = self.v1.list_namespaced_service(
@ -146,7 +145,6 @@ class KubernetesManager(base.OrchestratorBase):
svc_name,
self.conf.kubernetes.namespace,
)
LOG.info("Services in deployment %s deleted.", name)
self.v1extention.delete_collection_namespaced_deployment(
@ -154,14 +152,12 @@ class KubernetesManager(base.OrchestratorBase):
label_selector=selector,
field_selector='metadata.name=%s' % name
)
# Should delete pods after deleting deployment to avoid pods are
# recreated by k8s.
self.v1.delete_collection_namespaced_pod(
self.conf.kubernetes.namespace,
label_selector=selector
)
LOG.info("Pods in deployment %s deleted.", name)
LOG.info("Deployment %s deleted.", name)
@ -398,7 +394,8 @@ class KubernetesManager(base.OrchestratorBase):
if service_url:
func_url = '%s/execute' % service_url
data = utils.get_request_data(
self.conf, function_id, execution_id, input, entry, trust_id
self.conf, function_id, execution_id, input, entry, trust_id,
self.qinling_endpoint
)
LOG.debug(
'Invoke function %s, url: %s, data: %s',


+ 1
- 0
qinling/orchestrator/kubernetes/templates/deployment.j2 View File

@ -20,6 +20,7 @@ spec:
{{ key }}: {{ value }}
{% endfor %}
spec:
terminationGracePeriodSeconds: 0
containers:
- name: {{ container_name }}
image: {{ image }}


+ 1
- 0
qinling/orchestrator/kubernetes/templates/pod.j2 View File

@ -7,6 +7,7 @@ metadata:
{{ key }}: {{ value }}
{% endfor %}
spec:
terminationGracePeriodSeconds: 0
containers:
- name: {{ pod_name }}
image: {{ pod_image }}


+ 13
- 0
qinling/tests/unit/api/controllers/v1/test_function.py View File

@ -158,3 +158,16 @@ class TestFunctionController(base.APITest):
)
self.assertEqual(403, resp.status_int)
def test_delete_with_webhook(self):
db_func = self.create_function(
runtime_id=self.runtime_id, prefix=TEST_CASE_NAME
)
self.create_webhook(function_id=db_func.id, prefix=TEST_CASE_NAME)
resp = self.app.delete(
'/v1/functions/%s' % db_func.id,
expect_errors=True
)
self.assertEqual(403, resp.status_int)

+ 69
- 0
qinling/tests/unit/api/controllers/v1/test_webhook.py View File

@ -0,0 +1,69 @@
# Copyright 2018 Catalyst IT Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from qinling.tests.unit.api import base
TEST_CASE_NAME = 'TestWebhookController'
class TestWebhookController(base.APITest):
def setUp(self):
super(TestWebhookController, self).setUp()
db_func = self.create_function(prefix=TEST_CASE_NAME)
self.func_id = db_func.id
def test_crud(self):
# Create
body = {
'function_id': self.func_id,
'description': 'webhook test'
}
resp = self.app.post_json('/v1/webhooks', body)
self.assertEqual(201, resp.status_int)
webhook_id = resp.json.get('id')
self.assertIn(self.qinling_endpoint, resp.json.get('webhook_url'))
# Get
resp = self.app.get('/v1/webhooks/%s' % webhook_id)
self.assertEqual(200, resp.status_int)
self._assertDictContainsSubset(resp.json, body)
# List
resp = self.app.get('/v1/webhooks')
self.assertEqual(200, resp.status_int)
actual = self._assert_single_item(
resp.json['webhooks'], id=webhook_id
)
self._assertDictContainsSubset(actual, body)
# Update
resp = self.app.put_json(
'/v1/webhooks/%s' % webhook_id,
{'description': 'webhook test update'}
)
self.assertEqual(200, resp.status_int)
expected = {
'function_id': self.func_id,
'description': 'webhook test update'
}
resp = self.app.get('/v1/webhooks/%s' % webhook_id)
self.assertEqual(200, resp.status_int)
self._assertDictContainsSubset(resp.json, expected)
# Delete
resp = self.app.delete('/v1/webhooks/%s' % webhook_id)
self.assertEqual(204, resp.status_int)
resp = self.app.get('/v1/webhooks/%s' % webhook_id, expect_errors=True)
self.assertEqual(404, resp.status_int)

+ 19
- 3
qinling/tests/unit/base.py View File

@ -137,10 +137,13 @@ class DbTestCase(BaseTest):
(config.STORAGE_GROUP, config.storage_opts),
(config.KUBERNETES_GROUP, config.kubernetes_opts),
(config.ETCD_GROUP, config.etcd_opts),
(None, [config.launch_opt])
(None, [config.launch_opt]),
(None, config.default_opts)
]
for group, options in qinling_opts:
cfg.CONF.register_opts(list(options), group)
cls.qinling_endpoint = 'http://127.0.0.1:7070/'
cfg.CONF.set_default('qinling_endpoint', cls.qinling_endpoint)
db_api.setup_db()
@ -200,11 +203,24 @@ class DbTestCase(BaseTest):
job_params = {
'name': self.rand_name('job', prefix=prefix),
'function_id': function_id,
# 'auth_enable' is disabled by default, we create runtime for
# default tenant.
# 'auth_enable' is disabled by default
'project_id': DEFAULT_PROJECT_ID,
}
job_params.update(kwargs)
job = db_api.create_job(job_params)
return job
def create_webhook(self, function_id=None, prefix=None, **kwargs):
if not function_id:
function_id = self.create_function(prefix=prefix).id
webhook_params = {
'function_id': function_id,
# 'auth_enable' is disabled by default
'project_id': DEFAULT_PROJECT_ID,
}
webhook_params.update(kwargs)
webhook = db_api.create_webhook(webhook_params)
return webhook

+ 1
- 1
qinling/tests/unit/services/test_periodics.py View File

@ -50,7 +50,7 @@ class TestPeriodics(base.DbTestCase):
mock_k8s = mock.Mock()
mock_etcd_url.return_value = 'http://localhost:37718'
self.override_config('function_service_expiration', 1, 'engine')
engine = default_engine.DefaultEngine(mock_k8s)
engine = default_engine.DefaultEngine(mock_k8s, CONF.qinling_endpoint)
periodics.handle_function_service_expiration(self.ctx, engine)
self.assertEqual(1, mock_k8s.delete_function.call_count)


+ 18
- 0
qinling/utils/common.py View File

@ -12,6 +12,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import functools
import pdb
import sys
import warnings
@ -104,3 +105,20 @@ def disable_ssl_warnings(func):
return func(*args, **kwargs)
return wrapper
class ForkedPdb(pdb.Pdb):
"""A Pdb subclass that may be used from a forked multiprocessing child.
Usage:
from qinling.utils import common
common.ForkedPdb().set_trace()
"""
def interaction(self, *args, **kwargs):
_stdin = sys.stdin
try:
sys.stdin = file('/dev/stdin')
pdb.Pdb.interaction(self, *args, **kwargs)
finally:
sys.stdin = _stdin

+ 2
- 0
qinling/utils/constants.py View File

@ -11,8 +11,10 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
CURRENT_VERSION = 'v1'
EXECUTION_BY_JOB = 'Created by Job %s'
EXECUTION_BY_WEBHOOK = 'Created by Webhook %s'
PERIODIC_JOB_HANDLER = 'job_handler'
PERIODIC_FUNC_MAPPING_HANDLER = 'function_mapping_handler'


+ 22
- 0
qinling/utils/openstack/keystone.py View File

@ -138,3 +138,25 @@ def create_trust_context(trust_id, project_id):
auth_token=None,
is_admin=True
)
def get_qinling_endpoint():
'''Get Qinling service endpoint.'''
if CONF.qinling_endpoint:
return CONF.qinling_endpoint
region = CONF.keystone_authtoken.region_name
auth = v3.Password(
auth_url=CONF.keystone_authtoken.www_authenticate_uri,
username=CONF.keystone_authtoken.username,
password=CONF.keystone_authtoken.password,
project_name=CONF.keystone_authtoken.project_name,
user_domain_name=CONF.keystone_authtoken.user_domain_name,
project_domain_name=CONF.keystone_authtoken.project_domain_name,
)
sess = session.Session(auth=auth, verify=False)
endpoint = sess.get_endpoint(service_type='function-engine',
interface='public',
region_name=region)
return endpoint

+ 10
- 2
qinling_tempest_plugin/services/base.py View File

@ -14,16 +14,24 @@
import json
import six
from tempest.lib.common import rest_client
urlparse = six.moves.urllib.parse
class QinlingClientBase(rest_client.RestClient):
def __init__(self, auth_provider, **kwargs):
super(QinlingClientBase, self).__init__(auth_provider, **kwargs)
def get_list_objs(self, obj):
resp, body = self.get('/v1/%s' % obj)
def get_list_objs(self, obj, params=None):
url = '/v1/%s' % obj
query_string = ("?%s" % urlparse.urlencode(list(params.items()))
if params else "")
url += query_string
resp, body = self.get(url)
return resp, json.loads(body)
def delete_obj(self, obj, id):


+ 7
- 2
qinling_tempest_plugin/services/qinling_client.py View File

@ -38,8 +38,8 @@ class QinlingClient(client_base.QinlingClientBase):
return resp, body
def get_resources(self, res):
resp, body = self.get_list_objs(res)
def get_resources(self, res, params=None):
resp, body = self.get_list_objs(res, params=params)
return resp, body
@ -102,3 +102,8 @@ class QinlingClient(client_base.QinlingClientBase):
def get_function_workers(self, function_id):
return self.get_resources('functions/%s/workers' % function_id)
def create_webhook(self, function_id):
req_body = {"function_id": function_id}
resp, body = self.post_json('webhooks', req_body)
return resp, body

+ 16
- 12
qinling_tempest_plugin/tests/api/test_executions.py View File

@ -89,27 +89,31 @@ class ExecutionsTest(base.BaseQinlingTest):
resp, body = self.client.create_execution(self.function_id,
input='{"name": "Qinling"}')
self.assertEqual(201, resp.status)
execution_id = body['id']
execution_id_1 = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id, ignore_notfound=True)
execution_id_1, ignore_notfound=True)
self.assertEqual('success', body['status'])
# Create another execution without input
resp, body = self.client.create_execution(self.function_id)
self.assertEqual(201, resp.status)
execution_id_2 = body['id']
self.addCleanup(self.client.delete_resource, 'executions',
execution_id_2, ignore_notfound=True)
self.assertEqual('success', body['status'])
# Get executions
resp, body = self.client.get_resources('executions')
self.assertEqual(200, resp.status)
self.assertIn(
execution_id,
[execution['id'] for execution in body['executions']]
)
# Delete execution
resp = self.client.delete_resource('executions', execution_id)
expected = {execution_id_1, execution_id_2}
actual = set([execution['id'] for execution in body['executions']])
self.assertTrue(expected.issubset(actual))
# Delete executions
resp = self.client.delete_resource('executions', execution_id_1)
self.assertEqual(204, resp.status)
resp = self.client.delete_resource('executions', execution_id_2)
self.assertEqual(204, resp.status)
@decorators.idempotent_id('2199d1e6-de7d-4345-8745-a8184d6022b1')


+ 101
- 0
qinling_tempest_plugin/tests/api/test_webhooks.py View File

@ -0,0 +1,101 @@
# Copyright 2018 Catalyst IT Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import pkg_resources
import tempfile
import zipfile
import requests
from tempest.lib.common.utils import data_utils
from tempest.lib import decorators
from qinling_tempest_plugin.tests import base
class WebhooksTest(base.BaseQinlingTest):
name_prefix = 'WebhooksTest'
@classmethod
def resource_setup(cls):
super(WebhooksTest, cls).resource_setup()
cls.runtime_id = None
name = data_utils.rand_name('runtime', prefix=cls.name_prefix)
_, body = cls.admin_client.create_runtime(
'openstackqinling/python-runtime', name
)
cls.runtime_id = body['id']
@classmethod
def resource_cleanup(cls):
if cls.runtime_id:
cls.admin_client.delete_resource('runtimes', cls.runtime_id,
ignore_notfound=True)
super(WebhooksTest, cls).resource_cleanup()
def setUp(self):
super(WebhooksTest, self).setUp()
self.await_runtime_available(self.runtime_id)
self._create_function()
def _create_function(self, name='python_test.py'):
python_file_path = pkg_resources.resource_filename(
'qinling_tempest_plugin',
"functions/%s" % name
)
base_name, extention = os.path.splitext(python_file_path)
module_name = os.path.basename(base_name)
self.python_zip_file = os.path.join(
tempfile.gettempdir(),
'%s.zip' % module_name
)
if not os.path.isfile(self.python_zip_file):
zf = zipfile.ZipFile(self.python_zip_file, mode='w')
try:
# Use default compression mode, may change in future.
zf.write(
python_file_path,
'%s%s' % (module_name, extention),
compress_type=zipfile.ZIP_STORED
)
finally:
zf.close()
self.function_id = self.create_function(self.python_zip_file)
@decorators.idempotent_id('37DCD022-32D6-48D1-B90C-31D605DBE53B')
def test_webhook_invoke(self):
webhook_id, url = self.create_webhook()
resp = requests.post(url, data={'name': 'qinling'}, verify=False)
self.assertEqual(202, resp.status_code)
resp_exec_id = resp.json().get('execution_id')
self.addCleanup(self.client.delete_resource, 'executions',
resp_exec_id, ignore_notfound=True)
resp, body = self.client.get_resources(
'executions',
{'description': 'has:%s' % webhook_id}
)
self.assertEqual(200, resp.status)
self.assertEqual(1, len(body['executions']))
exec_id = body['executions'][0]['id']
self.assertEqual(resp_exec_id, exec_id)
self.await_execution_success(exec_id)
resp, body = self.client.get_execution_log(exec_id)
self.assertEqual(200, resp.status)
self.assertIn('qinling', body)

+ 9
- 0
qinling_tempest_plugin/tests/base.py View File

@ -98,3 +98,12 @@ class BaseQinlingTest(test.BaseTestCase):
function_id, ignore_notfound=True)
return function_id
def create_webhook(self):
resp, body = self.client.create_webhook(self.function_id)
self.assertEqual(201, resp.status)
webhook_id = body['id']
self.addCleanup(self.client.delete_resource, 'webhooks',
webhook_id, ignore_notfound=True)
return webhook_id, body['webhook_url']

+ 48
- 48
runtimes/python2/server.py View File

@ -21,7 +21,6 @@ import os
import sys
import time
import traceback
import zipfile
from flask import Flask
from flask import request
@ -34,6 +33,10 @@ app = Flask(__name__)
downloaded = False
downloading = False
DOWNLOAD_ERROR = "Failed to download function package from %s, error: %s"
INVOKE_ERROR = "Function execution failed because of too much resource " \
"consumption"
def setup_logger(loglevel):
global app
@ -53,6 +56,47 @@ def _print_trace():
print(''.join(line for line in lines))
def _get_responce(output, duration, logs, success, code):
return Response(
response=json.dumps(
{
'output': output,
'duration': duration,
'logs': logs,
'success': success
}
),
status=code,
mimetype='application/json'
)
def _download_package(url, zip_file, token=None):
app.logger.info('Downloading function, download_url:%s' % url)
headers = {}
if token:
headers = {'X-Auth-Token': token}
try:
r = requests.get(url, headers=headers, stream=True,
verify=False, timeout=5)
if r.status_code != 200:
return _get_responce(
DOWNLOAD_ERROR % (url, r.content), 0, '', False, 500
)
with open(zip_file, 'wb') as fd:
for chunk in r.iter_content(chunk_size=65535):
fd.write(chunk)
except Exception as e:
return _get_responce(
DOWNLOAD_ERROR % (url, str(e)), 0, '', False, 500
)
app.logger.info('Downloaded function package to %s' % zip_file)
def _invoke_function(execution_id, zip_file, module_name, method, arg, input,
return_dict):
"""Thie function is supposed to be running in a child process."""
@ -116,48 +160,16 @@ def execute():
)
while downloading:
# wait
time.sleep(3)
# download function package
if not downloading and not downloaded:
downloading = True
token = params.get('token')
headers = {}
if token:
headers = {'X-Auth-Token': token}
app.logger.info(
'Downloading function, download_url:%s, entry: %s' %
(download_url, entry)
)
# Get function code package from Qinling service.
r = requests.get(download_url, headers=headers, stream=True)
with open(zip_file, 'wb') as fd:
for chunk in r.iter_content(chunk_size=65535):
fd.write(chunk)
_download_package(download_url, zip_file, params.get('token'))
app.logger.info('Downloaded function package to %s' % zip_file)
downloading = False
downloaded = True
if downloaded:
if not zipfile.is_zipfile(zip_file):
return Response(
response=json.dumps(
{
'output': 'The function package is incorrect.',
'duration': 0,
'logs': '',
'success': False
}
),
status=500,
mimetype='application/json'
)
# Provide an openstack session to user's function
os_session = None
if auth_url:
@ -189,8 +201,7 @@ def execute():
# Process was killed unexpectedly or finished with error.
if p.exitcode != 0:
output = "Function execution failed because of too much resource " \
"consumption."
output = INVOKE_ERROR
success = False
else:
output = return_dict.get('result')
@ -201,18 +212,7 @@ def execute():
logs = f.read()
os.remove('%s.out' % execution_id)
return Response(
response=json.dumps(
{
'output': output,
'duration': duration,
'logs': logs,
'success': success
}
),
status=200,
mimetype='application/json'
)
return _get_responce(output, duration, logs, success, 200)
@app.route('/ping')


+ 7
- 0
tools/clear_resources.sh View File

@ -4,6 +4,13 @@ set -e
# export QINLING_URL=http://127.0.0.1:7070
function delete_resources(){
# Delete webhooks
ids=$(openstack webhook list -f yaml -c Id | awk '{print $3}')
for id in $ids
do
openstack webhook delete $id
done
# Delete jobs
ids=$(openstack job list -f yaml -c Id | awk '{print $3}')
for id in $ids


+ 0
- 97
tools/gate/dump_logs.sh View File

@ -1,97 +0,0 @@
#!/bin/bash
set +xe
# if we can't find kubectl, fail immediately because it is likely
# the whitespace linter fails - no point to collect logs.
if ! type "kubectl" &> /dev/null; then
exit $1
fi
echo "Capturing logs from environment."
mkdir -p ${LOGS_DIR}/k8s/etc
sudo cp -a /etc/kubernetes ${LOGS_DIR}/k8s/etc
sudo chmod 777 --recursive ${LOGS_DIR}/*
mkdir -p ${LOGS_DIR}/k8s
for OBJECT_TYPE in nodes \
namespace \
storageclass; do
kubectl get ${OBJECT_TYPE} -o yaml > ${LOGS_DIR}/k8s/${OBJECT_TYPE}.yaml
done
kubectl describe nodes > ${LOGS_DIR}/k8s/nodes.txt
for OBJECT_TYPE in svc \
pods \
jobs \
deployments \
daemonsets \
statefulsets \
configmaps \
secrets; do
kubectl get --all-namespaces ${OBJECT_TYPE} -o yaml > \
${LOGS_DIR}/k8s/${OBJECT_TYPE}.yaml
done
mkdir -p ${LOGS_DIR}/k8s/pods
kubectl get pods -a --all-namespaces -o json | jq -r \
'.items[].metadata | .namespace + " " + .name' | while read line; do
NAMESPACE=$(echo $line | awk '{print $1}')
NAME=$(echo $line | awk '{print $2}')
kubectl get --namespace $NAMESPACE pod $NAME -o json | jq -r \
'.spec.containers[].name' | while read line; do
CONTAINER=$(echo $line | awk '{print $1}')
kubectl logs $NAME --namespace $NAMESPACE -c $CONTAINER > \
${LOGS_DIR}/k8s/pods/$NAMESPACE-$NAME-$CONTAINER.txt
done
done
mkdir -p ${LOGS_DIR}/k8s/svc
kubectl get svc -o json --all-namespaces | jq -r \
'.items[].metadata | .namespace + " " + .name' | while read line; do
NAMESPACE=$(echo $line | awk '{print $1}')
NAME=$(echo $line | awk '{print $2}')
kubectl describe svc $NAME --namespace $NAMESPACE > \
${LOGS_DIR}/k8s/svc/$NAMESPACE-$NAME.txt
done
mkdir -p ${LOGS_DIR}/k8s/pvc
kubectl get pvc -o json --all-namespaces | jq -r \
'.items[].metadata | .namespace + " " + .name' | while read line; do
NAMESPACE=$(echo $line | awk '{print $1}')
NAME=$(echo $line | awk '{print $2}')
kubectl describe pvc $NAME --namespace $NAMESPACE > \
${LOGS_DIR}/k8s/pvc/$NAMESPACE-$NAME.txt
done
mkdir -p ${LOGS_DIR}/k8s/rbac
for OBJECT_TYPE in clusterroles \
roles \
clusterrolebindings \
rolebindings; do
kubectl get ${OBJECT_TYPE} -o yaml > ${LOGS_DIR}/k8s/rbac/${OBJECT_TYPE}.yaml
done
mkdir -p ${LOGS_DIR}/k8s/descriptions
for NAMESPACE in $(kubectl get namespaces -o name | awk -F '/' '{ print $NF }') ; do
for OBJECT in $(kubectl get all --show-all -n $NAMESPACE -o name) ; do