From 50dd3d67b384c27d1a36d5dc0f8ba02b7086f1ac Mon Sep 17 00:00:00 2001 From: eanylin Date: Thu, 1 Jun 2017 18:55:26 -0500 Subject: [PATCH 1/8] Initial Commit --- helm_shipyard/__init__.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 helm_shipyard/__init__.py diff --git a/helm_shipyard/__init__.py b/helm_shipyard/__init__.py new file mode 100644 index 00000000..f10bbbf6 --- /dev/null +++ b/helm_shipyard/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. From 3268c6a13dfc288c777e7454ee1fd17cb28a6530 Mon Sep 17 00:00:00 2001 From: eanylin Date: Thu, 1 Jun 2017 21:05:05 -0500 Subject: [PATCH 2/8] WIP - Initial API implementation with Falcon --- helm_shipyard/{ => control}/__init__.py | 0 helm_shipyard/control/api.py | 29 ++++++++++ helm_shipyard/control/base.py | 66 ++++++++++++++++++++++ helm_shipyard/control/middleware.py | 73 +++++++++++++++++++++++++ helm_shipyard/control/tasks.py | 20 +++++++ 5 files changed, 188 insertions(+) rename helm_shipyard/{ => control}/__init__.py (100%) create mode 100644 helm_shipyard/control/api.py create mode 100644 helm_shipyard/control/base.py create mode 100644 helm_shipyard/control/middleware.py create mode 100644 helm_shipyard/control/tasks.py diff --git a/helm_shipyard/__init__.py b/helm_shipyard/control/__init__.py similarity index 100% rename from helm_shipyard/__init__.py rename to helm_shipyard/control/__init__.py diff --git a/helm_shipyard/control/api.py b/helm_shipyard/control/api.py new file mode 100644 index 00000000..af905bb3 --- /dev/null +++ b/helm_shipyard/control/api.py @@ -0,0 +1,29 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .tasks import TasksResource +from .base import ShipyardRequest +from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware + +def start_api(state_manager): + control_api = falcon.API(request_type=ShipyardRequest, + middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) + + # API for managing region data + control_api.add_route('/region/{region_id}', TasksResource) + control_api.add_route('/region/{region_id}/server/{name}', TasksResource) + control_api.add_route('/region/{region_id}/service/{kind}', TasksResource) + + return control_api diff --git a/helm_shipyard/control/base.py b/helm_shipyard/control/base.py new file mode 100644 index 00000000..e380a995 --- /dev/null +++ b/helm_shipyard/control/base.py @@ -0,0 +1,66 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon.request as request +import uuid + +class BaseResource(object): + + def on_options(self, req, resp): + self_attrs = dir(self) + methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] + allowed_methods = [] + + for m in methods: + if 'on_' + m.lower() in self_attrs: + allowed_methods.append(m) + + resp.headers['Allow'] = ','.join(allowed_methods) + resp.status = falcon.HTTP_200 + + # By default, no one is authorized to use a resource + def authorize_roles(self, role_list): + return False + + +class ShipyardRequestContext(object): + + def __init__(self): + self.log_level = 'error' + self.user = None + self.roles = ['anyone'] + self.request_id = str(uuid.uuid4()) + self.external_marker = None + + def set_log_level(self, level): + if level in ['error', 'info', 'debug']: + self.log_level = level + + def set_user(self, user): + self.user = user + + def add_role(self, role): + self.roles.append(role) + + def add_roles(self, roles): + self.roles.extend(roles) + + def remove_role(self, role): + self.roles = [x for x in self.roles + if x != role] + + def set_external_marker(self, marker): + self.external_marker = str(marker)[:32] + +class ShipyardRequest(request.Request) + context_type = ShipyardRequestContext diff --git a/helm_shipyard/control/middleware.py b/helm_shipyard/control/middleware.py new file mode 100644 index 00000000..3a35370c --- /dev/null +++ b/helm_shipyard/control/middleware.py @@ -0,0 +1,73 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import falcon + +class AuthMiddleware(object): + + # Authentication + def process_request(self, req, resp): + ctx = req.context + token = req.get_header('X-Auth-Token') + + user = self.validate_token(token) + + if user is not None: + ctx.set_user(user) + user_roles = self.role_list(user) + ctx.add_roles(user_roles) + else: + ctx.add_role('anyone') + + # Authorization + def process_resource(self, req, resp, resource): + ctx = req.context + + if not resource.authorize_roles(ctx.roles): + raise falcon.HTTPUnauthorized('Authentication required', + ('This resource requires an authorized role.')) + + # Return the username associated with an authenticated token or None + def validate_token(self, token): + if token == '10': + return 'shipyard' + elif token == 'admin': + return 'admin' + else: + return None + + # Return the list of roles assigned to the username + # Roles need to be an enum + def role_list(self, username): + if username == 'shipyard': + return ['user'] + elif username == 'admin': + return ['user', 'admin'] + +class ContextMiddleware(object): + + def process_request(self, req, resp): + ctx = req.context + + requested_logging = req.get_header('X-Log-Level') + + if requested_logging == 'DEBUG' and 'admin' in ctx.roles: + ctx.set_log_level('debug') + elif requested_logging == 'INFO': + ctx.set_log_level('info') + +class LoggingMiddleware(object): + + def process_response(self, req, resp, resource, req_succeeded): + ctx = req.context diff --git a/helm_shipyard/control/tasks.py b/helm_shipyard/control/tasks.py new file mode 100644 index 00000000..7831d270 --- /dev/null +++ b/helm_shipyard/control/tasks.py @@ -0,0 +1,20 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .base import BaseResource + +class TasksResource(BaseResource): + +class TaskResource(BaseResource): From 09e5399a058e8ef82a5a43630865f7a1f84023d6 Mon Sep 17 00:00:00 2001 From: Scott Hussey Date: Tue, 6 Jun 2017 14:48:11 -0500 Subject: [PATCH 3/8] Initial Shipyard API skeleton --- setup.py | 30 ++++++++++++ .../tasks.py => shipyard_airflow/__init__.py | 9 +--- .../control/__init__.py | 0 .../control/api.py | 34 ++++++++++--- .../control/base.py | 12 ++++- .../control/middleware.py | 19 +++++++- shipyard_airflow/control/regions.py | 30 ++++++++++++ shipyard_airflow/setup.py | 48 +++++++++++++++++++ shipyard_airflow/shipyard.py | 39 +++++++++++++++ 9 files changed, 202 insertions(+), 19 deletions(-) create mode 100644 setup.py rename helm_shipyard/control/tasks.py => shipyard_airflow/__init__.py (79%) rename {helm_shipyard => shipyard_airflow}/control/__init__.py (100%) rename {helm_shipyard => shipyard_airflow}/control/api.py (52%) rename {helm_shipyard => shipyard_airflow}/control/base.py (88%) rename {helm_shipyard => shipyard_airflow}/control/middleware.py (79%) create mode 100644 shipyard_airflow/control/regions.py create mode 100644 shipyard_airflow/setup.py create mode 100644 shipyard_airflow/shipyard.py diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..6712db3d --- /dev/null +++ b/setup.py @@ -0,0 +1,30 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='shipyard_airflow', + version='0.1a1', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', + license='Apache 2.0', + packages=['shipyard_airflow', + 'shipyard_airflow.control'], + install_requires=[ + 'falcon', + 'uwsgi>1.4' + ] + ) diff --git a/helm_shipyard/control/tasks.py b/shipyard_airflow/__init__.py similarity index 79% rename from helm_shipyard/control/tasks.py rename to shipyard_airflow/__init__.py index 7831d270..2a385a45 100644 --- a/helm_shipyard/control/tasks.py +++ b/shipyard_airflow/__init__.py @@ -10,11 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -import falcon - -from .base import BaseResource - -class TasksResource(BaseResource): - -class TaskResource(BaseResource): +# limitations under the License. \ No newline at end of file diff --git a/helm_shipyard/control/__init__.py b/shipyard_airflow/control/__init__.py similarity index 100% rename from helm_shipyard/control/__init__.py rename to shipyard_airflow/control/__init__.py diff --git a/helm_shipyard/control/api.py b/shipyard_airflow/control/api.py similarity index 52% rename from helm_shipyard/control/api.py rename to shipyard_airflow/control/api.py index af905bb3..e57f6427 100644 --- a/helm_shipyard/control/api.py +++ b/shipyard_airflow/control/api.py @@ -12,18 +12,38 @@ # See the License for the specific language governing permissions and # limitations under the License. import falcon +import json + +from .regions import RegionsResource, RegionResource +from .base import ShipyardRequest, BaseResource -from .tasks import TasksResource -from .base import ShipyardRequest from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware -def start_api(state_manager): +def start_api(): control_api = falcon.API(request_type=ShipyardRequest, middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) - # API for managing region data - control_api.add_route('/region/{region_id}', TasksResource) - control_api.add_route('/region/{region_id}/server/{name}', TasksResource) - control_api.add_route('/region/{region_id}/service/{kind}', TasksResource) + control_api.add_route('/versions', VersionsResource()) + + # v1.0 of Drydock API + v1_0_routes = [ + # API for managing region data + ('/regions', RegionsResource()), + ('/regions/{region_id}', RegionResource()), + ] + + for path, res in v1_0_routes: + control_api.add_route('/api/v1.0' + path, res) return control_api + +class VersionsResource(BaseResource): + + authorized_roles = ['anyone'] + + def on_get(self, req, resp): + resp.body = json.dumps({'v1.0': { + 'path': '/api/v1.0', + 'status': 'stable' + }}) + resp.status = falcon.HTTP_200 \ No newline at end of file diff --git a/helm_shipyard/control/base.py b/shipyard_airflow/control/base.py similarity index 88% rename from helm_shipyard/control/base.py rename to shipyard_airflow/control/base.py index e380a995..bb7e051f 100644 --- a/helm_shipyard/control/base.py +++ b/shipyard_airflow/control/base.py @@ -16,6 +16,8 @@ import uuid class BaseResource(object): + authorized_roles = [] + def on_options(self, req, resp): self_attrs = dir(self) methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] @@ -30,7 +32,13 @@ class BaseResource(object): # By default, no one is authorized to use a resource def authorize_roles(self, role_list): - return False + authorized = set(self.authorized_roles) + applied = set(role_list) + + if authorized.isdisjoint(applied): + return False + else: + return True class ShipyardRequestContext(object): @@ -62,5 +70,5 @@ class ShipyardRequestContext(object): def set_external_marker(self, marker): self.external_marker = str(marker)[:32] -class ShipyardRequest(request.Request) +class ShipyardRequest(request.Request): context_type = ShipyardRequestContext diff --git a/helm_shipyard/control/middleware.py b/shipyard_airflow/control/middleware.py similarity index 79% rename from helm_shipyard/control/middleware.py rename to shipyard_airflow/control/middleware.py index 3a35370c..019053e2 100644 --- a/helm_shipyard/control/middleware.py +++ b/shipyard_airflow/control/middleware.py @@ -11,8 +11,8 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. - import falcon +import logging class AuthMiddleware(object): @@ -31,7 +31,7 @@ class AuthMiddleware(object): ctx.add_role('anyone') # Authorization - def process_resource(self, req, resp, resource): + def process_resource(self, req, resp, resource, params): ctx = req.context if not resource.authorize_roles(ctx.roles): @@ -67,7 +67,22 @@ class ContextMiddleware(object): elif requested_logging == 'INFO': ctx.set_log_level('info') + ext_marker = req.get_header('X-Context-Marker') + ctx.set_external_marker(ext_marker if ext_marker is not None else '') + class LoggingMiddleware(object): + def __init__(self): + self.logger = logging.getLogger('shipyard.control') + def process_response(self, req, resp, resource, req_succeeded): ctx = req.context + + extra = { + 'user': ctx.user, + 'req_id': ctx.request_id, + 'external_ctx': ctx.external_marker, + } + + resp.append_header('X-Shipyard-Req', ctx.request_id) + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) \ No newline at end of file diff --git a/shipyard_airflow/control/regions.py b/shipyard_airflow/control/regions.py new file mode 100644 index 00000000..32808528 --- /dev/null +++ b/shipyard_airflow/control/regions.py @@ -0,0 +1,30 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .base import BaseResource + +class RegionsResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp): + resp.status = falcon.HTTP_200 + +class RegionResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, region_id): + resp.status = falcon.HTTP_200 \ No newline at end of file diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py new file mode 100644 index 00000000..d6fdf76c --- /dev/null +++ b/shipyard_airflow/setup.py @@ -0,0 +1,48 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='drydock_provisioner', + version='0.1a1', + description='Bootstrapper for Kubernetes infrastructure', + url='http://github.com/att-comdev/drydock', + author='Scott Hussey - AT&T', + author_email='sh8121@att.com', + license='Apache 2.0', + packages=['drydock_provisioner', + 'drydock_provisioner.objects', + 'drydock_provisioner.ingester', + 'drydock_provisioner.ingester.plugins', + 'drydock_provisioner.statemgmt', + 'drydock_provisioner.orchestrator', + 'drydock_provisioner.control', + 'drydock_provisioner.drivers', + 'drydock_provisioner.drivers.oob', + 'drydock_provisioner.drivers.oob.pyghmi_driver', + 'drydock_provisioner.drivers.node', + 'drydock_provisioner.drivers.node.maasdriver', + 'drydock_provisioner.drivers.node.maasdriver.models', + 'drydock_provisioner.control'], + install_requires=[ + 'PyYAML', + 'pyghmi>=1.0.18', + 'netaddr', + 'falcon', + 'oslo.versionedobjects>=1.23.0', + 'requests', + 'oauthlib', + 'uwsgi>1.4', + ] + ) diff --git a/shipyard_airflow/shipyard.py b/shipyard_airflow/shipyard.py new file mode 100644 index 00000000..5f95458b --- /dev/null +++ b/shipyard_airflow/shipyard.py @@ -0,0 +1,39 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import control.api as api + +def start_shipyard(): + + # Setup root logger + logger = logging.getLogger('shipyard') + + logger.setLevel('DEBUG') + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Specalized format for API logging + logger = logging.getLogger('shipyard.control') + logger.propagate = False + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + return api.start_api() + +shipyard = start_shipyard() \ No newline at end of file From c7fee8fd1941be98f135cb02c484aa6ea5c7e660 Mon Sep 17 00:00:00 2001 From: eanylin Date: Wed, 7 Jun 2017 11:58:55 -0500 Subject: [PATCH 4/8] Add README and Minor Changes --- shipyard_airflow/README.md | 12 ++++++++++++ shipyard_airflow/control/api.py | 4 ++-- shipyard_airflow/setup.py | 34 ++++++++------------------------- 3 files changed, 22 insertions(+), 28 deletions(-) create mode 100644 shipyard_airflow/README.md diff --git a/shipyard_airflow/README.md b/shipyard_airflow/README.md new file mode 100644 index 00000000..16bbfb3f --- /dev/null +++ b/shipyard_airflow/README.md @@ -0,0 +1,12 @@ +## Shipyard Airflow ## + +A python REST workflow orchestrator + +To run: + +``` +$ virtualenv -p python2.7 /var/tmp/shipyard +$ . /var/tmp/shipyard/bin/activate +$ python setup.py install +$ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L +``` diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py index e57f6427..f1bc69bd 100644 --- a/shipyard_airflow/control/api.py +++ b/shipyard_airflow/control/api.py @@ -25,7 +25,7 @@ def start_api(): control_api.add_route('/versions', VersionsResource()) - # v1.0 of Drydock API + # v1.0 of Shipyard API v1_0_routes = [ # API for managing region data ('/regions', RegionsResource()), @@ -46,4 +46,4 @@ class VersionsResource(BaseResource): 'path': '/api/v1.0', 'status': 'stable' }}) - resp.status = falcon.HTTP_200 \ No newline at end of file + resp.status = falcon.HTTP_200 diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py index d6fdf76c..6712db3d 100644 --- a/shipyard_airflow/setup.py +++ b/shipyard_airflow/setup.py @@ -14,35 +14,17 @@ from setuptools import setup -setup(name='drydock_provisioner', +setup(name='shipyard_airflow', version='0.1a1', - description='Bootstrapper for Kubernetes infrastructure', - url='http://github.com/att-comdev/drydock', - author='Scott Hussey - AT&T', - author_email='sh8121@att.com', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', license='Apache 2.0', - packages=['drydock_provisioner', - 'drydock_provisioner.objects', - 'drydock_provisioner.ingester', - 'drydock_provisioner.ingester.plugins', - 'drydock_provisioner.statemgmt', - 'drydock_provisioner.orchestrator', - 'drydock_provisioner.control', - 'drydock_provisioner.drivers', - 'drydock_provisioner.drivers.oob', - 'drydock_provisioner.drivers.oob.pyghmi_driver', - 'drydock_provisioner.drivers.node', - 'drydock_provisioner.drivers.node.maasdriver', - 'drydock_provisioner.drivers.node.maasdriver.models', - 'drydock_provisioner.control'], + packages=['shipyard_airflow', + 'shipyard_airflow.control'], install_requires=[ - 'PyYAML', - 'pyghmi>=1.0.18', - 'netaddr', 'falcon', - 'oslo.versionedobjects>=1.23.0', - 'requests', - 'oauthlib', - 'uwsgi>1.4', + 'uwsgi>1.4' ] ) From aa0d30db235f82cf3e2045490129e2f28be63a0b Mon Sep 17 00:00:00 2001 From: eanylin Date: Wed, 7 Jun 2017 21:11:10 -0500 Subject: [PATCH 5/8] Add tasks.py --- shipyard_airflow/control/api.py | 4 +++- shipyard_airflow/control/tasks.py | 26 ++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 shipyard_airflow/control/tasks.py diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py index f1bc69bd..ea4d8a66 100644 --- a/shipyard_airflow/control/api.py +++ b/shipyard_airflow/control/api.py @@ -16,6 +16,7 @@ import json from .regions import RegionsResource, RegionResource from .base import ShipyardRequest, BaseResource +from .tasks import TaskResource from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware @@ -30,10 +31,11 @@ def start_api(): # API for managing region data ('/regions', RegionsResource()), ('/regions/{region_id}', RegionResource()), + ('/dags/{dag_id}/tasks/{task_id}', TaskResource()), ] for path, res in v1_0_routes: - control_api.add_route('/api/v1.0' + path, res) + control_api.add_route('/api/experimental' + path, res) return control_api diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py new file mode 100644 index 00000000..e07e49f3 --- /dev/null +++ b/shipyard_airflow/control/tasks.py @@ -0,0 +1,26 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import requests + +from .base import BaseResource + +class TaskResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, dag_id, task_id): + resp.status = falcon.HTTP_200 + req_url = 'http://localhost:32080/api/experimental/dags/' + dag_id + '/tasks/' + task_id + task_details = requests.get(req_url).json() From 8e8fdf1450d0322cb0f691ecaa207f68810c5cc5 Mon Sep 17 00:00:00 2001 From: eanylin Date: Thu, 8 Jun 2017 20:29:53 -0500 Subject: [PATCH 6/8] Add dag_runs for dag execution --- setup.py | 1 + shipyard_airflow/control/api.py | 2 ++ shipyard_airflow/control/dag_runs.py | 39 ++++++++++++++++++++++++++++ shipyard_airflow/control/tasks.py | 4 ++- shipyard_airflow/setup.py | 1 + 5 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 shipyard_airflow/control/dag_runs.py diff --git a/setup.py b/setup.py index 6712db3d..f5cf46a3 100644 --- a/setup.py +++ b/setup.py @@ -25,6 +25,7 @@ setup(name='shipyard_airflow', 'shipyard_airflow.control'], install_requires=[ 'falcon', + 'requests', 'uwsgi>1.4' ] ) diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py index ea4d8a66..8913b86c 100644 --- a/shipyard_airflow/control/api.py +++ b/shipyard_airflow/control/api.py @@ -17,6 +17,7 @@ import json from .regions import RegionsResource, RegionResource from .base import ShipyardRequest, BaseResource from .tasks import TaskResource +from .dag_runs import DagRunResource from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware @@ -32,6 +33,7 @@ def start_api(): ('/regions', RegionsResource()), ('/regions/{region_id}', RegionResource()), ('/dags/{dag_id}/tasks/{task_id}', TaskResource()), + ('/dags/{dag_id}/dag_runs', DagRunResource()), ] for path, res in v1_0_routes: diff --git a/shipyard_airflow/control/dag_runs.py b/shipyard_airflow/control/dag_runs.py new file mode 100644 index 00000000..e34c13ca --- /dev/null +++ b/shipyard_airflow/control/dag_runs.py @@ -0,0 +1,39 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import requests + +from .base import BaseResource + +class DagRunResource(BaseResource): + + authorized_roles = ['user'] + + def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None): + req_url = 'http://localhost:32080/api/experimental/dags/{}/dag_runs'.format(dag_id) + + response = requests.post(req_url, + json={ + "run_id": run_id, + "conf": conf, + "execution_date": execution_date, + }) + if not response.ok: + raise IOError() + else: + resp.status = falcon.HTTP_200 + + data = response.json() + + return data['message'] diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py index e07e49f3..27a02347 100644 --- a/shipyard_airflow/control/tasks.py +++ b/shipyard_airflow/control/tasks.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import falcon +import json import requests from .base import BaseResource @@ -22,5 +23,6 @@ class TaskResource(BaseResource): def on_get(self, req, resp, dag_id, task_id): resp.status = falcon.HTTP_200 - req_url = 'http://localhost:32080/api/experimental/dags/' + dag_id + '/tasks/' + task_id + req_url = 'http://localhost:32080/api/experimental/dags/{}/tasks/{}'.format(dag_id, task_id) task_details = requests.get(req_url).json() + resp.body = json.dumps(task_details) diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py index 6712db3d..f5cf46a3 100644 --- a/shipyard_airflow/setup.py +++ b/shipyard_airflow/setup.py @@ -25,6 +25,7 @@ setup(name='shipyard_airflow', 'shipyard_airflow.control'], install_requires=[ 'falcon', + 'requests', 'uwsgi>1.4' ] ) From 0eef85185c9d7246ce3e7d0440ba7814a3216aa7 Mon Sep 17 00:00:00 2001 From: eanylin Date: Fri, 9 Jun 2017 21:42:48 -0500 Subject: [PATCH 7/8] Update Codes Based on Feedbacks --- setup.py | 2 ++ shipyard_airflow/control/api.py | 3 ++- shipyard_airflow/control/base.py | 30 ++++++++++++++++++++++++++ shipyard_airflow/control/dag_runs.py | 17 +++++++++------ shipyard_airflow/control/middleware.py | 3 ++- shipyard_airflow/control/regions.py | 3 ++- shipyard_airflow/control/shipyard.conf | 3 +++ shipyard_airflow/control/tasks.py | 16 +++++++++++--- shipyard_airflow/setup.py | 2 ++ shipyard_airflow/shipyard.py | 3 ++- 10 files changed, 68 insertions(+), 14 deletions(-) create mode 100644 shipyard_airflow/control/shipyard.conf diff --git a/setup.py b/setup.py index f5cf46a3..85e81e05 100644 --- a/setup.py +++ b/setup.py @@ -26,6 +26,8 @@ setup(name='shipyard_airflow', install_requires=[ 'falcon', 'requests', + 'configparser', 'uwsgi>1.4' ] ) + diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py index 8913b86c..4a004aa8 100644 --- a/shipyard_airflow/control/api.py +++ b/shipyard_airflow/control/api.py @@ -18,10 +18,10 @@ from .regions import RegionsResource, RegionResource from .base import ShipyardRequest, BaseResource from .tasks import TaskResource from .dag_runs import DagRunResource - from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware def start_api(): + control_api = falcon.API(request_type=ShipyardRequest, middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) @@ -51,3 +51,4 @@ class VersionsResource(BaseResource): 'status': 'stable' }}) resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/base.py b/shipyard_airflow/control/base.py index bb7e051f..94945f2b 100644 --- a/shipyard_airflow/control/base.py +++ b/shipyard_airflow/control/base.py @@ -13,6 +13,8 @@ # limitations under the License. import falcon.request as request import uuid +import json +import ConfigParser class BaseResource(object): @@ -40,6 +42,33 @@ class BaseResource(object): else: return True + # Error Handling + def return_error(self, resp, status_code, message="", retry=False): + """ + Write a error message body and throw a Falcon exception to trigger an HTTP status + + :param resp: Falcon response object to update + :param status_code: Falcon status_code constant + :param message: Optional error message to include in the body + :param retry: Optional flag whether client should retry the operation. Can ignore if we rely solely on 4XX vs 5xx status codes + """ + resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) + resp.content_type = 'application/json' + resp.status = status_code + + # Get Config Data + def retrieve_config(self, resp, section="", variable=""): + config = ConfigParser.ConfigParser() + + # The current assumption is that shipyard.conf will be placed in a fixed path + # within the shipyard container - Path TBD + config.read('/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf') + + # Retrieve data from shipyard.conf + query_data = config.get(section, variable) + + return query_data + class ShipyardRequestContext(object): @@ -72,3 +101,4 @@ class ShipyardRequestContext(object): class ShipyardRequest(request.Request): context_type = ShipyardRequestContext + diff --git a/shipyard_airflow/control/dag_runs.py b/shipyard_airflow/control/dag_runs.py index e34c13ca..4fcaca84 100644 --- a/shipyard_airflow/control/dag_runs.py +++ b/shipyard_airflow/control/dag_runs.py @@ -13,6 +13,7 @@ # limitations under the License. import falcon import requests +import json from .base import BaseResource @@ -21,19 +22,21 @@ class DagRunResource(BaseResource): authorized_roles = ['user'] def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None): - req_url = 'http://localhost:32080/api/experimental/dags/{}/dag_runs'.format(dag_id) + # Retrieve URL + web_server_url = self.retrieve_config(resp, 'BASE', 'WEB_SERVER') + req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id) + response = requests.post(req_url, json={ "run_id": run_id, "conf": conf, "execution_date": execution_date, }) - if not response.ok: - raise IOError() - else: + + if response.ok: resp.status = falcon.HTTP_200 + else: + self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag') + return - data = response.json() - - return data['message'] diff --git a/shipyard_airflow/control/middleware.py b/shipyard_airflow/control/middleware.py index 019053e2..c4db2c76 100644 --- a/shipyard_airflow/control/middleware.py +++ b/shipyard_airflow/control/middleware.py @@ -85,4 +85,5 @@ class LoggingMiddleware(object): } resp.append_header('X-Shipyard-Req', ctx.request_id) - self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) \ No newline at end of file + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) + diff --git a/shipyard_airflow/control/regions.py b/shipyard_airflow/control/regions.py index 32808528..ac6d970a 100644 --- a/shipyard_airflow/control/regions.py +++ b/shipyard_airflow/control/regions.py @@ -27,4 +27,5 @@ class RegionResource(BaseResource): authorized_roles = ['user'] def on_get(self, req, resp, region_id): - resp.status = falcon.HTTP_200 \ No newline at end of file + resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/shipyard.conf b/shipyard_airflow/control/shipyard.conf new file mode 100644 index 00000000..6dadc48c --- /dev/null +++ b/shipyard_airflow/control/shipyard.conf @@ -0,0 +1,3 @@ +[BASE] +WEB_SERVER=http://localhost:32080 + diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py index 27a02347..4584c54c 100644 --- a/shipyard_airflow/control/tasks.py +++ b/shipyard_airflow/control/tasks.py @@ -22,7 +22,17 @@ class TaskResource(BaseResource): authorized_roles = ['user'] def on_get(self, req, resp, dag_id, task_id): - resp.status = falcon.HTTP_200 - req_url = 'http://localhost:32080/api/experimental/dags/{}/tasks/{}'.format(dag_id, task_id) + # Retrieve URL + web_server_url = self.retrieve_config(resp, 'BASE', 'WEB_SERVER') + + req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id) task_details = requests.get(req_url).json() - resp.body = json.dumps(task_details) + + if 'error' in task_details: + resp.status = falcon.HTTP_400 + resp.body = json.dumps(task_details) + return + else: + resp.status = falcon.HTTP_200 + resp.body = json.dumps(task_details) + diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py index f5cf46a3..85e81e05 100644 --- a/shipyard_airflow/setup.py +++ b/shipyard_airflow/setup.py @@ -26,6 +26,8 @@ setup(name='shipyard_airflow', install_requires=[ 'falcon', 'requests', + 'configparser', 'uwsgi>1.4' ] ) + diff --git a/shipyard_airflow/shipyard.py b/shipyard_airflow/shipyard.py index 5f95458b..887028b3 100644 --- a/shipyard_airflow/shipyard.py +++ b/shipyard_airflow/shipyard.py @@ -36,4 +36,5 @@ def start_shipyard(): return api.start_api() -shipyard = start_shipyard() \ No newline at end of file +shipyard = start_shipyard() + From 92a190ce4b4557fea090e7ad58bcef1a3e25570e Mon Sep 17 00:00:00 2001 From: eanylin Date: Sun, 11 Jun 2017 21:53:08 -0500 Subject: [PATCH 8/8] Update configparser to align with Python3 Will stick with Python3 as that is what is used in DryDock --- README.md | 1 - shipyard_airflow/README.md | 2 +- shipyard_airflow/control/base.py | 23 ++++++++++++------- shipyard_airflow/control/dag_runs.py | 33 ++++++++++++++++------------ shipyard_airflow/control/tasks.py | 21 +++++++++++------- 5 files changed, 48 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index 6fa6f852..e7ecf1cb 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,3 @@ # shipyard Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management -## Testing 1 2 3 diff --git a/shipyard_airflow/README.md b/shipyard_airflow/README.md index 16bbfb3f..88496910 100644 --- a/shipyard_airflow/README.md +++ b/shipyard_airflow/README.md @@ -5,7 +5,7 @@ A python REST workflow orchestrator To run: ``` -$ virtualenv -p python2.7 /var/tmp/shipyard +$ virtualenv -p python3 /var/tmp/shipyard $ . /var/tmp/shipyard/bin/activate $ python setup.py install $ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L diff --git a/shipyard_airflow/control/base.py b/shipyard_airflow/control/base.py index 94945f2b..05912178 100644 --- a/shipyard_airflow/control/base.py +++ b/shipyard_airflow/control/base.py @@ -11,10 +11,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import falcon.request as request +import falcon, falcon.request as request import uuid import json -import ConfigParser +import configparser +import os class BaseResource(object): @@ -57,17 +58,23 @@ class BaseResource(object): resp.status = status_code # Get Config Data - def retrieve_config(self, resp, section="", variable=""): - config = ConfigParser.ConfigParser() + def retrieve_config(self, section="", data=""): # The current assumption is that shipyard.conf will be placed in a fixed path # within the shipyard container - Path TBD - config.read('/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf') + path = '/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf' + + # Check that shipyard.conf exists + if os.path.isfile(path): + config = configparser.ConfigParser() + config.read(path) - # Retrieve data from shipyard.conf - query_data = config.get(section, variable) + # Retrieve data from shipyard.conf + query_data = config.get(section, data) - return query_data + return query_data + else: + return 'Error - Missing Configuration File' class ShipyardRequestContext(object): diff --git a/shipyard_airflow/control/dag_runs.py b/shipyard_airflow/control/dag_runs.py index 4fcaca84..7a462d3d 100644 --- a/shipyard_airflow/control/dag_runs.py +++ b/shipyard_airflow/control/dag_runs.py @@ -23,20 +23,25 @@ class DagRunResource(BaseResource): def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None): # Retrieve URL - web_server_url = self.retrieve_config(resp, 'BASE', 'WEB_SERVER') + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') - req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id) - - response = requests.post(req_url, - json={ - "run_id": run_id, - "conf": conf, - "execution_date": execution_date, - }) - - if response.ok: - resp.status = falcon.HTTP_200 - else: - self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag') + if 'Error' in web_server_url: + resp.status = falcon.HTTP_400 + resp.body = json.dumps({'Error': 'Missing Configuration File'}) return + else: + req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id) + + response = requests.post(req_url, + json={ + "run_id": run_id, + "conf": conf, + "execution_date": execution_date, + }) + + if response.ok: + resp.status = falcon.HTTP_200 + else: + self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag') + return diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py index 4584c54c..9cb2f6e8 100644 --- a/shipyard_airflow/control/tasks.py +++ b/shipyard_airflow/control/tasks.py @@ -23,16 +23,21 @@ class TaskResource(BaseResource): def on_get(self, req, resp, dag_id, task_id): # Retrieve URL - web_server_url = self.retrieve_config(resp, 'BASE', 'WEB_SERVER') + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') - req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id) - task_details = requests.get(req_url).json() - - if 'error' in task_details: + if 'Error' in web_server_url: resp.status = falcon.HTTP_400 - resp.body = json.dumps(task_details) + resp.body = json.dumps({'Error': 'Missing Configuration File'}) return else: - resp.status = falcon.HTTP_200 - resp.body = json.dumps(task_details) + req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id) + task_details = requests.get(req_url).json() + + if 'error' in task_details: + resp.status = falcon.HTTP_400 + resp.body = json.dumps(task_details) + return + else: + resp.status = falcon.HTTP_200 + resp.body = json.dumps(task_details)