diff --git a/README.md b/README.md index 05896f56..e883fab1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,2 @@ # shipyard Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management - -## Testing 4 5 6 diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..85e81e05 --- /dev/null +++ b/setup.py @@ -0,0 +1,33 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='shipyard_airflow', + version='0.1a1', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', + license='Apache 2.0', + packages=['shipyard_airflow', + 'shipyard_airflow.control'], + install_requires=[ + 'falcon', + 'requests', + 'configparser', + 'uwsgi>1.4' + ] + ) + diff --git a/shipyard_airflow/README.md b/shipyard_airflow/README.md new file mode 100644 index 00000000..88496910 --- /dev/null +++ b/shipyard_airflow/README.md @@ -0,0 +1,12 @@ +## Shipyard Airflow ## + +A python REST workflow orchestrator + +To run: + +``` +$ virtualenv -p python3 /var/tmp/shipyard +$ . /var/tmp/shipyard/bin/activate +$ python setup.py install +$ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L +``` diff --git a/shipyard_airflow/__init__.py b/shipyard_airflow/__init__.py new file mode 100644 index 00000000..2a385a45 --- /dev/null +++ b/shipyard_airflow/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. \ No newline at end of file diff --git a/shipyard_airflow/control/__init__.py b/shipyard_airflow/control/__init__.py new file mode 100644 index 00000000..f10bbbf6 --- /dev/null +++ b/shipyard_airflow/control/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/shipyard_airflow/control/api.py b/shipyard_airflow/control/api.py new file mode 100644 index 00000000..4a004aa8 --- /dev/null +++ b/shipyard_airflow/control/api.py @@ -0,0 +1,54 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json + +from .regions import RegionsResource, RegionResource +from .base import ShipyardRequest, BaseResource +from .tasks import TaskResource +from .dag_runs import DagRunResource +from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware + +def start_api(): + + control_api = falcon.API(request_type=ShipyardRequest, + middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()]) + + control_api.add_route('/versions', VersionsResource()) + + # v1.0 of Shipyard API + v1_0_routes = [ + # API for managing region data + ('/regions', RegionsResource()), + ('/regions/{region_id}', RegionResource()), + ('/dags/{dag_id}/tasks/{task_id}', TaskResource()), + ('/dags/{dag_id}/dag_runs', DagRunResource()), + ] + + for path, res in v1_0_routes: + control_api.add_route('/api/experimental' + path, res) + + return control_api + +class VersionsResource(BaseResource): + + authorized_roles = ['anyone'] + + def on_get(self, req, resp): + resp.body = json.dumps({'v1.0': { + 'path': '/api/v1.0', + 'status': 'stable' + }}) + resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/base.py b/shipyard_airflow/control/base.py new file mode 100644 index 00000000..05912178 --- /dev/null +++ b/shipyard_airflow/control/base.py @@ -0,0 +1,111 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon, falcon.request as request +import uuid +import json +import configparser +import os + +class BaseResource(object): + + authorized_roles = [] + + def on_options(self, req, resp): + self_attrs = dir(self) + methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH'] + allowed_methods = [] + + for m in methods: + if 'on_' + m.lower() in self_attrs: + allowed_methods.append(m) + + resp.headers['Allow'] = ','.join(allowed_methods) + resp.status = falcon.HTTP_200 + + # By default, no one is authorized to use a resource + def authorize_roles(self, role_list): + authorized = set(self.authorized_roles) + applied = set(role_list) + + if authorized.isdisjoint(applied): + return False + else: + return True + + # Error Handling + def return_error(self, resp, status_code, message="", retry=False): + """ + Write a error message body and throw a Falcon exception to trigger an HTTP status + + :param resp: Falcon response object to update + :param status_code: Falcon status_code constant + :param message: Optional error message to include in the body + :param retry: Optional flag whether client should retry the operation. Can ignore if we rely solely on 4XX vs 5xx status codes + """ + resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry}) + resp.content_type = 'application/json' + resp.status = status_code + + # Get Config Data + def retrieve_config(self, section="", data=""): + + # The current assumption is that shipyard.conf will be placed in a fixed path + # within the shipyard container - Path TBD + path = '/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf' + + # Check that shipyard.conf exists + if os.path.isfile(path): + config = configparser.ConfigParser() + config.read(path) + + # Retrieve data from shipyard.conf + query_data = config.get(section, data) + + return query_data + else: + return 'Error - Missing Configuration File' + + +class ShipyardRequestContext(object): + + def __init__(self): + self.log_level = 'error' + self.user = None + self.roles = ['anyone'] + self.request_id = str(uuid.uuid4()) + self.external_marker = None + + def set_log_level(self, level): + if level in ['error', 'info', 'debug']: + self.log_level = level + + def set_user(self, user): + self.user = user + + def add_role(self, role): + self.roles.append(role) + + def add_roles(self, roles): + self.roles.extend(roles) + + def remove_role(self, role): + self.roles = [x for x in self.roles + if x != role] + + def set_external_marker(self, marker): + self.external_marker = str(marker)[:32] + +class ShipyardRequest(request.Request): + context_type = ShipyardRequestContext + diff --git a/shipyard_airflow/control/dag_runs.py b/shipyard_airflow/control/dag_runs.py new file mode 100644 index 00000000..7a462d3d --- /dev/null +++ b/shipyard_airflow/control/dag_runs.py @@ -0,0 +1,47 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import requests +import json + +from .base import BaseResource + +class DagRunResource(BaseResource): + + authorized_roles = ['user'] + + def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None): + # Retrieve URL + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') + + if 'Error' in web_server_url: + resp.status = falcon.HTTP_400 + resp.body = json.dumps({'Error': 'Missing Configuration File'}) + return + else: + req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id) + + response = requests.post(req_url, + json={ + "run_id": run_id, + "conf": conf, + "execution_date": execution_date, + }) + + if response.ok: + resp.status = falcon.HTTP_200 + else: + self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag') + return + diff --git a/shipyard_airflow/control/middleware.py b/shipyard_airflow/control/middleware.py new file mode 100644 index 00000000..c4db2c76 --- /dev/null +++ b/shipyard_airflow/control/middleware.py @@ -0,0 +1,89 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import logging + +class AuthMiddleware(object): + + # Authentication + def process_request(self, req, resp): + ctx = req.context + token = req.get_header('X-Auth-Token') + + user = self.validate_token(token) + + if user is not None: + ctx.set_user(user) + user_roles = self.role_list(user) + ctx.add_roles(user_roles) + else: + ctx.add_role('anyone') + + # Authorization + def process_resource(self, req, resp, resource, params): + ctx = req.context + + if not resource.authorize_roles(ctx.roles): + raise falcon.HTTPUnauthorized('Authentication required', + ('This resource requires an authorized role.')) + + # Return the username associated with an authenticated token or None + def validate_token(self, token): + if token == '10': + return 'shipyard' + elif token == 'admin': + return 'admin' + else: + return None + + # Return the list of roles assigned to the username + # Roles need to be an enum + def role_list(self, username): + if username == 'shipyard': + return ['user'] + elif username == 'admin': + return ['user', 'admin'] + +class ContextMiddleware(object): + + def process_request(self, req, resp): + ctx = req.context + + requested_logging = req.get_header('X-Log-Level') + + if requested_logging == 'DEBUG' and 'admin' in ctx.roles: + ctx.set_log_level('debug') + elif requested_logging == 'INFO': + ctx.set_log_level('info') + + ext_marker = req.get_header('X-Context-Marker') + ctx.set_external_marker(ext_marker if ext_marker is not None else '') + +class LoggingMiddleware(object): + + def __init__(self): + self.logger = logging.getLogger('shipyard.control') + + def process_response(self, req, resp, resource, req_succeeded): + ctx = req.context + + extra = { + 'user': ctx.user, + 'req_id': ctx.request_id, + 'external_ctx': ctx.external_marker, + } + + resp.append_header('X-Shipyard-Req', ctx.request_id) + self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra) + diff --git a/shipyard_airflow/control/regions.py b/shipyard_airflow/control/regions.py new file mode 100644 index 00000000..ac6d970a --- /dev/null +++ b/shipyard_airflow/control/regions.py @@ -0,0 +1,31 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon + +from .base import BaseResource + +class RegionsResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp): + resp.status = falcon.HTTP_200 + +class RegionResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, region_id): + resp.status = falcon.HTTP_200 + diff --git a/shipyard_airflow/control/shipyard.conf b/shipyard_airflow/control/shipyard.conf new file mode 100644 index 00000000..6dadc48c --- /dev/null +++ b/shipyard_airflow/control/shipyard.conf @@ -0,0 +1,3 @@ +[BASE] +WEB_SERVER=http://localhost:32080 + diff --git a/shipyard_airflow/control/tasks.py b/shipyard_airflow/control/tasks.py new file mode 100644 index 00000000..9cb2f6e8 --- /dev/null +++ b/shipyard_airflow/control/tasks.py @@ -0,0 +1,43 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import falcon +import json +import requests + +from .base import BaseResource + +class TaskResource(BaseResource): + + authorized_roles = ['user'] + + def on_get(self, req, resp, dag_id, task_id): + # Retrieve URL + web_server_url = self.retrieve_config('BASE', 'WEB_SERVER') + + if 'Error' in web_server_url: + resp.status = falcon.HTTP_400 + resp.body = json.dumps({'Error': 'Missing Configuration File'}) + return + else: + req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id) + task_details = requests.get(req_url).json() + + if 'error' in task_details: + resp.status = falcon.HTTP_400 + resp.body = json.dumps(task_details) + return + else: + resp.status = falcon.HTTP_200 + resp.body = json.dumps(task_details) + diff --git a/shipyard_airflow/setup.py b/shipyard_airflow/setup.py new file mode 100644 index 00000000..85e81e05 --- /dev/null +++ b/shipyard_airflow/setup.py @@ -0,0 +1,33 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from setuptools import setup + +setup(name='shipyard_airflow', + version='0.1a1', + description='API for managing Airflow-based orchestration', + url='http://github.com/att-comdev/shipyard', + author='Anthony Lin - AT&T', + author_email='al498u@att.com', + license='Apache 2.0', + packages=['shipyard_airflow', + 'shipyard_airflow.control'], + install_requires=[ + 'falcon', + 'requests', + 'configparser', + 'uwsgi>1.4' + ] + ) + diff --git a/shipyard_airflow/shipyard.py b/shipyard_airflow/shipyard.py new file mode 100644 index 00000000..887028b3 --- /dev/null +++ b/shipyard_airflow/shipyard.py @@ -0,0 +1,40 @@ +# Copyright 2017 AT&T Intellectual Property. All other rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import logging +import control.api as api + +def start_shipyard(): + + # Setup root logger + logger = logging.getLogger('shipyard') + + logger.setLevel('DEBUG') + ch = logging.StreamHandler() + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + ch.setFormatter(formatter) + logger.addHandler(ch) + + # Specalized format for API logging + logger = logging.getLogger('shipyard.control') + logger.propagate = False + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s') + + ch = logging.StreamHandler() + ch.setFormatter(formatter) + logger.addHandler(ch) + + return api.start_api() + +shipyard = start_shipyard() +