Merge pull request #4 from eanylin/helm_shipyard

Initial Commit

README.md

@@ -1,4 +1,2 @@
 # shipyard
 Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management
-
-## Testing 4 5 6

setup.py

@@ -0,0 +1,33 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+
+setup(name='shipyard_airflow',
+      version='0.1a1',
+      description='API for managing Airflow-based orchestration',
+      url='http://github.com/att-comdev/shipyard',
+      author='Anthony Lin - AT&T',
+      author_email='al498u@att.com',
+      license='Apache 2.0',
+      packages=['shipyard_airflow',
+                'shipyard_airflow.control'],
+      install_requires=[
+        'falcon',
+        'requests',
+        'configparser',
+        'uwsgi>1.4'
+      ]
+     )
+

shipyard_airflow/README.md

@@ -0,0 +1,12 @@
+## Shipyard Airflow ##
+
+A python REST workflow orchestrator
+
+To run:
+
+```
+$ virtualenv -p python3 /var/tmp/shipyard
+$ . /var/tmp/shipyard/bin/activate
+$ python setup.py install
+$ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L
+```

shipyard_airflow/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

shipyard_airflow/control/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

shipyard_airflow/control/api.py

@@ -0,0 +1,54 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+
+from .regions import RegionsResource, RegionResource
+from .base import ShipyardRequest, BaseResource
+from .tasks import TaskResource
+from .dag_runs import DagRunResource
+from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware
+
+def start_api():
+
+    control_api = falcon.API(request_type=ShipyardRequest,
+                             middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()])
+
+    control_api.add_route('/versions', VersionsResource())
+
+    # v1.0 of Shipyard API
+    v1_0_routes = [
+        # API for managing region data
+        ('/regions', RegionsResource()),
+        ('/regions/{region_id}', RegionResource()),
+        ('/dags/{dag_id}/tasks/{task_id}', TaskResource()),
+        ('/dags/{dag_id}/dag_runs', DagRunResource()),
+    ]
+
+    for path, res in v1_0_routes:
+        control_api.add_route('/api/experimental' + path, res)
+
+    return control_api
+
+class VersionsResource(BaseResource):
+
+    authorized_roles = ['anyone']
+
+    def on_get(self, req, resp):
+        resp.body = json.dumps({'v1.0': {
+                                    'path': '/api/v1.0',
+                                    'status': 'stable'
+                                }})
+        resp.status = falcon.HTTP_200
+

shipyard_airflow/control/base.py

@@ -0,0 +1,111 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon, falcon.request as request
+import uuid
+import json
+import configparser
+import os
+
+class BaseResource(object):
+
+    authorized_roles = []
+
+    def on_options(self, req, resp):
+        self_attrs = dir(self)
+        methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH']
+        allowed_methods = []
+
+        for m in methods:
+            if 'on_' + m.lower() in self_attrs:
+                allowed_methods.append(m)
+
+        resp.headers['Allow'] = ','.join(allowed_methods)
+        resp.status = falcon.HTTP_200
+
+    # By default, no one is authorized to use a resource
+    def authorize_roles(self, role_list):
+        authorized = set(self.authorized_roles)
+        applied = set(role_list)
+
+        if authorized.isdisjoint(applied):
+            return False
+        else:
+            return True
+
+    # Error Handling
+    def return_error(self, resp, status_code, message="", retry=False):
+        """
+        Write a error message body and throw a Falcon exception to trigger an HTTP status
+
+        :param resp: Falcon response object to update
+        :param status_code: Falcon status_code constant
+        :param message: Optional error message to include in the body
+        :param retry: Optional flag whether client should retry the operation. Can ignore if we rely solely on 4XX vs 5xx status codes
+        """
+        resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry})
+        resp.content_type = 'application/json'
+        resp.status = status_code
+
+    # Get Config Data
+    def retrieve_config(self, section="", data=""):
+
+        # The current assumption is that shipyard.conf will be placed in a fixed path
+        # within the shipyard container - Path TBD
+        path = '/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf'
+        
+        # Check that shipyard.conf exists
+        if os.path.isfile(path):
+            config = configparser.ConfigParser()
+            config.read(path)
+
+            # Retrieve data from shipyard.conf
+            query_data = config.get(section, data)
+
+            return query_data
+        else:
+            return 'Error - Missing Configuration File'
+
+
+class ShipyardRequestContext(object):
+
+    def __init__(self):
+        self.log_level = 'error'
+        self.user = None
+        self.roles = ['anyone']
+        self.request_id = str(uuid.uuid4())
+        self.external_marker = None
+
+    def set_log_level(self, level):
+        if level in ['error', 'info', 'debug']:
+            self.log_level = level
+
+    def set_user(self, user):
+        self.user = user
+
+    def add_role(self, role):
+        self.roles.append(role)
+
+    def add_roles(self, roles):
+        self.roles.extend(roles)
+
+    def remove_role(self, role):
+        self.roles = [x for x in self.roles
+                      if x != role]
+
+    def set_external_marker(self, marker):
+        self.external_marker = str(marker)[:32]
+
+class ShipyardRequest(request.Request):
+    context_type = ShipyardRequestContext
+

shipyard_airflow/control/dag_runs.py

@@ -0,0 +1,47 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import requests
+import json
+
+from .base import BaseResource
+
+class DagRunResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id)
+ 
+            response = requests.post(req_url,
+                                     json={
+                                     "run_id": run_id,
+                                     "conf": conf,
+                                     "execution_date": execution_date,
+                                 })
+
+            if response.ok:
+                resp.status = falcon.HTTP_200
+            else:
+                self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag')
+                return
+

shipyard_airflow/control/middleware.py

@@ -0,0 +1,89 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import logging
+
+class AuthMiddleware(object):
+
+    # Authentication
+    def process_request(self, req, resp):
+        ctx = req.context
+        token = req.get_header('X-Auth-Token')
+
+        user = self.validate_token(token)
+
+        if user is not None:
+            ctx.set_user(user)
+            user_roles = self.role_list(user)
+            ctx.add_roles(user_roles)
+        else:
+            ctx.add_role('anyone')
+
+    # Authorization
+    def process_resource(self, req, resp, resource, params):
+        ctx = req.context
+
+        if not resource.authorize_roles(ctx.roles):
+            raise falcon.HTTPUnauthorized('Authentication required',
+                                          ('This resource requires an authorized role.'))
+
+    # Return the username associated with an authenticated token or None
+    def validate_token(self, token):
+        if token == '10':
+            return 'shipyard'
+        elif token == 'admin':
+            return 'admin'
+        else:
+            return None
+
+    # Return the list of roles assigned to the username
+    # Roles need to be an enum
+    def role_list(self, username):
+        if username == 'shipyard':
+            return ['user']
+        elif username == 'admin':
+            return ['user', 'admin']
+
+class ContextMiddleware(object):
+
+    def process_request(self, req, resp):
+        ctx = req.context
+
+        requested_logging = req.get_header('X-Log-Level')
+
+        if requested_logging == 'DEBUG' and 'admin' in ctx.roles:
+            ctx.set_log_level('debug')
+        elif requested_logging == 'INFO':
+            ctx.set_log_level('info')
+
+        ext_marker = req.get_header('X-Context-Marker')
+        ctx.set_external_marker(ext_marker if ext_marker is not None else '')
+
+class LoggingMiddleware(object):
+
+    def __init__(self):
+        self.logger = logging.getLogger('shipyard.control')
+
+    def process_response(self, req, resp, resource, req_succeeded):
+        ctx = req.context
+
+        extra = {
+            'user': ctx.user,
+            'req_id': ctx.request_id,
+            'external_ctx': ctx.external_marker,
+        }
+
+        resp.append_header('X-Shipyard-Req', ctx.request_id)
+        self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra)
+

shipyard_airflow/control/regions.py

@@ -0,0 +1,31 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+
+from .base import BaseResource
+
+class RegionsResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp):
+        resp.status = falcon.HTTP_200
+
+class RegionResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, region_id):
+        resp.status = falcon.HTTP_200
+

shipyard_airflow/control/shipyard.conf

@@ -0,0 +1,3 @@
+[BASE]
+WEB_SERVER=http://localhost:32080
+

shipyard_airflow/control/tasks.py

@@ -0,0 +1,43 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class TaskResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, dag_id, task_id):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id)
+            task_details = requests.get(req_url).json()
+
+            if 'error' in task_details:
+                resp.status = falcon.HTTP_400
+                resp.body = json.dumps(task_details)
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = json.dumps(task_details)
+

shipyard_airflow/setup.py

@@ -0,0 +1,33 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from setuptools import setup
+
+setup(name='shipyard_airflow',
+      version='0.1a1',
+      description='API for managing Airflow-based orchestration',
+      url='http://github.com/att-comdev/shipyard',
+      author='Anthony Lin - AT&T',
+      author_email='al498u@att.com',
+      license='Apache 2.0',
+      packages=['shipyard_airflow',
+                'shipyard_airflow.control'],
+      install_requires=[
+        'falcon',
+        'requests',
+        'configparser',
+        'uwsgi>1.4'
+      ]
+     )
+

shipyard_airflow/shipyard.py

@@ -0,0 +1,40 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import control.api as api
+
+def start_shipyard():
+    
+    # Setup root logger
+    logger = logging.getLogger('shipyard')
+
+    logger.setLevel('DEBUG')
+    ch = logging.StreamHandler()
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    # Specalized format for API logging
+    logger = logging.getLogger('shipyard.control')
+    logger.propagate = False
+    formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s')
+
+    ch = logging.StreamHandler()
+    ch.setFormatter(formatter)
+    logger.addHandler(ch)
+
+    return api.start_api()
+
+shipyard = start_shipyard()
+