Expand the Falcon API Framework

Leverage the functionalities of airflow CLI and the
airflow-rest-api-plugin

shipyard_airflow/control/airflow_connections.py

@@ -0,0 +1,122 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+# We need to be able to add/delete connections so that we can create/delete
+# connection endpoints that Airflow needs to connect to
+class AirflowAddConnectionResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, action, conn_id, protocol, host, port):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            if action == 'add':
+                # Concatenate to form the connection URL, Remove whitespaces
+                conn_uri = "%s %s %s %s %s" % (protocol, '://', host, ':', port)
+                conn_uri = conn_uri.replace(' ', '')
+
+                # Form the request URL towards Airflow
+                req_url = '{}/admin/rest_api/api?api=connections&add=true&conn_id={}&conn_uri={}'.format(web_server_url, conn_id, conn_uri)
+            else:
+                self.return_error(resp, falcon.HTTP_400, 'Invalid Paremeters for Adding Airflow Connection')
+                return
+
+            response = requests.get(req_url).json()
+       
+            # Return output
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+
+
+# Delete a particular connection endpoint
+class AirflowDeleteConnectionResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, action, conn_id):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            if action == 'delete':
+                # Form the request URL towards Airflow
+                req_url = '{}/admin/rest_api/api?api=connections&delete=true&conn_id={}'.format(web_server_url, conn_id)
+            else:
+                self.return_error(resp, falcon.HTTP_400, 'Invalid Paremeters for Deleting Airflow Connection')
+                return
+
+            response = requests.get(req_url).json()
+
+            # Return output
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+
+
+# List all current connection endpoints
+class AirflowListConnectionsResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, action):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            if action == 'list':
+                # Form the request URL towards Airflow
+                req_url = '{}/admin/rest_api/api?api=connections&list=true'.format(web_server_url)
+            else:
+                self.return_error(resp, falcon.HTTP_400, 'Invalid Paremeters for listing Airflow Connections')
+                return
+
+            response = requests.get(req_url).json()
+
+            # Return output
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+

shipyard_airflow/control/airflow_dag_state.py

@@ -0,0 +1,43 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class GetDagStateResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, dag_id, execution_date):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            req_url = '{}/admin/rest_api/api?api=dag_state&dag_id={}&execution_date={}'.format(web_server_url, dag_id, execution_date)
+            response = requests.get(req_url).json()
+       
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+

shipyard_airflow/control/airflow_get_task_status.py

@@ -0,0 +1,43 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class GetTaskStatusResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, dag_id, task_id, execution_date):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            req_url = '{}/admin/rest_api/api?api=task_state&dag_id={}&task_id={}&execution_date={}'.format(web_server_url, dag_id, task_id, execution_date)
+            response = requests.get(req_url).json()
+       
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+

shipyard_airflow/control/airflow_get_version.py

@@ -0,0 +1,43 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class GetAirflowVersionResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            # Get Airflow Version
+            req_url = '{}/admin/rest_api/api?api=version'.format(web_server_url)
+            response = requests.get(req_url).json()
+       
+            if response["output"]:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]
+            else:
+                self.return_error(resp, falcon.HTTP_400, 'Fail to Retrieve Airflow Version')
+                return
+

shipyard_airflow/control/airflow_list_dags.py

@@ -0,0 +1,44 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class ListDagsResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            # List available dags
+            req_url = '{}/admin/rest_api/api?api=list_dags'.format(web_server_url)
+            response = requests.get(req_url).json()
+       
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+

shipyard_airflow/control/airflow_list_tasks.py

@@ -0,0 +1,44 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from .base import BaseResource
+
+class ListTasksResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, dag_id):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            # Retrieve all tasks belonging to a particular Dag
+            req_url = '{}/admin/rest_api/api?api=list_tasks&dag_id={}'.format(web_server_url, dag_id)
+            response = requests.get(req_url).json()
+       
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+                resp.body = response["output"]["stdout"]
+

shipyard_airflow/control/airflow_trigger_dag.py

@@ -0,0 +1,49 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import falcon
+import json
+import requests
+
+from dateutil.parser import parse
+from .base import BaseResource
+
+class TriggerDagRunResource(BaseResource):
+
+    authorized_roles = ['user']
+
+    def on_get(self, req, resp, dag_id, run_id):
+        # Retrieve URL
+        web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
+
+        if 'Error' in web_server_url:
+            resp.status = falcon.HTTP_400
+            resp.body = json.dumps({'Error': 'Missing Configuration File'})
+            return
+        else:
+            # Trigger the execution of a Dag
+            # Associate Dag execution with a Run ID
+            req_url = '{}/admin/rest_api/api?api=trigger_dag&dag_id={}&run_id={}'.format(web_server_url, dag_id, run_id)
+            response = requests.get(req_url).json()
+
+            if response["output"]["stderr"]:
+                resp.status = falcon.HTTP_400
+                resp.body = response["output"]["stderr"]
+                return
+            else:
+                resp.status = falcon.HTTP_200
+
+                # Return time of execution so that we can use it to query dag/task status
+                dt = parse(response["response_time"])
+                resp.body = dt.strftime('%Y-%m-%dT%H:%M:%S')
+

shipyard_airflow/control/api.py

@@ -18,6 +18,15 @@ from .regions import RegionsResource, RegionResource
 from .base import ShipyardRequest, BaseResource
 from .tasks import TaskResource
 from .dag_runs import DagRunResource
+from .airflow_get_task_status import GetTaskStatusResource
+from .airflow_list_tasks import ListTasksResource
+from .airflow_list_dags import ListDagsResource
+from .airflow_dag_state import GetDagStateResource
+from .airflow_trigger_dag import TriggerDagRunResource
+from .airflow_connections import AirflowAddConnectionResource
+from .airflow_connections import AirflowDeleteConnectionResource
+from .airflow_connections import AirflowListConnectionsResource
+from .airflow_get_version import GetAirflowVersionResource
 from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware
 
 def start_api():
@@ -34,10 +43,19 @@ def start_api():
         ('/regions/{region_id}', RegionResource()),
         ('/dags/{dag_id}/tasks/{task_id}', TaskResource()),
         ('/dags/{dag_id}/dag_runs', DagRunResource()),
+        ('/list_dags', ListDagsResource()),
+        ('/task_state/dags/{dag_id}/tasks/{task_id}/execution_date/{execution_date}', GetTaskStatusResource()),
+        ('/dag_state/dags/{dag_id}/execution_date/{execution_date}', GetDagStateResource()),
+        ('/list_tasks/dags/{dag_id}', ListTasksResource()),
+        ('/trigger_dag/dags/{dag_id}/run_id/{run_id}', TriggerDagRunResource()),
+        ('/connections/{action}/conn_id/{conn_id}/protocol/{protocol}/host/{host}/port/{port}', AirflowAddConnectionResource()),
+        ('/connections/{action}/conn_id/{conn_id}', AirflowDeleteConnectionResource()),
+        ('/connections/{action}', AirflowListConnectionsResource()),
+        ('/airflow/version', GetAirflowVersionResource()),
     ]
 
     for path, res in v1_0_routes:
-        control_api.add_route('/api/experimental' + path, res)
+        control_api.add_route('/api/v1.0' + path, res)
 
     return control_api