Browse Source

Merge pull request #4 from eanylin/helm_shipyard

Initial Commit
changes/87/569187/1
Alan Meadows 4 years ago
committed by GitHub
parent
commit
ffda0f8427
  1. 2
      README.md
  2. 33
      setup.py
  3. 12
      shipyard_airflow/README.md
  4. 13
      shipyard_airflow/__init__.py
  5. 13
      shipyard_airflow/control/__init__.py
  6. 54
      shipyard_airflow/control/api.py
  7. 111
      shipyard_airflow/control/base.py
  8. 47
      shipyard_airflow/control/dag_runs.py
  9. 89
      shipyard_airflow/control/middleware.py
  10. 31
      shipyard_airflow/control/regions.py
  11. 3
      shipyard_airflow/control/shipyard.conf
  12. 43
      shipyard_airflow/control/tasks.py
  13. 33
      shipyard_airflow/setup.py
  14. 40
      shipyard_airflow/shipyard.py

2
README.md

@ -1,4 +1,2 @@
# shipyard
Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management
## Testing 4 5 6

33
setup.py

@ -0,0 +1,33 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from setuptools import setup
setup(name='shipyard_airflow',
version='0.1a1',
description='API for managing Airflow-based orchestration',
url='http://github.com/att-comdev/shipyard',
author='Anthony Lin - AT&T',
author_email='al498u@att.com',
license='Apache 2.0',
packages=['shipyard_airflow',
'shipyard_airflow.control'],
install_requires=[
'falcon',
'requests',
'configparser',
'uwsgi>1.4'
]
)

12
shipyard_airflow/README.md

@ -0,0 +1,12 @@
## Shipyard Airflow ##
A python REST workflow orchestrator
To run:
```
$ virtualenv -p python3 /var/tmp/shipyard
$ . /var/tmp/shipyard/bin/activate
$ python setup.py install
$ uwsgi --http :9000 -w shipyard_airflow.shipyard --callable shipyard -L
```

13
shipyard_airflow/__init__.py

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

13
shipyard_airflow/control/__init__.py

@ -0,0 +1,13 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

54
shipyard_airflow/control/api.py

@ -0,0 +1,54 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
import json
from .regions import RegionsResource, RegionResource
from .base import ShipyardRequest, BaseResource
from .tasks import TaskResource
from .dag_runs import DagRunResource
from .middleware import AuthMiddleware, ContextMiddleware, LoggingMiddleware
def start_api():
control_api = falcon.API(request_type=ShipyardRequest,
middleware=[AuthMiddleware(), ContextMiddleware(), LoggingMiddleware()])
control_api.add_route('/versions', VersionsResource())
# v1.0 of Shipyard API
v1_0_routes = [
# API for managing region data
('/regions', RegionsResource()),
('/regions/{region_id}', RegionResource()),
('/dags/{dag_id}/tasks/{task_id}', TaskResource()),
('/dags/{dag_id}/dag_runs', DagRunResource()),
]
for path, res in v1_0_routes:
control_api.add_route('/api/experimental' + path, res)
return control_api
class VersionsResource(BaseResource):
authorized_roles = ['anyone']
def on_get(self, req, resp):
resp.body = json.dumps({'v1.0': {
'path': '/api/v1.0',
'status': 'stable'
}})
resp.status = falcon.HTTP_200

111
shipyard_airflow/control/base.py

@ -0,0 +1,111 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon, falcon.request as request
import uuid
import json
import configparser
import os
class BaseResource(object):
authorized_roles = []
def on_options(self, req, resp):
self_attrs = dir(self)
methods = ['GET', 'HEAD', 'POST', 'PUT', 'DELETE', 'PATCH']
allowed_methods = []
for m in methods:
if 'on_' + m.lower() in self_attrs:
allowed_methods.append(m)
resp.headers['Allow'] = ','.join(allowed_methods)
resp.status = falcon.HTTP_200
# By default, no one is authorized to use a resource
def authorize_roles(self, role_list):
authorized = set(self.authorized_roles)
applied = set(role_list)
if authorized.isdisjoint(applied):
return False
else:
return True
# Error Handling
def return_error(self, resp, status_code, message="", retry=False):
"""
Write a error message body and throw a Falcon exception to trigger an HTTP status
:param resp: Falcon response object to update
:param status_code: Falcon status_code constant
:param message: Optional error message to include in the body
:param retry: Optional flag whether client should retry the operation. Can ignore if we rely solely on 4XX vs 5xx status codes
"""
resp.body = json.dumps({'type': 'error', 'message': message, 'retry': retry})
resp.content_type = 'application/json'
resp.status = status_code
# Get Config Data
def retrieve_config(self, section="", data=""):
# The current assumption is that shipyard.conf will be placed in a fixed path
# within the shipyard container - Path TBD
path = '/home/ubuntu/att-comdev/shipyard/shipyard_airflow/control/shipyard.conf'
# Check that shipyard.conf exists
if os.path.isfile(path):
config = configparser.ConfigParser()
config.read(path)
# Retrieve data from shipyard.conf
query_data = config.get(section, data)
return query_data
else:
return 'Error - Missing Configuration File'
class ShipyardRequestContext(object):
def __init__(self):
self.log_level = 'error'
self.user = None
self.roles = ['anyone']
self.request_id = str(uuid.uuid4())
self.external_marker = None
def set_log_level(self, level):
if level in ['error', 'info', 'debug']:
self.log_level = level
def set_user(self, user):
self.user = user
def add_role(self, role):
self.roles.append(role)
def add_roles(self, roles):
self.roles.extend(roles)
def remove_role(self, role):
self.roles = [x for x in self.roles
if x != role]
def set_external_marker(self, marker):
self.external_marker = str(marker)[:32]
class ShipyardRequest(request.Request):
context_type = ShipyardRequestContext

47
shipyard_airflow/control/dag_runs.py

@ -0,0 +1,47 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
import requests
import json
from .base import BaseResource
class DagRunResource(BaseResource):
authorized_roles = ['user']
def on_post(self, req, resp, dag_id, run_id=None, conf=None, execution_date=None):
# Retrieve URL
web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
if 'Error' in web_server_url:
resp.status = falcon.HTTP_400
resp.body = json.dumps({'Error': 'Missing Configuration File'})
return
else:
req_url = '{}/api/experimental/dags/{}/dag_runs'.format(web_server_url, dag_id)
response = requests.post(req_url,
json={
"run_id": run_id,
"conf": conf,
"execution_date": execution_date,
})
if response.ok:
resp.status = falcon.HTTP_200
else:
self.return_error(resp, falcon.HTTP_400, 'Fail to Execute Dag')
return

89
shipyard_airflow/control/middleware.py

@ -0,0 +1,89 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
import logging
class AuthMiddleware(object):
# Authentication
def process_request(self, req, resp):
ctx = req.context
token = req.get_header('X-Auth-Token')
user = self.validate_token(token)
if user is not None:
ctx.set_user(user)
user_roles = self.role_list(user)
ctx.add_roles(user_roles)
else:
ctx.add_role('anyone')
# Authorization
def process_resource(self, req, resp, resource, params):
ctx = req.context
if not resource.authorize_roles(ctx.roles):
raise falcon.HTTPUnauthorized('Authentication required',
('This resource requires an authorized role.'))
# Return the username associated with an authenticated token or None
def validate_token(self, token):
if token == '10':
return 'shipyard'
elif token == 'admin':
return 'admin'
else:
return None
# Return the list of roles assigned to the username
# Roles need to be an enum
def role_list(self, username):
if username == 'shipyard':
return ['user']
elif username == 'admin':
return ['user', 'admin']
class ContextMiddleware(object):
def process_request(self, req, resp):
ctx = req.context
requested_logging = req.get_header('X-Log-Level')
if requested_logging == 'DEBUG' and 'admin' in ctx.roles:
ctx.set_log_level('debug')
elif requested_logging == 'INFO':
ctx.set_log_level('info')
ext_marker = req.get_header('X-Context-Marker')
ctx.set_external_marker(ext_marker if ext_marker is not None else '')
class LoggingMiddleware(object):
def __init__(self):
self.logger = logging.getLogger('shipyard.control')
def process_response(self, req, resp, resource, req_succeeded):
ctx = req.context
extra = {
'user': ctx.user,
'req_id': ctx.request_id,
'external_ctx': ctx.external_marker,
}
resp.append_header('X-Shipyard-Req', ctx.request_id)
self.logger.info("%s - %s" % (req.uri, resp.status), extra=extra)

31
shipyard_airflow/control/regions.py

@ -0,0 +1,31 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
from .base import BaseResource
class RegionsResource(BaseResource):
authorized_roles = ['user']
def on_get(self, req, resp):
resp.status = falcon.HTTP_200
class RegionResource(BaseResource):
authorized_roles = ['user']
def on_get(self, req, resp, region_id):
resp.status = falcon.HTTP_200

3
shipyard_airflow/control/shipyard.conf

@ -0,0 +1,3 @@
[BASE]
WEB_SERVER=http://localhost:32080

43
shipyard_airflow/control/tasks.py

@ -0,0 +1,43 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import falcon
import json
import requests
from .base import BaseResource
class TaskResource(BaseResource):
authorized_roles = ['user']
def on_get(self, req, resp, dag_id, task_id):
# Retrieve URL
web_server_url = self.retrieve_config('BASE', 'WEB_SERVER')
if 'Error' in web_server_url:
resp.status = falcon.HTTP_400
resp.body = json.dumps({'Error': 'Missing Configuration File'})
return
else:
req_url = '{}/api/experimental/dags/{}/tasks/{}'.format(web_server_url, dag_id, task_id)
task_details = requests.get(req_url).json()
if 'error' in task_details:
resp.status = falcon.HTTP_400
resp.body = json.dumps(task_details)
return
else:
resp.status = falcon.HTTP_200
resp.body = json.dumps(task_details)

33
shipyard_airflow/setup.py

@ -0,0 +1,33 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from setuptools import setup
setup(name='shipyard_airflow',
version='0.1a1',
description='API for managing Airflow-based orchestration',
url='http://github.com/att-comdev/shipyard',
author='Anthony Lin - AT&T',
author_email='al498u@att.com',
license='Apache 2.0',
packages=['shipyard_airflow',
'shipyard_airflow.control'],
install_requires=[
'falcon',
'requests',
'configparser',
'uwsgi>1.4'
]
)

40
shipyard_airflow/shipyard.py

@ -0,0 +1,40 @@
# Copyright 2017 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import control.api as api
def start_shipyard():
# Setup root logger
logger = logging.getLogger('shipyard')
logger.setLevel('DEBUG')
ch = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)
# Specalized format for API logging
logger = logging.getLogger('shipyard.control')
logger.propagate = False
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(user)s - %(req_id)s - %(external_ctx)s - %(message)s')
ch = logging.StreamHandler()
ch.setFormatter(formatter)
logger.addHandler(ch)
return api.start_api()
shipyard = start_shipyard()
Loading…
Cancel
Save