Refactor shipyard to UCP target layout

Refactor Shipyard to be better able to leverage common
packages and conform with the target UCP standard layout.

This change supports the same tox entrypoints at
the root level, but the preferred approach is to use make
targets defined in the Makefile such as 'make tests' and
'make lint'

The previous tox.ini has moved and been
tailored to the specifics of each subproject at
src/bin/*/tox.ini

Autotmatic generation of the policy and configuration
files has been removed from the sphinx build for now
but these files will be automatically generated locally
into the docs source by using a 'make docs' command.
This may need to be revisited later to re-enable the
automatic generation of these files such that readthedocs
would still support the project layout.

Change-Id: Ifdc1cd4cf35fb3c5923414c677b781a60a9bae42

.dockerignore

@@ -0,0 +1,119 @@
+# Byte-compiled / optimized / DLL files
+**/__pycache__/
+*.py[cod]
+*$py.class
+
+# Unit tests
+src/bin/*/tests/unit/
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+**/env/
+**/build/
+**/develop-eggs/
+**/dist/
+**/downloads/
+**/eggs/
+**/.eggs/
+lib/
+**/lib64/
+parts/
+**/sdist/
+var/
+wheels/
+**/*.egg-info/
+.installed.cfg
+*.egg
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+**/htmlcov/
+cov/*
+**/.tox/
+**/.coverage
+**/.coverage.*
+**/.cache
+nosetests.xml
+coverage.xml
+*.cover
+.hypothesis/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/*/_static/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# dotenv
+.env
+
+# virtualenv
+.venv
+venv/
+ENV/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache/
+
+# Generated bogus docs
+ChangeLog
+AUTHORS
+
+# build/lint artifacts
+*.tgz
+**/*.tgz
+/charts/shipyard/charts
+/charts/shipyard/requirements.lock
+.DS_Store
+
+# vscode
+.vscode/
+
+!src/lib/

.gitignore

@@ -64,7 +64,6 @@ instance/
 
 # Sphinx documentation
 docs/_build/
-docs/*/_static/
 
 # PyBuilder
 target/

Makefile

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+BUILD_CTX                  ?= src/bin
 DOCKER_REGISTRY            ?= quay.io
 IMAGE_PREFIX               ?= attcomdev
 IMAGE_TAG                  ?= latest
@@ -39,6 +40,7 @@ $(IMAGE_NAME):
 # Create tgz of the chart
 .PHONY: charts
 charts: clean
+	tools/helm_tk.sh $(HELM)
 	$(HELM) dep up charts/shipyard
 	$(HELM) package charts/shipyard
 
@@ -55,6 +57,12 @@ dry-run: clean
 .PHONY: docs
 docs: clean build_docs
 
+
+.PHONY: tests
+tests:
+	cd $(BUILD_CTX)/shipyard_airflow; tox
+	cd $(BUILD_CTX)/shipyard_client; tox
+
 # Make targets intended for use by the primary targets above.
 
 .PHONY: run
@@ -67,15 +75,17 @@ build_airflow:
 
 .PHONY: build_shipyard
 build_shipyard:
-	docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile .
+	docker build -t $(IMAGE) --label $(LABEL) -f $(IMAGE_DIR)/Dockerfile --build-arg ctx_base=$(BUILD_CTX) .
 
 .PHONY: clean
 clean:
-	rm -rf build
+	cd $(BUILD_CTX)/shipyard_airflow; rm -rf build
+	cd $(BUILD_CTX)/shipyard_client; rm -rf build
 
 .PHONY: pep8
 pep8:
-	tox -e pep8
+	cd $(BUILD_CTX)/shipyard_airflow; tox -e pep8
+	cd $(BUILD_CTX)/shipyard_client; tox -e pep8
 
 .PHONY: helm_lint
 helm_lint: clean

charts/shipyard/values.yaml

@@ -346,7 +346,7 @@ conf:
     workflow_orchestrator:get_renderedconfigdocs: rule:admin_required
   paste:
     app:shipyard-api:
-      paste.app_factory: shipyard_airflow.shipyard:paste_start_shipyard
+      paste.app_factory: shipyard_airflow.shipyard_api:paste_start_shipyard
     pipeline:main:
       pipeline: authtoken shipyard-api
     filter:authtoken:

docs/requirements.txt

@@ -0,0 +1,5 @@
+#
+# Requirements for creating documentation only.
+#
+sphinx>=1.6.2
+sphinx_rtd_theme==0.2.4

etc/shipyard/shipyard.conf.sample → docs/source/_static/shipyard.conf.sample

@@ -4,11 +4,11 @@
 [airflow]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # Airflow worker url scheme (string value)
-#worker_endpoint_scheme = 'http'
+#worker_endpoint_scheme = http
 
 # Airflow worker port (integer value)
 #worker_port = 8793
@@ -17,7 +17,7 @@
 [armada]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The service type for the service playing the role of Armada. The specified
@@ -29,7 +29,7 @@
 [base]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The web server for Airflow (string value)
@@ -54,7 +54,7 @@
 [deckhand]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The service type for the service playing the role of Deckhand. The specified
@@ -66,7 +66,7 @@
 [drydock]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The service type for the service playing the role of Drydock. The specified
@@ -75,6 +75,16 @@
 #service_type = physicalprovisioner
 
 
+[k8s_logs]
+
+#
+# From shipyard_api
+#
+
+# Namespace of UCP Pods (string value)
+#ucp_namespace = ucp
+
+
 [keystone_authtoken]
 
 #
@@ -89,6 +99,22 @@
 # *not* be the same endpoint the service user utilizes for validating tokens,
 # because normal end users may not be able to reach that endpoint. (string
 # value)
+# Deprecated group/name - [keystone_authtoken]/auth_uri
+#www_authenticate_uri = <None>
+
+# DEPRECATED: Complete "public" Identity API endpoint. This endpoint should not
+# be an "admin" endpoint, as it should be accessible by all end users.
+# Unauthenticated clients are redirected to this endpoint to authenticate.
+# Although this endpoint should ideally be unversioned, client support in the
+# wild varies. If you're using a versioned v2 endpoint here, then this should
+# *not* be the same endpoint the service user utilizes for validating tokens,
+# because normal end users may not be able to reach that endpoint. This option
+# is deprecated in favor of www_authenticate_uri and will be removed in the S
+# release. (string value)
+# This option is deprecated for removal since Queens.
+# Its value may be silently ignored in the future.
+# Reason: The auth_uri option is deprecated in favor of www_authenticate_uri
+# and will be removed in the S  release.
 #auth_uri = <None>
 
 # API version of the admin Identity API endpoint. (string value)
@@ -161,7 +187,10 @@
 # in the cache. If ENCRYPT, token data is encrypted and authenticated in the
 # cache. If the value is not one of these options or empty, auth_token will
 # raise an exception on initialization. (string value)
-# Allowed values: None, MAC, ENCRYPT
+# Possible values:
+# None - <No description provided>
+# MAC - <No description provided>
+# ENCRYPT - <No description provided>
 #memcache_security_strategy = None
 
 # (Optional, mandatory if memcache_security_strategy is defined) This string is
@@ -249,31 +278,26 @@
 #auth_section = <None>
 
 
-[k8s_logs]
-
-#
-# From shipyard_airflow
-#
-
-# The namespace of the UCP Pods (string value)
-#ucp_namespace = ucp
-
-
 [logging]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The default logging level for the root logger. ERROR=40, WARNING=30, INFO=20,
 # DEBUG=10 (integer value)
 #log_level = 10
 
+# The logging levels for named loggers. Use standard representations for
+# logging levels: ERROR. WARN, INFO, DEBUG. Configuration file format:
+# named_log_levels = keystoneauth:INFO,othlgr:WARN (dict value)
+#named_log_levels = keystoneauth:20,keystonemiddleware:20
+
 
 [requests_config]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # Airflow logs retrieval connect timeout (in seconds) (integer value)
@@ -299,7 +323,7 @@
 [shipyard]
 
 #
-# From shipyard_airflow
+# From shipyard_api
 #
 
 # The service type for the service playing the role of Shipyard. The specified

docs/source/_static/shipyard.policy.yaml.sample

@@ -0,0 +1,61 @@
+# Actions requiring admin authority
+#"admin_required": "role:admin"
+
+# List workflow actions invoked by users
+# GET  /api/v1.0/actions
+#"workflow_orchestrator:list_actions": "rule:admin_required"
+
+# Create a workflow action
+# POST  /api/v1.0/actions
+#"workflow_orchestrator:create_action": "rule:admin_required"
+
+# Retrieve an action by its id
+# GET  /api/v1.0/actions/{action_id}
+#"workflow_orchestrator:get_action": "rule:admin_required"
+
+# Retrieve an action step by its id
+# GET  /api/v1.0/actions/{action_id}/steps/{step_id}
+#"workflow_orchestrator:get_action_step": "rule:admin_required"
+
+# Retrieve logs of an action step by its id
+# GET  /api/v1.0/actions/{action_id}/steps/{step_id}/logs
+#"workflow_orchestrator:get_action_step_logs": "rule:admin_required"
+
+# Retrieve an action validation by its id
+# GET  /api/v1.0/actions/{action_id}/validations/{validation_id}
+#"workflow_orchestrator:get_action_validation": "rule:admin_required"
+
+# Send a control to an action
+# POST  /api/v1.0/actions/{action_id}/control/{control_verb}
+#"workflow_orchestrator:invoke_action_control": "rule:admin_required"
+
+# Retrieve the status of the configdocs
+# GET  /api/v1.0/configdocs
+#"workflow_orchestrator:get_configdocs_status": "rule:admin_required"
+
+# Ingest configuration documents for the site design
+# POST  /api/v1.0/configdocs/{collection_id}
+#"workflow_orchestrator:create_configdocs": "rule:admin_required"
+
+# Retrieve a collection of configuration documents
+# GET  /api/v1.0/configdocs/{collection_id}
+#"workflow_orchestrator:get_configdocs": "rule:admin_required"
+
+# Move documents from the Shipyard buffer to the committed documents
+# POST  /api/v1.0/commitconfigdocs
+#"workflow_orchestrator:commit_configdocs": "rule:admin_required"
+
+# Retrieve the configuration documents rendered by Deckhand into a
+# complete design
+# GET  /api/v1.0/renderedconfigdocs
+#"workflow_orchestrator:get_renderedconfigdocs": "rule:admin_required"
+
+# Retrieve the list of workflows (DAGs) that have been invoked in
+# Airflow, whether via Shipyard or scheduled
+# GET  /api/v1.0/workflows
+#"workflow_orchestrator:list_workflows": "rule:admin_required"
+
+# Retrieve the detailed information for a workflow (DAG) from Airflow
+# GET  /api/v1.0/workflows/{id}
+#"workflow_orchestrator:get_workflow": "rule:admin_required"
+

docs/source/conf.py

@@ -34,19 +34,9 @@ import sphinx_rtd_theme
 extensions = [
     'sphinx.ext.autodoc',
     'sphinx.ext.todo',
-    'sphinx.ext.viewcode',
-    'oslo_config.sphinxconfiggen',
-    'oslo_policy.sphinxpolicygen'
+    'sphinx.ext.viewcode'
 ]
 
-# oslo_config.sphinxconfiggen options
-config_generator_config_file = '../../generator/config-generator.conf'
-sample_config_basename = '_static/shipyard'
-
-# oslo_policy.sphinxpolicygen options
-policy_generator_config_file = '../../generator/policy-generator.conf'
-sample_policy_basename = '_static/shipyard'
-
 # Add any paths that contain templates here, relative to this directory.
 # templates_path = []
 

etc/shipyard/api-paste.ini

@@ -16,7 +16,7 @@
 #Used to configure uWSGI middleware pipeline
 
 [app:shipyard-api]
-paste.app_factory = shipyard_airflow.shipyard:paste_start_shipyard
+paste.app_factory = shipyard_airflow.shipyard_api:paste_start_shipyard
 
 [pipeline:main]
 pipeline = authtoken shipyard-api

images/airflow/requirements.txt

@@ -20,8 +20,6 @@ psycopg2==2.7.3.1
 docker-py==1.6.0
 apache-airflow[crypto,celery,postgres,hive,hdfs,jdbc]==1.9.0
 python-openstackclient==3.11.0
-sphinx>=1.6.2
-sphinx_rtd_theme==0.2.4
 
 # Dependencies for other UCP components
 git+https://github.com/att-comdev/deckhand.git@3cdf3d2d896d43c6e3bc26170522c3eee0d7158f#egg=deckhand

images/shipyard/Dockerfile

@@ -20,6 +20,7 @@ ENV LC_ALL C.UTF-8
 ENV LANG C.UTF-8
 
 ARG DEBIAN_FRONTEND=noninteractive
+ARG ctx_base=src/bin
 
 # Expose port 9000 for application
 EXPOSE $PORT
@@ -29,21 +30,30 @@ ENTRYPOINT ["/home/shipyard/entrypoint.sh"]
 CMD ["server"]
 
 # Create shipyard user
-RUN useradd -ms /bin/bash shipyard
+RUN useradd -ms /bin/bash shipyard \
+    && mkdir -p /home/shipyard/shipyard \
+    && mkdir -p /home/shipyard/shipyard_client
 
 # Copy entrypoint.sh to /home/shipyard
-COPY entrypoint.sh /home/shipyard/entrypoint.sh
-# Change permissions
+COPY ${ctx_base}/shipyard_airflow/entrypoint.sh /home/shipyard/entrypoint.sh
+# Change permissions and set up directories
 RUN chown -R shipyard: /home/shipyard \
     && chmod +x /home/shipyard/entrypoint.sh
 
-# Set work directory and install dependencies
-WORKDIR /home/shipyard/shipyard
-COPY requirements.txt /home/shipyard/shipyard
-RUN pip3 install -r requirements.txt
+# Requirements
+COPY ${ctx_base}/shipyard_airflow/requirements.txt /home/shipyard/api_requirements.txt
+RUN pip3 install -r /home/shipyard/api_requirements.txt
 
-COPY . /home/shipyard/shipyard
-RUN python3 setup.py install
+COPY ${ctx_base}/shipyard_client/requirements.txt /home/shipyard/client_requirements.txt
+RUN pip3 install -r /home/shipyard/client_requirements.txt
+
+# Shipyard source and installation
+COPY ${ctx_base}/shipyard_client /home/shipyard/shipyard_client/
+RUN cd /home/shipyard/shipyard_client \
+    && python3 setup.py install
+COPY ${ctx_base}/shipyard_airflow /home/shipyard/shipyard/
+RUN cd /home/shipyard/shipyard \
+    && python3 setup.py install
 
 # Set user to shipyard
 USER shipyard

requirements.readthedocs.txt

@@ -0,0 +1 @@
+src/bin/shipyard_airflow/

setup.cfg

@@ -1,38 +0,0 @@
-[metadata]
-name = shipyard
-summary = Directed acyclic graph controller for Kubernetes and OpenStack control plane life cycle management
-description-file = README.md
-
-author = undercloud team
-home-page = https://github.com/att-comdev/shipyard
-classifier =
-    Intended Audience :: Information Technology
-    Intended Audience :: System Administrators
-    License :: OSI Approved :: Apache Software License
-    Operating System :: POSIX :: Linux
-    Programming Language :: Python
-    Programming Language :: Python :: 3
-    Programming Language :: Python :: 3.5
-
-[files]
-packages =
-    shipyard_airflow
-    shipyard_client
-
-[entry_points]
-oslo.config.opts =
-    shipyard_airflow = shipyard_airflow.conf.opts:list_opts
-oslo.policy.policies =
-    shipyard_airflow = shipyard_airflow.policy:list_policies
-console_scripts =
-    shipyard = shipyard_client.cli.commands:shipyard
-    upgrade_db = shipyard_airflow.shipyard_upgrade_db:upgrade_db
-
-[build_sphinx]
-source-dir = docs/source
-build-dir = docs/build
-all_files = 1
-warning-is-error = 1
-
-[upload_sphinx]
-upload-dir = docs/build/html

.coveragerc → src/bin/shipyard_airflow/.coveragerc

@@ -2,4 +2,3 @@
 # omit third party code and unit tests
 omit =
     shipyard_airflow/plugins/rest_api_plugin.py
-    shipyard_client/tests/*

entrypoint.sh → src/bin/shipyard_airflow/entrypoint.sh

@@ -14,17 +14,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -ex
-
-CMD="shipyard"
-PORT=${PORT:-9000}
-HTTP_TIMEOUT=${HTTP_TIMEOUT:-600}
-# Number of uWSGI workers to handle API request
-SHIPYARD_API_WORKERS=${SHIPYARD_API_WORKERS:-"4"}
-#Threads per worker
-SHIPYARD_API_THREADS=${SHIPYARD_API_THREADS:-"1"}
-
+set -e
 if [ "$1" = 'server' ]; then
+    set -x
+    PORT=${PORT:-9000}
+    HTTP_TIMEOUT=${HTTP_TIMEOUT:-600}
+    # Number of uWSGI workers to handle API request
+    SHIPYARD_API_WORKERS=${SHIPYARD_API_WORKERS:-"4"}
+    #Threads per worker
+    SHIPYARD_API_THREADS=${SHIPYARD_API_THREADS:-"1"}
     # Start shipyard application
     exec uwsgi \
         --http :${PORT} \
@@ -36,6 +34,7 @@ if [ "$1" = 'server' ]; then
         --workers $SHIPYARD_API_WORKERS \
         --http-timeout ${HTTP_TIMEOUT}
 else
+    CMD="shipyard"
     # Execute shipyard command
     exec ${CMD} $@
 fi

etc/shipyard/policy.yaml.sample → src/bin/shipyard_airflow/etc/shipyard/policy.yaml.sample

@@ -9,19 +9,19 @@
 # POST  /api/v1.0/actions
 #"workflow_orchestrator:create_action": "rule:admin_required"
 
-# Retreive an action by its id
+# Retrieve an action by its id
 # GET  /api/v1.0/actions/{action_id}
 #"workflow_orchestrator:get_action": "rule:admin_required"
 
-# Retreive an action step by its id
+# Retrieve an action step by its id
 # GET  /api/v1.0/actions/{action_id}/steps/{step_id}
 #"workflow_orchestrator:get_action_step": "rule:admin_required"
 
-# Retreive the logs of an action step by its id
+# Retrieve logs of an action step by its id
 # GET  /api/v1.0/actions/{action_id}/steps/{step_id}/logs
 #"workflow_orchestrator:get_action_step_logs": "rule:admin_required"
 
-# Retreive an action validation by its id
+# Retrieve an action validation by its id
 # GET  /api/v1.0/actions/{action_id}/validations/{validation_id}
 #"workflow_orchestrator:get_action_validation": "rule:admin_required"
 
@@ -29,6 +29,10 @@
 # POST  /api/v1.0/actions/{action_id}/control/{control_verb}
 #"workflow_orchestrator:invoke_action_control": "rule:admin_required"
 
+# Retrieve the status of the configdocs
+# GET  /api/v1.0/configdocs
+#"workflow_orchestrator:get_configdocs_status": "rule:admin_required"
+
 # Ingest configuration documents for the site design
 # POST  /api/v1.0/configdocs/{collection_id}
 #"workflow_orchestrator:create_configdocs": "rule:admin_required"

src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample

@@ -0,0 +1,332 @@
+[DEFAULT]
+
+
+[airflow]
+
+#
+# From shipyard_api
+#
+
+# Airflow worker url scheme (string value)
+#worker_endpoint_scheme = http
+
+# Airflow worker port (integer value)
+#worker_port = 8793
+
+
+[armada]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Armada. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = armada
+
+
+[base]
+
+#
+# From shipyard_api
+#
+
+# The web server for Airflow (string value)
+#web_server = http://localhost:32080/
+
+# Seconds to wait to connect to the airflow api (integer value)
+#airflow_api_connect_timeout = 5
+
+# Seconds to wait for a response from the airflow api (integer value)
+#airflow_api_read_timeout = 60
+
+# The database for shipyard (string value)
+#postgresql_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/shipyard
+
+# The database for airflow (string value)
+#postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow
+
+# The direcotry containing the alembic.ini file (string value)
+#alembic_ini_path = /home/shipyard/shipyard
+
+
+[deckhand]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Deckhand. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = deckhand
+
+
+[drydock]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Drydock. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = physicalprovisioner
+
+
+[k8s_logs]
+
+#
+# From shipyard_api
+#
+
+# Namespace of UCP Pods (string value)
+#ucp_namespace = ucp
+
+
+[keystone_authtoken]
+
+#
+# From keystonemiddleware.auth_token
+#
+
+# Complete "public" Identity API endpoint. This endpoint should not be an
+# "admin" endpoint, as it should be accessible by all end users.
+# Unauthenticated clients are redirected to this endpoint to authenticate.
+# Although this endpoint should ideally be unversioned, client support in the
+# wild varies. If you're using a versioned v2 endpoint here, then this should
+# *not* be the same endpoint the service user utilizes for validating tokens,
+# because normal end users may not be able to reach that endpoint. (string
+# value)
+# Deprecated group/name - [keystone_authtoken]/auth_uri
+#www_authenticate_uri = <None>
+
+# DEPRECATED: Complete "public" Identity API endpoint. This endpoint should not
+# be an "admin" endpoint, as it should be accessible by all end users.
+# Unauthenticated clients are redirected to this endpoint to authenticate.
+# Although this endpoint should ideally be unversioned, client support in the
+# wild varies. If you're using a versioned v2 endpoint here, then this should
+# *not* be the same endpoint the service user utilizes for validating tokens,
+# because normal end users may not be able to reach that endpoint. This option
+# is deprecated in favor of www_authenticate_uri and will be removed in the S
+# release. (string value)
+# This option is deprecated for removal since Queens.
+# Its value may be silently ignored in the future.
+# Reason: The auth_uri option is deprecated in favor of www_authenticate_uri
+# and will be removed in the S  release.
+#auth_uri = <None>
+
+# API version of the admin Identity API endpoint. (string value)
+#auth_version = <None>
+
+# Do not handle authorization requests within the middleware, but delegate the
+# authorization decision to downstream WSGI components. (boolean value)
+#delay_auth_decision = false
+
+# Request timeout value for communicating with Identity API server. (integer
+# value)
+#http_connect_timeout = <None>
+
+# How many times are we trying to reconnect when communicating with Identity
+# API Server. (integer value)
+#http_request_max_retries = 3
+
+# Request environment key where the Swift cache object is stored. When
+# auth_token middleware is deployed with a Swift cache, use this option to have
+# the middleware share a caching backend with swift. Otherwise, use the
+# ``memcached_servers`` option instead. (string value)
+#cache = <None>
+
+# Required if identity server requires client certificate (string value)
+#certfile = <None>
+
+# Required if identity server requires client certificate (string value)
+#keyfile = <None>
+
+# A PEM encoded Certificate Authority to use when verifying HTTPs connections.
+# Defaults to system CAs. (string value)
+#cafile = <None>
+
+# Verify HTTPS connections. (boolean value)
+#insecure = false
+
+# The region in which the identity server can be found. (string value)
+#region_name = <None>
+
+# DEPRECATED: Directory used to cache files related to PKI tokens. This option
+# has been deprecated in the Ocata release and will be removed in the P
+# release. (string value)
+# This option is deprecated for removal since Ocata.
+# Its value may be silently ignored in the future.
+# Reason: PKI token format is no longer supported.
+#signing_dir = <None>
+
+# Optionally specify a list of memcached server(s) to use for caching. If left
+# undefined, tokens will instead be cached in-process. (list value)
+# Deprecated group/name - [keystone_authtoken]/memcache_servers
+#memcached_servers = <None>
+
+# In order to prevent excessive effort spent validating tokens, the middleware
+# caches previously-seen tokens for a configurable duration (in seconds). Set
+# to -1 to disable caching completely. (integer value)
+#token_cache_time = 300
+
+# DEPRECATED: Determines the frequency at which the list of revoked tokens is
+# retrieved from the Identity service (in seconds). A high number of revocation
+# events combined with a low cache duration may significantly reduce
+# performance. Only valid for PKI tokens. This option has been deprecated in
+# the Ocata release and will be removed in the P release. (integer value)
+# This option is deprecated for removal since Ocata.
+# Its value may be silently ignored in the future.
+# Reason: PKI token format is no longer supported.
+#revocation_cache_time = 10
+
+# (Optional) If defined, indicate whether token data should be authenticated or
+# authenticated and encrypted. If MAC, token data is authenticated (with HMAC)
+# in the cache. If ENCRYPT, token data is encrypted and authenticated in the
+# cache. If the value is not one of these options or empty, auth_token will
+# raise an exception on initialization. (string value)
+# Possible values:
+# None - <No description provided>
+# MAC - <No description provided>
+# ENCRYPT - <No description provided>
+#memcache_security_strategy = None
+
+# (Optional, mandatory if memcache_security_strategy is defined) This string is
+# used for key derivation. (string value)
+#memcache_secret_key = <None>
+
+# (Optional) Number of seconds memcached server is considered dead before it is
+# tried again. (integer value)
+#memcache_pool_dead_retry = 300
+
+# (Optional) Maximum total number of open connections to every memcached
+# server. (integer value)
+#memcache_pool_maxsize = 10
+
+# (Optional) Socket timeout in seconds for communicating with a memcached
+# server. (integer value)
+#memcache_pool_socket_timeout = 3
+
+# (Optional) Number of seconds a connection to memcached is held unused in the
+# pool before it is closed. (integer value)
+#memcache_pool_unused_timeout = 60
+
+# (Optional) Number of seconds that an operation will wait to get a memcached
+# client connection from the pool. (integer value)
+#memcache_pool_conn_get_timeout = 10
+
+# (Optional) Use the advanced (eventlet safe) memcached client pool. The
+# advanced pool will only work under python 2.x. (boolean value)
+#memcache_use_advanced_pool = false
+
+# (Optional) Indicate whether to set the X-Service-Catalog header. If False,
+# middleware will not ask for service catalog on token validation and will not
+# set the X-Service-Catalog header. (boolean value)
+#include_service_catalog = true
+
+# Used to control the use and type of token binding. Can be set to: "disabled"
+# to not check token binding. "permissive" (default) to validate binding
+# information if the bind type is of a form known to the server and ignore it
+# if not. "strict" like "permissive" but if the bind type is unknown the token
+# will be rejected. "required" any form of token binding is needed to be
+# allowed. Finally the name of a binding method that must be present in tokens.
+# (string value)
+#enforce_token_bind = permissive
+
+# DEPRECATED: If true, the revocation list will be checked for cached tokens.
+# This requires that PKI tokens are configured on the identity server. (boolean
+# value)
+# This option is deprecated for removal since Ocata.
+# Its value may be silently ignored in the future.
+# Reason: PKI token format is no longer supported.
+#check_revocations_for_cached = false
+
+# DEPRECATED: Hash algorithms to use for hashing PKI tokens. This may be a
+# single algorithm or multiple. The algorithms are those supported by Python
+# standard hashlib.new(). The hashes will be tried in the order given, so put
+# the preferred one first for performance. The result of the first hash will be
+# stored in the cache. This will typically be set to multiple values only while
+# migrating from a less secure algorithm to a more secure one. Once all the old
+# tokens are expired this option should be set to a single value for better
+# performance. (list value)
+# This option is deprecated for removal since Ocata.
+# Its value may be silently ignored in the future.
+# Reason: PKI token format is no longer supported.
+#hash_algorithms = md5
+
+# A choice of roles that must be present in a service token. Service tokens are
+# allowed to request that an expired token can be used and so this check should
+# tightly control that only actual services should be sending this token. Roles
+# here are applied as an ANY check so any role in this list must be present.
+# For backwards compatibility reasons this currently only affects the
+# allow_expired check. (list value)
+#service_token_roles = service
+
+# For backwards compatibility reasons we must let valid service tokens pass
+# that don't pass the service_token_roles check as valid. Setting this true
+# will become the default in a future release and should be enabled if
+# possible. (boolean value)
+#service_token_roles_required = false
+
+# Authentication type to load (string value)
+# Deprecated group/name - [keystone_authtoken]/auth_plugin
+#auth_type = <None>
+
+# Config Section from which to load plugin specific options (string value)
+#auth_section = <None>
+
+
+[logging]
+
+#
+# From shipyard_api
+#
+
+# The default logging level for the root logger. ERROR=40, WARNING=30, INFO=20,
+# DEBUG=10 (integer value)
+#log_level = 10
+
+# The logging levels for named loggers. Use standard representations for
+# logging levels: ERROR. WARN, INFO, DEBUG. Configuration file format:
+# named_log_levels = keystoneauth:INFO,othlgr:WARN (dict value)
+#named_log_levels = keystoneauth:20,keystonemiddleware:20
+
+
+[requests_config]
+
+#
+# From shipyard_api
+#
+
+# Airflow logs retrieval connect timeout (in seconds) (integer value)
+#airflow_log_connect_timeout = 5
+
+# Airflow logs retrieval timeout (in seconds) (integer value)
+#airflow_log_read_timeout = 300
+
+# Deckhand client connect timeout (in seconds) (integer value)
+#deckhand_client_connect_timeout = 5
+
+# Deckhand client timeout (in seconds) for GET, PUT, POST and DELETE request
+# (integer value)
+#deckhand_client_read_timeout = 300
+
+# UCP component validation connect timeout (in seconds) (integer value)
+#validation_connect_timeout = 5
+
+# UCP component validation timeout (in seconds) (integer value)
+#validation_read_timeout = 300
+
+
+[shipyard]
+
+#
+# From shipyard_api
+#
+
+# The service type for the service playing the role of Shipyard. The specified
+# type is used to perform the service lookup in the Keystone service catalog.
+# (string value)
+#service_type = shipyard

generator/config-generator.conf → src/bin/shipyard_airflow/generator/config-generator.conf

@@ -1,5 +1,5 @@
 [DEFAULT]
 output_file = etc/shipyard/shipyard.conf.sample
 wrap_width=79
-namespace = shipyard_airflow
-namespace = keystonemiddleware.auth_token
+namespace = shipyard_api
+namespace = keystonemiddleware.auth_token

generator/policy-generator.conf → src/bin/shipyard_airflow/generator/policy-generator.conf

@@ -1,3 +1,3 @@
 [DEFAULT]
 output_file = etc/shipyard/policy.yaml.sample
-namespace = shipyard_airflow
+namespace = shipyard_api

requirements.txt → src/bin/shipyard_airflow/requirements.txt

@@ -12,27 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+# API requirements
+
 alembic==0.9.5
-arrow==0.10.0  # API and Client
+arrow==0.10.0
 configparser==3.5.0
 falcon==1.2.0
 jsonschema==2.6.0
-keystoneauth1==2.13.0  # API and Client
-keystonemiddleware==4.17.0
-oslo.config==4.11.0
-oslo.policy==1.25.1
+keystoneauth1==3.4.0
+keystonemiddleware==4.21.0
+oslo.config==5.2.0
+oslo.policy==1.33.1
 PasteDeploy==1.5.2
-pbr!=2.1.0,>=2.0.0 # Apache-2.0
 psycopg2==2.7.3.1
 python-dateutil==2.6.1
 python-memcached==1.58
-requests==2.18.4  # API and Client
+requests==2.18.4
+setuptools==39.0.1
 SQLAlchemy==1.1.13
 ulid==1.1
 uwsgi==2.0.15
-
-# Client
-click==6.7
-click-default-group==1.2
-PTable==0.9.2
-pyyaml==3.12

src/bin/shipyard_airflow/setup.py

@@ -0,0 +1,42 @@
+# Copyright 2017 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import setuptools
+
+setuptools.setup(
+    name='shipyard_api',
+    version='0.1a1',
+    description=('Directed acyclic graph controller for Kubernetes and '
+                 'OpenStack control plane life cycle management'),
+    url='https://github.com/att-comdev/shipyard',
+    author='AT&T - AIC UCP Developers',
+    license='Apache 2.0',
+    packages=setuptools.find_packages(),
+    entry_points={
+        'oslo.config.opts':
+            'shipyard_api = shipyard_airflow.conf.opts:list_opts',
+        'oslo.policy.policies':
+            'shipyard_api = shipyard_airflow.policy:list_policies',
+        'console_scripts':
+            'upgrade_db = shipyard_airflow.shipyard_upgrade_db:upgrade_db',
+    },
+    classifiers=[
+        'Intended Audience :: Information Technology',
+        'Intended Audience :: System Administrators',
+        'License :: OSI Approved :: Apache Software License',
+        'Operating System :: POSIX :: Linux',
+        'Programming Language :: Python',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+    ],
+)