Shipyard-270 - Rendered document schema validation check
The story requires Shipyard to get the rendered document from Deckhand before proceeding with the rest of the workflow. This is to ensure that the schema validation at Deckhand is good and to fail early if need be. This p.s. attempts to make use of the deckhand client to get the list of rendered documents. Note that we will need to install the DeckHand client libraries in the Airflow image in order for this to happen. Change-Id: I22f4165be101edfa4cef9bead6dd3d3765146c6e
This commit is contained in:
parent
5cc0b5b986
commit
8d486b22db
@ -66,8 +66,10 @@ RUN pip3 install -r /tmp/requirements.txt
|
|||||||
# https://github.com/puckel/docker-airflow/issues/77
|
# https://github.com/puckel/docker-airflow/issues/77
|
||||||
RUN pip3 uninstall -y snakebite || true
|
RUN pip3 uninstall -y snakebite || true
|
||||||
|
|
||||||
RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner
|
# Install Armada, DeckHand and DryDock Client Libraries
|
||||||
RUN pip3 install -e git://github.com/att-comdev/armada.git#egg=armada
|
RUN pip3 install -e git://github.com/att-comdev/armada.git#egg=armada
|
||||||
|
RUN pip3 install -e git://github.com/att-comdev/deckhand.git#egg=deckhand
|
||||||
|
RUN pip3 install -e git://github.com/att-comdev/drydock.git#egg=drydock_provisioner
|
||||||
|
|
||||||
# Create airflow user
|
# Create airflow user
|
||||||
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow
|
RUN useradd -ms /bin/bash -d ${AIRFLOW_HOME} airflow
|
||||||
|
@ -38,4 +38,15 @@ def get_design_deckhand(parent_dag_name, child_dag_name, args):
|
|||||||
sub_dag_name=child_dag_name,
|
sub_dag_name=child_dag_name,
|
||||||
dag=dag)
|
dag=dag)
|
||||||
|
|
||||||
|
shipyard_retrieve_rendered_doc = DeckhandOperator(
|
||||||
|
task_id='shipyard_retrieve_rendered_doc',
|
||||||
|
shipyard_conf=config_path,
|
||||||
|
action='shipyard_retrieve_rendered_doc',
|
||||||
|
main_dag_name=parent_dag_name,
|
||||||
|
sub_dag_name=child_dag_name,
|
||||||
|
dag=dag)
|
||||||
|
|
||||||
|
# Define dependencies
|
||||||
|
shipyard_retrieve_rendered_doc.set_upstream(deckhand_design)
|
||||||
|
|
||||||
return dag
|
return dag
|
||||||
|
@ -12,6 +12,7 @@
|
|||||||
# See the License for the specific language governing permissions and
|
# See the License for the specific language governing permissions and
|
||||||
# limitations under the License.
|
# limitations under the License.
|
||||||
|
|
||||||
|
import configparser
|
||||||
import logging
|
import logging
|
||||||
import os
|
import os
|
||||||
import requests
|
import requests
|
||||||
@ -21,7 +22,10 @@ from airflow.models import BaseOperator
|
|||||||
from airflow.utils.decorators import apply_defaults
|
from airflow.utils.decorators import apply_defaults
|
||||||
from airflow.plugins_manager import AirflowPlugin
|
from airflow.plugins_manager import AirflowPlugin
|
||||||
from airflow.exceptions import AirflowException
|
from airflow.exceptions import AirflowException
|
||||||
|
from keystoneauth1.identity import v3 as keystone_v3
|
||||||
|
from keystoneauth1 import session as keystone_session
|
||||||
|
|
||||||
|
from deckhand.client import deckhand_client
|
||||||
from service_endpoint import ucp_service_endpoint
|
from service_endpoint import ucp_service_endpoint
|
||||||
from service_token import shipyard_service_token
|
from service_token import shipyard_service_token
|
||||||
|
|
||||||
@ -57,6 +61,7 @@ class DeckhandOperator(BaseOperator):
|
|||||||
# Initialize Variables
|
# Initialize Variables
|
||||||
deckhand_design_version = None
|
deckhand_design_version = None
|
||||||
redeploy_server = None
|
redeploy_server = None
|
||||||
|
revision_id = None
|
||||||
|
|
||||||
# Define task_instance
|
# Define task_instance
|
||||||
task_instance = context['task_instance']
|
task_instance = context['task_instance']
|
||||||
@ -100,20 +105,33 @@ class DeckhandOperator(BaseOperator):
|
|||||||
else:
|
else:
|
||||||
raise AirflowException('Failed to retrieve revision ID!')
|
raise AirflowException('Failed to retrieve revision ID!')
|
||||||
|
|
||||||
# Validate Design using DeckHand
|
|
||||||
elif self.action == 'deckhand_validate_site_design':
|
|
||||||
# Retrieve revision_id from xcom
|
# Retrieve revision_id from xcom
|
||||||
# Note that in the case of 'deploy_site', the dag_id will
|
# Note that in the case of 'deploy_site', the dag_id will
|
||||||
# be 'deploy_site.deckhand_get_design_version' for the
|
# be 'deploy_site.deckhand_get_design_version' for the
|
||||||
# 'deckhand_get_design_version' task. We need to extract
|
# 'deckhand_get_design_version' task. We need to extract
|
||||||
# the xcom value from it in order to get the value of the
|
# the xcom value from it in order to get the value of the
|
||||||
# last committed revision ID
|
# last committed revision ID
|
||||||
context['revision_id'] = task_instance.xcom_pull(
|
revision_id = task_instance.xcom_pull(
|
||||||
task_ids='deckhand_get_design_version',
|
task_ids='deckhand_get_design_version',
|
||||||
dag_id=self.main_dag_name + '.deckhand_get_design_version')
|
dag_id=self.main_dag_name + '.deckhand_get_design_version')
|
||||||
|
|
||||||
logging.info("Revision ID is %d", context['revision_id'])
|
logging.info("Revision ID is %d", revision_id)
|
||||||
self.deckhand_validate_site(context)
|
|
||||||
|
# Retrieve Rendered Document from DeckHand
|
||||||
|
if self.action == 'shipyard_retrieve_rendered_doc':
|
||||||
|
if revision_id:
|
||||||
|
self.retrieve_rendered_doc(context,
|
||||||
|
revision_id=revision_id)
|
||||||
|
else:
|
||||||
|
raise AirflowException('Invalid revision ID!')
|
||||||
|
|
||||||
|
# Validate Design using DeckHand
|
||||||
|
elif self.action == 'deckhand_validate_site_design':
|
||||||
|
if revision_id:
|
||||||
|
self.deckhand_validate_site(context,
|
||||||
|
revision_id=revision_id)
|
||||||
|
else:
|
||||||
|
raise AirflowException('Invalid revision ID!')
|
||||||
|
|
||||||
# No action to perform
|
# No action to perform
|
||||||
else:
|
else:
|
||||||
@ -160,14 +178,14 @@ class DeckhandOperator(BaseOperator):
|
|||||||
raise AirflowException("Failed to retrieve committed revision!")
|
raise AirflowException("Failed to retrieve committed revision!")
|
||||||
|
|
||||||
@shipyard_service_token
|
@shipyard_service_token
|
||||||
def deckhand_validate_site(self, context):
|
def deckhand_validate_site(self, context, revision_id):
|
||||||
# Retrieve Keystone Token and assign to X-Auth-Token Header
|
# Retrieve Keystone Token and assign to X-Auth-Token Header
|
||||||
x_auth_token = {"X-Auth-Token": context['svc_token']}
|
x_auth_token = {"X-Auth-Token": context['svc_token']}
|
||||||
|
|
||||||
# Form Validation Endpoint
|
# Form Validation Endpoint
|
||||||
validation_endpoint = os.path.join(context['svc_endpoint'],
|
validation_endpoint = os.path.join(context['svc_endpoint'],
|
||||||
'revisions',
|
'revisions',
|
||||||
str(context['revision_id']),
|
str(revision_id),
|
||||||
'validations')
|
'validations')
|
||||||
logging.info(validation_endpoint)
|
logging.info(validation_endpoint)
|
||||||
|
|
||||||
@ -202,6 +220,45 @@ class DeckhandOperator(BaseOperator):
|
|||||||
else:
|
else:
|
||||||
raise AirflowException("DeckHand Site Design Validation Failed!")
|
raise AirflowException("DeckHand Site Design Validation Failed!")
|
||||||
|
|
||||||
|
def retrieve_rendered_doc(self, context, revision_id):
|
||||||
|
|
||||||
|
# Initialize Variables
|
||||||
|
auth = None
|
||||||
|
keystone_auth = {}
|
||||||
|
rendered_doc = []
|
||||||
|
sess = None
|
||||||
|
|
||||||
|
# Read and parse shiyard.conf
|
||||||
|
config = configparser.ConfigParser()
|
||||||
|
config.read(self.shipyard_conf)
|
||||||
|
|
||||||
|
# Construct Session Argument
|
||||||
|
for attr in ('auth_url', 'password', 'project_domain_name',
|
||||||
|
'project_name', 'username', 'user_domain_name'):
|
||||||
|
keystone_auth[attr] = config.get('keystone_authtoken', attr)
|
||||||
|
|
||||||
|
# Set up keystone session
|
||||||
|
auth = keystone_v3.Password(**keystone_auth)
|
||||||
|
sess = keystone_session.Session(auth=auth)
|
||||||
|
|
||||||
|
logging.info("Setting up DeckHand Client...")
|
||||||
|
|
||||||
|
# Set up DeckHand Client
|
||||||
|
deckhandclient = deckhand_client.Client(session=sess)
|
||||||
|
|
||||||
|
logging.info("Retrieving Rendered Document...")
|
||||||
|
|
||||||
|
# Retrieve Rendered Document
|
||||||
|
try:
|
||||||
|
rendered_doc = deckhandclient.revisions.documents(revision_id,
|
||||||
|
rendered=True)
|
||||||
|
|
||||||
|
logging.info("Successfully Retrieved Rendered Document")
|
||||||
|
logging.info(rendered_doc)
|
||||||
|
|
||||||
|
except:
|
||||||
|
raise AirflowException("Failed to Retrieve Rendered Document!")
|
||||||
|
|
||||||
|
|
||||||
class DeckhandOperatorPlugin(AirflowPlugin):
|
class DeckhandOperatorPlugin(AirflowPlugin):
|
||||||
name = 'deckhand_operator_plugin'
|
name = 'deckhand_operator_plugin'
|
||||||
|
Loading…
Reference in New Issue
Block a user