Update DryDock Operator
There is a need to update the DryDock Operator as DryDock has been integrated with DeckHand. The YAMLs should be retrieved from DeckHand rather than from nginx server. This P.S. updates the DryDock Operator to account for the recent changes. It also reintroduce the workflow for DeckHand in deploy_site dag. Change-Id: I4f00ad198da1cbab426394d56f7eb90ef260e0a1
This commit is contained in:
Anthony Lin
parent
b6d7af07fa
commit
d1bf3f0e69
|
|
@ -20,8 +20,10 @@ from airflow.operators import ConcurrencyCheckOperator
|
|||
from airflow.operators.python_operator import PythonOperator
|
||||
from airflow.operators.subdag_operator import SubDagOperator
|
||||
|
||||
from deckhand_get_design import get_design_deckhand
|
||||
from armada_deploy_site import deploy_site_armada
|
||||
from drydock_deploy_site import deploy_site_drydock
|
||||
from validate_site_design import validate_site_design
|
||||
"""
|
||||
NOTE: We are currently in the process of reviewing and merging patch sets
|
||||
from various UCP components that are needed for the 'deploy_site' dag to
|
||||
|
|
@ -36,8 +38,10 @@ along with the integration testing and UCP code reviews/merge.
|
|||
|
||||
ARMADA_BUILD_DAG_NAME = 'armada_build'
|
||||
DAG_CONCURRENCY_CHECK_DAG_NAME = 'dag_concurrency_check'
|
||||
DECKHAND_GET_DESIGN_VERSION = 'deckhand_get_design_version'
|
||||
DRYDOCK_BUILD_DAG_NAME = 'drydock_build'
|
||||
PARENT_DAG_NAME = 'deploy_site'
|
||||
VALIDATE_SITE_DESIGN_DAG_NAME = 'validate_site_design'
|
||||
|
||||
default_args = {
|
||||
'owner': 'airflow',
|
||||
|
|
@ -73,6 +77,20 @@ concurrency_check = ConcurrencyCheckOperator(
|
|||
on_failure_callback=failure_handlers.step_failure_handler,
|
||||
dag=dag)
|
||||
|
||||
get_design_version = SubDagOperator(
|
||||
subdag=get_design_deckhand(
|
||||
PARENT_DAG_NAME, DECKHAND_GET_DESIGN_VERSION, args=default_args),
|
||||
task_id=DECKHAND_GET_DESIGN_VERSION,
|
||||
on_failure_callback=failure_handlers.step_failure_handler,
|
||||
dag=dag)
|
||||
|
||||
validate_site_design = SubDagOperator(
|
||||
subdag=validate_site_design(
|
||||
PARENT_DAG_NAME, VALIDATE_SITE_DESIGN_DAG_NAME, args=default_args),
|
||||
task_id=VALIDATE_SITE_DESIGN_DAG_NAME,
|
||||
on_failure_callback=failure_handlers.step_failure_handler,
|
||||
dag=dag)
|
||||
|
||||
drydock_build = SubDagOperator(
|
||||
subdag=deploy_site_drydock(
|
||||
PARENT_DAG_NAME, DRYDOCK_BUILD_DAG_NAME, args=default_args),
|
||||
|
|
@ -89,5 +107,7 @@ armada_build = SubDagOperator(
|
|||
|
||||
# DAG Wiring
|
||||
concurrency_check.set_upstream(action_xcom)
|
||||
drydock_build.set_upstream(concurrency_check)
|
||||
get_design_version.set_upstream(concurrency_check)
|
||||
validate_site_design.set_upstream(get_design_version)
|
||||
drydock_build.set_upstream(validate_site_design)
|
||||
armada_build.set_upstream(drydock_build)
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ from airflow.utils.decorators import apply_defaults
|
|||
|
||||
import drydock_provisioner.drydock_client.client as client
|
||||
import drydock_provisioner.drydock_client.session as session
|
||||
from get_k8s_pod_port_ip import get_pod_port_ip
|
||||
from service_endpoint import ucp_service_endpoint
|
||||
from service_token import shipyard_service_token
|
||||
|
||||
|
|
@ -104,37 +103,14 @@ class DryDockOperator(BaseOperator):
|
|||
task_ids='create_drydock_client',
|
||||
dag_id=self.sub_dag_name + '.create_drydock_client')
|
||||
|
||||
# Based on the current logic used in CI/CD pipeline, we will
|
||||
# point to the nginx server on the genesis host which is hosting
|
||||
# the drydock YAMLs. That will be the URL for 'design_ref'
|
||||
# NOTE: This is a temporary hack and will be removed once
|
||||
# Artifactory or Deckhand is able to host the YAMLs
|
||||
# NOTE: Testing of the Operator was performed with a nginx server
|
||||
# on the genesis host that is listening on port 6880. The name of
|
||||
# the YAML file is 'drydock.yaml' under the ucp directory. We will
|
||||
# use this assumption for now.
|
||||
# TODO: This logic will be updated once DeckHand is integrated
|
||||
# with DryDock
|
||||
logging.info("Retrieving information of Tiller pod to obtain Genesis "
|
||||
"Node IP...")
|
||||
|
||||
# Retrieve Genesis Node IP
|
||||
genesis_node_ip = self.get_genesis_node_ip(context)
|
||||
|
||||
# Form the URL path that we will use to retrieve the DryDock YAMLs
|
||||
# Return Exceptions if we are not able to retrieve the URL
|
||||
if genesis_node_ip:
|
||||
schema = 'http://'
|
||||
nginx_host_port = genesis_node_ip + ':6880'
|
||||
drydock_yaml = 'ucp/drydock.yaml'
|
||||
self.design_ref = os.path.join(schema,
|
||||
nginx_host_port,
|
||||
drydock_yaml)
|
||||
# Retrieve Deckhand Design Reference
|
||||
self.design_ref = self.get_deckhand_design_ref(context)
|
||||
|
||||
if self.design_ref:
|
||||
logging.info("Drydock YAMLs will be retrieved from %s",
|
||||
self.design_ref)
|
||||
else:
|
||||
raise AirflowException("Unable to Retrieve Genesis Node IP!")
|
||||
raise AirflowException("Unable to Retrieve Design Reference!")
|
||||
|
||||
# Read shipyard.conf
|
||||
config = configparser.ConfigParser()
|
||||
|
|
@ -312,17 +288,32 @@ class DryDockOperator(BaseOperator):
|
|||
else:
|
||||
raise AirflowException("Failed to execute/complete task!")
|
||||
|
||||
@get_pod_port_ip('tiller')
|
||||
def get_genesis_node_ip(self, context, *args):
|
||||
def get_deckhand_design_ref(self, context):
|
||||
|
||||
# Get IP and port information of Pods from context
|
||||
k8s_pods_ip_port = context['pods_ip_port']
|
||||
# Retrieve DeckHand Endpoint Information
|
||||
context['svc_type'] = 'deckhand'
|
||||
context['svc_endpoint'] = ucp_service_endpoint(self, context)
|
||||
logging.info("Deckhand endpoint is %s", context['svc_endpoint'])
|
||||
|
||||
# Tiller will take the IP of the Genesis Node. Retrieve
|
||||
# the IP of tiller to get the IP of the Genesis Node
|
||||
genesis_ip = k8s_pods_ip_port['tiller'].get('ip')
|
||||
# Retrieve revision_id from xcom
|
||||
# Note that in the case of 'deploy_site', the dag_id will be
|
||||
# 'deploy_site.deckhand_get_design_version.deckhand_get_design_version'
|
||||
# for the 'deckhand_get_design_version' task. We need to extract
|
||||
# the xcom value from it in order to get the value of the last
|
||||
# committed revision ID
|
||||
committed_revision_id = context['task_instance'].xcom_pull(
|
||||
task_ids='deckhand_get_design_version',
|
||||
dag_id=self.main_dag_name + '.deckhand_get_design_version' * 2)
|
||||
|
||||
return genesis_ip
|
||||
# Form DeckHand Design Reference Path that we will use to retrieve
|
||||
# the DryDock YAMLs
|
||||
deckhand_path = "deckhand+" + context['svc_endpoint']
|
||||
deckhand_design_ref = os.path.join(deckhand_path,
|
||||
"revisions",
|
||||
str(committed_revision_id),
|
||||
"rendered-documents")
|
||||
|
||||
return deckhand_design_ref
|
||||
|
||||
|
||||
class DryDockClientPlugin(AirflowPlugin):
|
||||
|
|
|
|||
Loading…
Reference in New Issue