airship
/
shipyard
Code Issues Proposed changes

Update Armada Operator 

As part of integration testing, we will need to be able
to read the Armada YAML file from remote server as the
integration work between Armada and DeckHand is still
ongoing at the moment.

This P.S. updates the logic in which the Armada Operator
retrieves the Armada YAML

We are aligning the directory and file names with what is
in CI/CD. This will require minor changes to the DryDock
Operator as well.

Change-Id: I26c7c1dc3628662f34fbf7cd5ef916cf34f08243
This commit is contained in:
Anthony Lin 2017-11-03 00:19:51 +00:00
parent a5be242c31
commit d9179c8b1a
2 changed files with 41 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

70
shipyard_airflow/plugins/armada_operator.py
View File

@ -13,6 +13,8 @@
# limitations under the License.
import logging
import os
import requests
from urllib.parse import urlparse
from airflow.models import BaseOperator
@ -89,12 +91,12 @@ class ArmadaOperator(BaseOperator):
task_ids='create_armada_client',
dag_id=self.sub_dag_name + '.create_armada_client')
# Retrieve Tiller Information and assign to context 'query'
context['query'] = self.get_tiller_info(context)
# Armada API Call
# Armada Status
if self.action == 'armada_status':
# Retrieve Tiller Information and assign to context 'query'
context['query'] = self.get_tiller_info(context)
self.get_armada_status(context, armada_client)
# Armada Validate
@ -103,16 +105,10 @@ class ArmadaOperator(BaseOperator):
# Armada Apply
elif self.action == 'armada_apply':
# Retrieve Tiller Information and assign to context 'query'
context['query'] = self.get_tiller_info(context)
self.armada_apply(context, armada_client)
# Armada Get Releases
elif self.action == 'armada_get_releases':
# Retrieve Tiller Information and assign to context 'query'
context['query'] = self.get_tiller_info(context)
self.armada_get_releases(context, armada_client)
else:
@ -189,17 +185,8 @@ class ArmadaOperator(BaseOperator):
armada_manifest = None
valid_armada_yaml = {}
# At this point in time, testing of the operator is being done by
# reading the armada.yaml file on airflow and feeding it to Armada as
# a string. We will assume that the file name is fixed and will always
# be 'armada_site.yaml'. This will change in the near future when
# Armada is integrated with DeckHand.
yaml_path = '/usr/local/airflow/plugins/armada_site.yaml'
# TODO: We will implement the new approach when Armada and DeckHand
# integration is completed.
with open(yaml_path, 'r') as armada_yaml:
armada_manifest = armada_yaml.read()
# Retrieve Armada Manifest
armada_manifest = self.get_armada_yaml(context)
# Validate armada yaml file
logging.info("Armada Validate")
@ -220,18 +207,8 @@ class ArmadaOperator(BaseOperator):
override_values = []
chart_set = []
# At this point in time, testing of the operator is being done by
# reading the armada.yaml file on airflow and feeding it to Armada as
# a string. We will assume that the file name is fixed and will always
# be 'armada_site.yaml'. This will change in the near future when
# Armada is integrated with DeckHand.
yaml_path = '/usr/local/airflow/plugins/armada_site.yaml'
# TODO: We will implement the new approach when Armada and DeckHand
# integration is completed. Override and chart_set will be considered
# at that time.
with open(yaml_path, 'r') as armada_yaml:
armada_manifest = armada_yaml.read()
# Retrieve Armada Manifest
armada_manifest = self.get_armada_yaml(context)
# Execute Armada Apply to install the helm charts in sequence
logging.info("Armada Apply")
@ -266,6 +243,35 @@ class ArmadaOperator(BaseOperator):
else:
raise AirflowException("Failed to retrieve Armada Releases")
def get_armada_yaml(self, context):
# Initialize Variables
genesis_node_ip = None
# At this point in time, testing of the operator is being done by
# retrieving the armada.yaml from the nginx container on the Genesis
# node and feeding it to Armada as a string. We will assume that the
# file name is fixed and will always be 'armada_site.yaml'. This file
# will always be under the osh directory. This will change in the near
# future when Armada is integrated with DeckHand.
genesis_node_ip = context['query'].get('tiller_host')
# Form Endpoint
schema = 'http://'
nginx_host_port = genesis_node_ip + ':6880'
armada_yaml = 'osh/armada.yaml'
design_ref = os.path.join(schema, nginx_host_port, armada_yaml)
logging.info("Armada YAML will be retrieved from %s", design_ref)
# TODO: We will implement the new approach when Armada and DeckHand
# integration is completed.
try:
armada_manifest = requests.get(design_ref).text
except requests.exceptions.RequestException as e:
raise AirflowException(e)
return armada_manifest
class ArmadaOperatorPlugin(AirflowPlugin):
name = 'armada_operator_plugin'

6
shipyard_airflow/plugins/drydock_operators.py
View File

@ -111,8 +111,8 @@ class DryDockOperator(BaseOperator):
# Artifactory or Deckhand is able to host the YAMLs
# NOTE: Testing of the Operator was performed with a nginx server
# on the genesis host that is listening on port 6880. The name of
# the YAML file is 'drydock.yaml'. We will use this assumption
# for now.
# the YAML file is 'drydock.yaml' under the ucp directory. We will
# use this assumption for now.
# TODO: This logic will be updated once DeckHand is integrated
# with DryDock
logging.info("Retrieving information of Tiller pod to obtain Genesis "
@ -126,7 +126,7 @@ class DryDockOperator(BaseOperator):
if genesis_node_ip:
schema = 'http://'
nginx_host_port = genesis_node_ip + ':6880'
drydock_yaml = 'drydock.yaml'
drydock_yaml = 'ucp/drydock.yaml'
self.design_ref = os.path.join(schema,
nginx_host_port,
drydock_yaml)