Promenade validateDesign for Shipyard

SHIPYARD-342: Shipyard Integration with Promenade to Validate Design

Calls Promenade validateDesign API to validate site design.

Change-Id: Ia763983ed9857d4e5b13cfb11d3654e75e6578a4
changes/87/569187/1
One-Fine-Day 5 years ago committed by Aaron Sheffield
parent 676cfff124
commit aca1b60f22
  1. 2
      charts/shipyard/values.yaml
  2. 12
      docs/source/_static/shipyard.conf.sample
  3. 12
      src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample
  4. 23
      src/bin/shipyard_airflow/shipyard_airflow/conf/config.py
  5. 4
      src/bin/shipyard_airflow/shipyard_airflow/control/helpers/configdocs_helper.py
  6. 2
      src/bin/shipyard_airflow/shipyard_airflow/control/service_endpoints.py
  7. 9
      src/bin/shipyard_airflow/shipyard_airflow/dags/validate_site_design.py
  8. 44
      src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_base_operator.py
  9. 105
      src/bin/shipyard_airflow/shipyard_airflow/plugins/promenade_validate_site_design.py
  10. 5
      src/bin/shipyard_airflow/shipyard_airflow/plugins/service_token.py
  11. 17
      src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_base_operator.py
  12. 17
      src/bin/shipyard_airflow/shipyard_airflow/plugins/ucp_preflight_check_operator.py
  13. 4
      src/bin/shipyard_airflow/tests/unit/control/test.conf
  14. 2
      tools/resources/shipyard.conf

@ -373,6 +373,8 @@ conf:
destroy_node_query_interval: 30
destroy_node_task_timeout: 900
cluster_join_check_backoff_time: 120
promenade:
service_type: kubernetesprovisioner
keystone_authtoken:
delay_auth_decision: true
auth_type: password

@ -294,6 +294,18 @@
#named_log_levels = keystoneauth:20,keystonemiddleware:20
[promenade]
#
# From shipyard_api
#
# The service type for the service playing the role of Promenade. The specified
# type is used to perform the service lookup in the Keystone service catalog.
# (string value)
#service_type = kubernetesprovisioner
[requests_config]
#

@ -294,6 +294,18 @@
#named_log_levels = keystoneauth:20,keystonemiddleware:20
[promenade]
#
# From shipyard_api
#
# The service type for the service playing the role of Promenade. The specified
# type is used to perform the service lookup in the Keystone service catalog.
# (string value)
#service_type = kubernetesprovisioner
[requests_config]
#

@ -89,7 +89,7 @@ SECTIONS = [
),
ConfigSection(
name='shipyard',
title='Shipyard connection info',
title='Shipyard Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -104,7 +104,7 @@ SECTIONS = [
),
ConfigSection(
name='deckhand',
title='Deckhand connection info',
title='Deckhand Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -119,7 +119,7 @@ SECTIONS = [
),
ConfigSection(
name='armada',
title='Armada connection info',
title='Armada Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -134,7 +134,7 @@ SECTIONS = [
),
ConfigSection(
name='drydock',
title='Drydock connection info',
title='Drydock Connection Info',
options=[
cfg.StrOpt(
'service_type',
@ -147,6 +147,21 @@ SECTIONS = [
),
]
),
ConfigSection(
name='promenade',
title='Promenade Connection Info',
options=[
cfg.StrOpt(
'service_type',
default='kubernetesprovisioner',
help=(
'The service type for the service playing the role '
'of Promenade. The specified type is used to perform '
'the service lookup in the Keystone service catalog.'
)
),
]
),
ConfigSection(
name='requests_config',
title='Requests Configuration',

@ -732,6 +732,10 @@ def _get_validation_endpoints():
'name': 'Armada',
'url': val_ep.format(get_endpoint(Endpoints.ARMADA))
},
{
'name': 'Promenade',
'url': val_ep.format(get_endpoint(Endpoints.PROMENADE))
}
]

@ -39,6 +39,7 @@ class Endpoints(enum.Enum):
DRYDOCK = 'drydock'
ARMADA = 'armada'
DECKHAND = 'deckhand'
PROMENADE = 'promenade'
def _get_service_type(endpoint):
@ -57,6 +58,7 @@ def _get_service_type(endpoint):
Endpoints.DRYDOCK: CONF.drydock.service_type,
Endpoints.ARMADA: CONF.armada.service_type,
Endpoints.DECKHAND: CONF.deckhand.service_type,
Endpoints.PROMENADE: CONF.promenade.service_type
}
return endpoint_values.get(endpoint)
raise AppError(

@ -16,6 +16,7 @@ from airflow.models import DAG
from airflow.operators import ArmadaValidateDesignOperator
from airflow.operators import DeckhandValidateSiteDesignOperator
from airflow.operators import DrydockValidateDesignOperator
from airflow.operators import PromenadeValidateSiteDesignOperator
from config_path import config_path
@ -52,4 +53,12 @@ def validate_site_design(parent_dag_name, child_dag_name, args):
retries=3,
dag=dag)
promenade_validate_docs = PromenadeValidateSiteDesignOperator(
task_id='promenade_validate_site_design',
shipyard_conf=config_path,
main_dag_name=parent_dag_name,
sub_dag_name=child_dag_name,
retries=3,
dag=dag)
return dag

@ -12,14 +12,26 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import logging
import os
from airflow.utils.decorators import apply_defaults
from airflow.plugins_manager import AirflowPlugin
from airflow.exceptions import AirflowException
from service_endpoint import ucp_service_endpoint
from service_token import shipyard_service_token
from ucp_base_operator import UcpBaseOperator
try:
from service_endpoint import ucp_service_endpoint
except ImportError:
from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
try:
from service_token import shipyard_service_token
except ImportError:
from shipyard_airflow.plugins.service_token import shipyard_service_token
try:
from ucp_base_operator import UcpBaseOperator
except ImportError:
from shipyard_airflow.plugins.ucp_base_operator import UcpBaseOperator
LOG = logging.getLogger(__name__)
@ -35,6 +47,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
@apply_defaults
def __init__(self,
deckhand_design_ref=None,
deckhand_svc_type='deckhand',
promenade_svc_endpoint=None,
promenade_svc_type='kubernetesprovisioner',
redeploy_server=None,
@ -42,6 +56,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
*args, **kwargs):
"""Initialization of PromenadeBaseOperator object.
:param deckhand_design_ref: A URI reference to the design documents
:param deckhand_svc_type: Deckhand Service Type
:param promenade_svc_endpoint: Promenade Service Endpoint
:param promenade_svc_type: Promenade Service Type
:param redeploy_server: Server to be redeployed
@ -55,6 +71,8 @@ class PromenadeBaseOperator(UcpBaseOperator):
pod_selector_pattern=[{'pod_pattern': 'promenade-api',
'container': 'promenade-api'}],
*args, **kwargs)
self.deckhand_design_ref = deckhand_design_ref
self.deckhand_svc_type = deckhand_svc_type
self.promenade_svc_endpoint = promenade_svc_endpoint
self.promenade_svc_type = promenade_svc_type
self.redeploy_server = redeploy_server
@ -85,6 +103,26 @@ class PromenadeBaseOperator(UcpBaseOperator):
LOG.info("Promenade endpoint is %s", self.promenade_svc_endpoint)
# Retrieve Deckhand Endpoint Information
deckhand_svc_endpoint = ucp_service_endpoint(
self, svc_type=self.deckhand_svc_type)
LOG.info("Deckhand endpoint is %s", deckhand_svc_endpoint)
# Form Deckhand Design Reference Path
# This URL will be used to retrieve the Site Design YAMLs
deckhand_path = "deckhand+" + deckhand_svc_endpoint
self.deckhand_design_ref = os.path.join(deckhand_path,
"revisions",
str(self.revision_id),
"rendered-documents")
if self.deckhand_design_ref:
LOG.info("Design YAMLs will be retrieved from %s",
self.deckhand_design_ref)
else:
raise AirflowException("Unable to Retrieve Deckhand Revision "
"%d!" % self.revision_id)
class PromenadeBaseOperatorPlugin(AirflowPlugin):

@ -0,0 +1,105 @@
# Copyright 2018 AT&T Intellectual Property. All other rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import json
import logging
import os
import requests
from airflow.exceptions import AirflowException
from airflow.plugins_manager import AirflowPlugin
try:
from promenade_base_operator import PromenadeBaseOperator
except ImportError:
from shipyard_airflow.plugins.promenade_base_operator import (
PromenadeBaseOperator
)
LOG = logging.getLogger(__name__)
class PromenadeValidateSiteDesignOperator(PromenadeBaseOperator):
"""
Promenade Validate Site Design Operator
This operator will trigger promenade to invoke the validateDesign
Promenade API
"""
def do_execute(self):
LOG.info("Validating site design...")
# Form Validation Endpoint
validation_endpoint = os.path.join(self.promenade_svc_endpoint,
'validatedesign')
LOG.info("Validation Endpoint is %s", validation_endpoint)
# Define Headers and Payload
headers = {
'Content-Type': 'application/json',
'X-Auth-Token': self.svc_token
}
payload = {
'rel': "design",
'href': self.deckhand_design_ref,
'type': "application/x-yaml"
}
# Requests Promenade to validate site design
LOG.info("Waiting for Promenade to validate site design...")
try:
design_validate_response = requests.post(validation_endpoint,
headers=headers,
data=json.dumps(payload))
except requests.exceptions.RequestException as e:
raise AirflowException(e)
# Convert response to string
validate_site_design = design_validate_response.text
# Print response
LOG.info("Retrieving Promenade validate site design response...")
try:
validate_site_design_dict = json.loads(validate_site_design)
LOG.info(validate_site_design_dict)
except json.JSONDecodeError as e:
raise AirflowException(e)
# Check if site design is valid
status = validate_site_design_dict.get('status',
'unspecified')
if status.lower() == 'success':
LOG.info("Promenade Site Design has been successfully validated")
else:
# Dump logs from Promenade pods
self.get_k8s_logs()
raise AirflowException("Promenade Site Design Validation Failed "
"with status: {}!".format(status))
class PromenadeValidateSiteDesignOperatorPlugin(AirflowPlugin):
"""Creates PromenadeValidateSiteDesginOperator in Airflow."""
name = 'promenade_validate_site_design_operator'
operators = [PromenadeValidateSiteDesignOperator]

@ -18,7 +18,10 @@ import time
from airflow.exceptions import AirflowException
from service_session import ucp_keystone_session
try:
from service_session import ucp_keystone_session
except ImportError:
from shipyard_airflow.plugins.service_session import ucp_keystone_session
def shipyard_service_token(func):

@ -20,9 +20,20 @@ from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
from get_k8s_logs import get_pod_logs
from get_k8s_logs import K8sLoggingException
from xcom_puller import XcomPuller
try:
from get_k8s_logs import get_pod_logs
except ImportError:
from shipyard_airflow.plugins.get_k8s_logs import get_pod_logs
try:
from get_k8s_logs import K8sLoggingException
except ImportError:
from shipyard_airflow.plugins.get_k8s_logs import K8sLoggingException
try:
from xcom_puller import XcomPuller
except ImportError:
from shipyard_airflow.plugins.xcom_puller import XcomPuller
LOG = logging.getLogger(__name__)

@ -20,9 +20,20 @@ from airflow.models import BaseOperator
from airflow.plugins_manager import AirflowPlugin
from airflow.utils.decorators import apply_defaults
from service_endpoint import ucp_service_endpoint
from xcom_puller import XcomPuller
from xcom_pusher import XcomPusher
try:
from service_endpoint import ucp_service_endpoint
except ImportError:
from shipyard_airflow.plugins.service_endpoint import ucp_service_endpoint
try:
from xcom_puller import XcomPuller
except ImportError:
from shipyard_airflow.plugins.xcom_puller import XcomPuller
try:
from xcom_pusher import XcomPusher
except ImportError:
from shipyard_airflow.plugins.xcom_pusher import XcomPusher
LOG = logging.getLogger(__name__)

@ -29,6 +29,10 @@ project_domain_name = default
project_name = service
user_domain_name = default
username = shipyard
[k8s_logs]
ucp_namespace = ucp
[promenade]
service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300

@ -42,6 +42,8 @@ user_domain_name = default
username = shipyard
[k8s_logs]
ucp_namespace = ucp
[promenade]
service_type = kubernetesprovisioner
[requests_config]
airflow_log_connect_timeout = 5
airflow_log_read_timeout = 300

Loading…
Cancel
Save