Add validation phases

This patch introduces ability to validate phases using kubeval.
Appropriate functionality was embedded into phase/plan validate
command.

Change-Id: I1e1ccae2b7e4948bdc97a199c96c07a3eb7292b2
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
Relates-To: #503
Closes: #2
Closes: #19

manifests/function/validator/kustomization.yaml

@@ -0,0 +1,2 @@
+generators:
+  - template.yaml

manifests/function/validator/template.yaml

@@ -0,0 +1,26 @@
+apiVersion: airshipit.org/v1alpha1
+kind: Templater
+metadata:
+  name: validator-config-patch-template
+  annotations:
+    config.kubernetes.io/function: |
+      container:
+        image: quay.io/airshipit/templater:v2
+        envs:
+        - AIRSHIPCTL_CURRENT_PHASE
+        - AIRSHIPCTL_CURRENT_PLAN
+template: |
+  {{- $currentPhase := env "AIRSHIPCTL_CURRENT_PHASE" }}
+  {{- $currentPlan := env "AIRSHIPCTL_CURRENT_PLAN" }}
+  apiVersion: builtin
+  kind: PatchStrategicMergeTransformer
+  metadata:
+    name: smp
+  patches: |-
+    ---
+    apiVersion: airshipit.org/v1alpha1
+    kind: KubevalOptions
+    metadata:
+      name: kubeval-options
+    phaseName: {{ $currentPhase }}
+    planName: {{ $currentPlan }}

manifests/function/workers-capd/workers.yaml

@@ -27,7 +27,7 @@ spec:
   clusterName: "target-cluster"
   replicas: ${ WORKER_MACHINE_COUNT }
   selector:
-    matchLabels:
+    matchLabels: {}
   template:
     spec:
       clusterName: "target-cluster"

manifests/function/workers-capg/workers.yaml

@@ -7,7 +7,7 @@ spec:
   clusterName: "target-cluster"
   replicas: "${WORKER_MACHINE_COUNT}"
   selector:
-    matchLabels:
+    matchLabels: {}
   template:
     spec:
       clusterName: "target-cluster"

manifests/function/workers-capo/workers.yaml

@@ -7,7 +7,7 @@ spec:
   clusterName: target-cluster
   replicas: 0
   selector:
-    matchLabels: null
+    matchLabels: {}
   template:
     spec:
       bootstrap:

manifests/function/workers-capz/v0.4.8/workers.yaml

@@ -7,7 +7,7 @@ spec:
   clusterName: target-cluster
   replicas: 3
   selector:
-    matchLabels: null
+    matchLabels: {}
   template:
     spec:
       bootstrap:

manifests/function/workers-capz/v0.4.9/workers.yaml

@@ -8,7 +8,7 @@ spec:
   clusterName: target-cluster
   replicas: 3
   selector:
-    matchLabels: null
+    matchLabels: {}
   template:
     spec:
       bootstrap:

manifests/global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml

@@ -177,6 +177,16 @@ spec:
                     name must be unique.
                   type: string
               type: object
+            firmware:
+              description: firmware holds the reference for creating and consuming hardware profiles
+              properties:
+                simultaneousMultithreadingDisabled:
+                  type: boolean
+                sriovEnabled:
+                  type: boolean
+                virtualizationDisabled:
+                  type: boolean
+              type: object
             online:
               description: Should the server be online?
               type: boolean

manifests/phases/executors.yaml

@@ -363,3 +363,31 @@ configRef:
   kind: ConfigMap
   name: kubectl-wait-pods
   apiVersion: v1
+---
+apiVersion: airshipit.org/v1alpha1
+kind: GenericContainer
+metadata:
+  name: document-validation
+  labels:
+    airshipit.org/deploy-k8s: "false"
+spec:
+  type: krm
+  image: quay.io/airshipit/kubeval-validator:latest
+  envVars:
+    - VALIDATOR_PREVENT_CLEANUP # Validator won't cleanup its working directory after finish
+    - VALIDATOR_PLAN_VALIDATION # Validator will not use phase-specific settings for validation
+    - VALIDATOR_REWRITE_SCHEMAS # Validator will rewrite schemas for kubeval if they already exist
+  mounts:
+    - type: bind
+      src: airshipctl/manifests
+      dst: /manifests
+      rw: false
+    - type: bind
+      src: ~/.airship
+      dst: /workdir
+      rw: true
+  hostNetwork: true
+configRef:
+  apiVersion: airshipit.org/v1alpha1
+  kind: KubevalOptions
+  name: kubeval-options

manifests/site/test-site/phases/kustomization.yaml

@@ -2,5 +2,7 @@ resources:
   - ../kubeconfig
   - ../../../phases
   - catalogue.yaml
+  - validation-config.yaml
 transformers:
   - ../../../function/bootstrap/replacements
+  - ../../../function/validator

manifests/site/test-site/phases/validation-config.yaml

@@ -0,0 +1,73 @@
+apiVersion: airshipit.org/v1alpha1
+kind: KubevalOptions
+metadata:
+  name: kubeval-options
+  labels:
+    airshipit.org/deploy-k8s: "false"
+siteConfig:
+  strict: true
+  kubernetesVersion: "1.16.0"
+  ignoreMissingSchemas: false
+planName: AIRSHIPCTL_CURRENT_PLAN
+planConfigs:
+  phasePlan:
+    kindsToSkip:
+      - Clusterctl
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/versions-catalogue.yaml
+      - function/airshipctl-schemas/network-catalogue.yaml
+phaseName: AIRSHIPCTL_CURRENT_PHASE
+phaseConfigs:
+  initinfra-ephemeral:
+    kindsToSkip:
+      - Clusterctl
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/versions-catalogue.yaml
+      - function/airshipctl-schemas/network-catalogue.yaml
+  clusterctl-init-ephemeral:
+    crdList:
+      - function/cert-manager/v1.1.0/upstream/cert-manager.yaml
+  controlplane-ephemeral:
+    kindsToSkip:
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/network-catalogue.yaml
+      - function/airshipctl-schemas/versions-catalogue.yaml
+      - function/capi/v0.3.7/crd/bases/cluster.x-k8s.io_clusters.yaml
+      - function/cacpk/v0.3.7/crd/bases/controlplane.cluster.x-k8s.io_kubeadmcontrolplanes.yaml
+      - function/capm3/v0.4.0/crd/bases/infrastructure.cluster.x-k8s.io_metal3clusters.yaml
+      - function/capm3/v0.4.0/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml
+      - global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml
+  clusterctl-init-target:
+    crdList:
+      - function/cert-manager/v1.1.0/upstream/cert-manager.yaml
+  initinfra-target:
+    kindsToSkip:
+      - Clusterctl
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/network-catalogue.yaml
+      - function/airshipctl-schemas/versions-catalogue.yaml
+  workers-target:
+    crdList:
+      - global/crd/baremetal-operator/metal3.io_baremetalhosts_crd.yaml
+  workers-classification:
+    kindsToSkip:
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/network-catalogue.yaml
+      - function/airshipctl-schemas/versions-catalogue.yaml
+      - function/cabpk/v0.3.7/crd/bases/bootstrap.cluster.x-k8s.io_kubeadmconfigtemplates.yaml
+      - function/capi/v0.3.7/crd/bases/cluster.x-k8s.io_machinedeployments.yaml
+      - function/capm3/v0.4.0/crd/bases/infrastructure.cluster.x-k8s.io_metal3machinetemplates.yaml
+      - function/hwcc/crd/bases/metal3.io_hardwareclassifications.yaml
+  workload-target:
+    kindsToSkip:
+      - VariableCatalogue
+    crdList:
+      - function/airshipctl-schemas/network-catalogue.yaml
+      - function/airshipctl-schemas/versions-catalogue.yaml
+      - function/flux/helm-controller/upstream/crd/bases/helm.toolkit.fluxcd.io_helmreleases.yaml
+      - function/flux/source-controller/upstream/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml

pkg/api/v1alpha1/genericcontainer_types.go

@@ -35,6 +35,11 @@ const (
 	KubeConfigEnvKeyContext = "KCTL_CONTEXT"
 	// KubeConfigEnv uses as a kubeconfig env variable
 	KubeConfigEnv = KubeConfigEnvKey + "=" + KubeConfigPath
+
+	// ValidatorPreventCleanup is an env variable that prevents validator to clean up its working directory after finish
+	ValidatorPreventCleanup = "VALIDATOR_PREVENT_CLEANUP"
+	// ValidatorPlanValidation is an env variable that tells validator not to use phase-specific config for validation
+	ValidatorPlanValidation = "VALIDATOR_PLAN_VALIDATION"
 )
 
 // +kubebuilder:object:root=true

pkg/document/bundle.go

@@ -59,10 +59,31 @@ type Bundle interface {
 	Append(Document) error
 }
 
+// DocFactoryFunc is a type of function which returns (Document, error) and can be used on demand
+type DocFactoryFunc func() (Document, error)
+
 // BundleFactoryFunc is a function that returns bundle, can be used to build bundle on demand
 // instead of inplace, useful, when you don't know if bundle will be needed or not, see phase for detail
 type BundleFactoryFunc func() (Bundle, error)
 
+// BundleFactoryFromBytes is a function which returns BundleFactoryFunc based on new bundle from bytes
+func BundleFactoryFromBytes(data []byte) BundleFactoryFunc {
+	return func() (Bundle, error) {
+		return NewBundleFromBytes(data)
+	}
+}
+
+// BundleFactoryFromDocRoot is a function which returns BundleFactoryFunc based on new bundle from DocumentRoot path
+func BundleFactoryFromDocRoot(docRootFunc func() (string, error)) BundleFactoryFunc {
+	return func() (Bundle, error) {
+		path, err := docRootFunc()
+		if err != nil {
+			return nil, err
+		}
+		return NewBundleByPath(path)
+	}
+}
+
 // NewBundleByPath is a function which builds new document.Bundle from kustomize rootPath using default FS object
 // example: document.NewBundleByPath("path/to/phase-root")
 func NewBundleByPath(rootPath string) (Bundle, error) {

pkg/document/constants.go

@@ -36,6 +36,15 @@ const (
 	ClusterctlMetadataKind    = "Metadata"
 	ClusterctlMetadataVersion = "v1alpha3"
 	ClusterctlMetadataGroup   = "clusterctl.cluster.x-k8s.io"
+
+	// DocumentValidationGroup defines Group for document-validation container
+	DocumentValidationGroup = "airshipit.org"
+	// DocumentValidationVersion defines Version for document-validation container
+	DocumentValidationVersion = "v1alpha1"
+	// DocumentValidationKind defines Kind for document-validation container
+	DocumentValidationKind = "GenericContainer"
+	// DocumentValidationName defines Name for document-validation container
+	DocumentValidationName = "document-validation"
 )
 
 // KustomizationFile is used for kustomization file

pkg/document/selectors.go

@@ -190,6 +190,14 @@ func NewClusterctlMetadataSelector() Selector {
 		ClusterctlMetadataKind)
 }
 
+// NewValidatorExecutorSelector returns selector to get validator executor documents
+func NewValidatorExecutorSelector() Selector {
+	return NewSelector().ByGvk(DocumentValidationGroup,
+		DocumentValidationVersion,
+		DocumentValidationKind).
+		ByName(DocumentValidationName)
+}
+
 //GetSecretData returns data located with a given key of a given document
 func GetSecretData(docBundle Bundle, apiSelector types.Selector, key string) ([]byte, error) {
 	s := NewSelector()

pkg/phase/client.go

@@ -15,8 +15,8 @@
 package phase
 
 import (
+	"bytes"
 	"io"
-	"io/ioutil"
 	"path/filepath"
 
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -32,6 +32,7 @@ import (
 	"opendev.org/airship/airshipctl/pkg/phase/executors"
 	executorerrors "opendev.org/airship/airshipctl/pkg/phase/executors/errors"
 	"opendev.org/airship/airshipctl/pkg/phase/ifc"
+	"opendev.org/airship/airshipctl/pkg/util"
 )
 
 // ExecutorRegistry returns map with executor factories
@@ -60,22 +61,33 @@ type phase struct {
 	processor events.EventProcessor
 }
 
+func (p *phase) defaultBundleFactory() document.BundleFactoryFunc {
+	return document.BundleFactoryFromDocRoot(p.DocumentRoot)
+}
+
+func (p *phase) defaultDocFactory() document.DocFactoryFunc {
+	return func() (document.Document, error) {
+		return p.helper.ExecutorDoc(ifc.ID{Name: p.apiObj.Name, Namespace: p.apiObj.Namespace})
+	}
+}
+
 // Executor returns executor interface associated with the phase
 func (p *phase) Executor() (ifc.Executor, error) {
-	executorDoc, err := p.helper.ExecutorDoc(ifc.ID{Name: p.apiObj.Name, Namespace: p.apiObj.Namespace})
+	return p.executor(p.defaultDocFactory(), p.defaultBundleFactory())
+}
+
+func (p *phase) executor(docFactory document.DocFactoryFunc,
+	bundleFactory document.BundleFactoryFunc) (ifc.Executor, error) {
+	executorDoc, err := docFactory()
 	if err != nil {
 		return nil, err
 	}
 
-	var bundleFactory document.BundleFactoryFunc = func() (document.Bundle, error) {
-		docRoot, bundleFactoryFuncErr := p.DocumentRoot()
-		if bundleFactoryFuncErr != nil {
-			return nil, bundleFactoryFuncErr
+	refGVK := schema.GroupVersionKind{
+		Group:   executorDoc.GetGroup(),
+		Version: executorDoc.GetVersion(),
+		Kind:    executorDoc.GetKind(),
 	}
-		return document.NewBundleByPath(docRoot)
-	}
-
-	refGVK := p.apiObj.Config.ExecutorRef.GroupVersionKind()
 	// Look for executor factory defined in registry
 	executorFactory, found := p.registry()[refGVK]
 	if !found {
@@ -134,18 +146,34 @@ func (p *phase) Run(ro ifc.RunOptions) error {
 
 // Validate makes sure that phase is properly configured
 func (p *phase) Validate() error {
-	// Check that we can render documents supplied to phase
-	err := p.Render(ioutil.Discard, false, ifc.RenderOptions{})
-	if err != nil {
-		return err
-	}
-
-	// Check that executor if properly configured
 	executor, err := p.Executor()
 	if err != nil {
 		return err
 	}
-	return executor.Validate()
+	if err = executor.Validate(); err != nil {
+		return err
+	}
+
+	buf := &bytes.Buffer{}
+	if err = executor.Render(buf, ifc.RenderOptions{FilterSelector: document.NewSelector()}); err != nil {
+		return err
+	}
+
+	defer p.processor.Close()
+
+	executor, err = p.executor(func() (document.Document, error) {
+		return p.helper.PhaseConfigBundle().
+			SelectOne(document.NewValidatorExecutorSelector())
+	}, document.BundleFactoryFromBytes(buf.Bytes()))
+	if err != nil {
+		return err
+	}
+
+	ch := make(chan events.Event)
+	go func() {
+		executor.Run(ch, ifc.RunOptions{})
+	}()
+	return p.processor.Process(ch)
 }
 
 // Render executor documents
@@ -219,7 +247,12 @@ type plan struct {
 
 // Validate makes sure that phase plan is properly configured
 func (p *plan) Validate() error {
-	for _, step := range p.apiObj.Phases {
+	util.Setenv(util.EnvVar{Key: v1alpha1.ValidatorPreventCleanup}, util.EnvVar{Key: v1alpha1.ValidatorPlanValidation})
+	for i, step := range p.apiObj.Phases {
+		log.Printf("validating phase: %s\n", step.Name)
+		if i == len(p.apiObj.Phases)-1 {
+			util.Unsetenv(util.EnvVar{Key: v1alpha1.ValidatorPreventCleanup})
+		}
 		phaseRunner, err := p.phaseClient.PhaseByID(ifc.ID{Name: step.Name})
 		if err != nil {
 			return err
@@ -239,7 +272,7 @@ func (p *plan) Run(ro ifc.RunOptions) error {
 			return err
 		}
 
-		log.Printf("executing phase: %s\n", step)
+		log.Printf("executing phase: %s\n", step.Name)
 		if err = phaseRunner.Run(ro); err != nil {
 			return err
 		}

pkg/phase/client_test.go

@@ -69,7 +69,7 @@ func TestClientPhaseExecutor(t *testing.T) {
 
 	for _, tt := range tests {
 		tt := tt
-		t.Run("", func(t *testing.T) {
+		t.Run(tt.name, func(t *testing.T) {
 			conf := tt.configFunc(t)
 			helper, err := phase.NewHelper(conf)
 			require.NoError(t, err)
@@ -77,7 +77,8 @@ func TestClientPhaseExecutor(t *testing.T) {
 			client := phase.NewClient(helper, phase.InjectRegistry(tt.registryFunc))
 			require.NotNil(t, client)
 			p, err := client.PhaseByID(tt.phaseID)
-			require.NotNil(t, client)
+			require.NotNil(t, p)
+			require.NoError(t, err)
 			executor, err := p.Executor()
 			if tt.errContains != "" {
 				require.Error(t, err)
@@ -150,13 +151,15 @@ func TestPhaseValidate(t *testing.T) {
 			configFunc:   testConfig,
 			phaseID:      ifc.ID{Name: "capi_init"},
 			registryFunc: fakeRegistry,
+			errContains: `document filtered by selector [Group="airshipit.org", Version="v1alpha1", ` +
+				`Kind="GenericContainer", Name="document-validation"] found no documents`,
 		},
 		{
 			name:         "Error no document entry point",
 			configFunc:   testConfig,
 			phaseID:      ifc.ID{Name: "no_entry_point"},
 			registryFunc: fakeRegistry,
-			errContains:  "documentEntryPoint is not defined for the phase 'no_entry_point' in namespace ''",
+			errContains:  "executor identified by 'airshipit.org/v1alpha1, Kind=SomeExecutor' is not found",
 		},
 		{
 			name:         "Error no executor",
@@ -391,13 +394,15 @@ func TestPlanValidate(t *testing.T) {
 			configFunc:   testConfig,
 			planID:       ifc.ID{Name: "init"},
 			registryFunc: fakeRegistry,
+			errContains: `document filtered by selector [Group="airshipit.org", Version="v1alpha1", ` +
+				`Kind="GenericContainer", Name="document-validation"] found no documents`,
 		},
 		{
 			name:         "Invalid fake executor",
 			configFunc:   testConfig,
 			planID:       ifc.ID{Name: "plan_invalid_phase"},
 			registryFunc: fakeRegistry,
-			errContains:  "documentEntryPoint is not defined for the phase 'no_entry_point' in namespace ''",
+			errContains:  "executor identified by 'airshipit.org/v1alpha1, Kind=SomeExecutor' is not found",
 		},
 		{
 			name:         "Phase does not exist",
@@ -430,7 +435,7 @@ func TestPlanValidate(t *testing.T) {
 	}
 }
 
-func fakeExecFactory(config ifc.ExecutorConfig) (ifc.Executor, error) {
+func fakeExecFactory(_ ifc.ExecutorConfig) (ifc.Executor, error) {
 	return fakeExecutor{}, nil
 }
 

pkg/phase/command.go

@@ -298,6 +298,7 @@ func (c *ValidateCommand) RunE() error {
 		return err
 	}
 
+	util.Setenv(util.EnvVar{Key: "AIRSHIPCTL_CURRENT_PHASE", Value: c.Options.PhaseID.Name})
 	helper, err := NewHelper(cfg)
 	if err != nil {
 		return err
@@ -364,6 +365,7 @@ func (c *PlanValidateCommand) RunE() error {
 		return err
 	}
 
+	util.Setenv(util.EnvVar{Key: "AIRSHIPCTL_CURRENT_PLAN", Value: c.Options.PlanID.Name})
 	helper, err := NewHelper(cfg)
 	if err != nil {
 		return err

pkg/phase/command_test.go

@@ -606,6 +606,8 @@ func TestValidateCommand(t *testing.T) {
 				}
 				return conf, nil
 			},
+			errContains: `document filtered by selector [Group="airshipit.org", Version="v1alpha1", ` +
+				`Kind="GenericContainer", Name="document-validation"] found no documents`,
 		},
 	}
 

pkg/phase/helper.go

@@ -258,8 +258,8 @@ func (helper *Helper) ExecutorDoc(phaseID ifc.ID) (document.Document, error) {
 	if err != nil {
 		return nil, err
 	}
-	phaseConfig := phaseObj.Config
 
+	phaseConfig := phaseObj.Config
 	if phaseConfig.ExecutorRef == nil {
 		return nil, errors.ErrExecutorRefNotDefined{PhaseName: phaseID.Name, PhaseNamespace: phaseID.Namespace}
 	}

pkg/phase/helper_test.go

@@ -475,7 +475,6 @@ func TestHelperExecutorDoc(t *testing.T) {
 				conf.Manifests["dummy_manifest"].MetadataPath = brokenMetaPath
 				return conf
 			},
-			errContains: "no such file or directory",
 			helperErr: true,
 		},
 		{

tools/document/validate_site_docs.sh

@@ -20,7 +20,6 @@ set -xe
 # The location of sites whose manifests should be validated.
 # This are relative to MANIFEST_ROOT above
 : ${SITE_ROOT:="$(basename "${PWD}")/manifests/site"}
-: ${SCHEMAS_ROOT:="${PWD}/manifests/function/airshipctl-schemas"}
 : ${MANIFEST_REPO_URL:="https://review.opendev.org/airship/airshipctl"}
 : ${SITE:="test-workload"}
 : ${CONTEXT:="kind-airship"}
@@ -97,13 +96,14 @@ if [ -f "${AIRSHIPKUBECONFIG}" ]; then
   cp "${AIRSHIPKUBECONFIG}" "${AIRSHIPKUBECONFIG_BACKUP}"
 fi
 
-
 generate_airshipconf "default"
 
-catalogues=("versions" "networking")
-
 phase_plans=$(airshipctl --airshipconf ${AIRSHIPCONFIG} plan list | grep "PhasePlan" | awk -F '/' '{print $2}' | awk '{print $1}')
 for plan in $phase_plans; do
+  # Perform static validation first, add support of all plans later
+  if [ "$plan" = "phasePlan" ]; then
+    airshipctl --airshipconf ${AIRSHIPCONFIG} plan validate $plan
+  fi
 
   cluster_list=$(airshipctl --airshipconf ${AIRSHIPCONFIG} cluster list)
   # Loop over all cluster types and phases for the given site
@@ -123,9 +123,6 @@ for plan in $phase_plans; do
     # In the meantime, as new phases are added, please add them here as well.
     phases=$(airshipctl --airshipconf ${AIRSHIPCONFIG} phase list --plan $plan -c $cluster | grep Phase | awk -F '/' '{print $2}' | awk '{print $1}' || true)
 
-    # apply catalogue CRDs
-    ${KUBECTL} --context ${CLUSTER} --kubeconfig ${KUBECONFIG} apply -k ${SCHEMAS_ROOT}
-
     for phase in $phases; do
       # Guard against bootstrap or initinfra being missing, which could be the case for some configs
       echo -e "\n*** Rendering ${cluster}/${phase}"
@@ -135,10 +132,6 @@ for plan in $phase_plans; do
       # e.g., load CRDs from initinfra first, so they're present when validating later phases
       ${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} phase render ${phase} -s executor -k CustomResourceDefinition >${TMP}/${phase}-crds.yaml
 
-      # extract rendered catalogue CRs
-      ${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} phase render ${phase} -s executor -k NetworkCatalogue >${TMP}/${phase}-networking.yaml
-      ${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} phase render ${phase} -s executor -k VersionsCatalogue >${TMP}/${phase}-versions.yaml
-
       if [ -s ${TMP}/${phase}-crds.yaml ]; then
         ${KUBECTL} --context ${CLUSTER} apply -f ${TMP}/${phase}-crds.yaml
       fi
@@ -146,14 +139,6 @@ for plan in $phase_plans; do
       # step 2: dry-run the entire phase
       ${ACTL} phase run --dry-run ${phase}
 
-      # catalogues have the label deploy-k8s: false, so they won't get applied during the dry-run
-      # and will have to be applied manually here
-      for catalogue in "${catalogues[@]}"
-      do
-        if [ -s ${TMP}/${phase}-${catalogue}.yaml ]; then
-          ${KUBECTL} --context ${CLUSTER} --kubeconfig ${KUBECONFIG} apply -f ${TMP}/$phase-${catalogue}.yaml --dry-run=client
-        fi
-      done
     done
     # Delete cluster kubeconfig
     rm ${KUBECONFIG}

tools/validate_docs

@@ -29,8 +29,13 @@ TMP=$(KIND_URL=${KIND_URL} ./tools/document/get_kind.sh)
 export KIND=${TMP}/kind
 export KUBECTL_URL
 
+sites_to_skip=(az-test-site docker-test-site gcp-test-site openstack-test-site)
+
 for site_root in ${SITE_ROOTS}; do
   for site in $(ls ${MANIFEST_ROOT}/${site_root}); do
+      if [[ " ${sites_to_skip[@]} " =~ " ${site} " ]]; then
+         continue
+      fi
       echo -e "\nValidating site: ${MANIFEST_ROOT}/${site_root}/${site}\n****************"
       MANIFEST_ROOT=${MANIFEST_ROOT} SITE_ROOT=${site_root} SITE=${site} \
         ./tools/document/validate_site_docs.sh

zuul.d/jobs.yaml

@@ -61,7 +61,7 @@
 
 - job:
     name: airship-airshipctl-validate-site-docs
-    timeout: 5400
+    timeout: 6600
     pre-run:
       - playbooks/airship-airshipctl-deploy-docker.yaml
     run: playbooks/airshipctl-gate-runner.yaml