Add multiple kubeconifg sources to ClusterMap

Change ClusterMap API object to support multiple kubeconfig sources
for a cluster. If one kubeconfig source fails, kubeconfig builder
will not fail and move on to the next one. This behaviour will allow
to support cases when ephemeral cluster is not accesible anymore or
when target cluster is not yet accessible.

For more information please read issue #460 in airshipctl github

Relates-To: #460
Related-To: #460

Change-Id: I7cd32f78cd7c4ad8814eac357424c24216f40d76

cmd/phase/run.go

@@ -60,11 +60,6 @@ func NewRunCommand(cfgFactory config.Factory) *cobra.Command {
 		"wait-timeout",
 		0,
 		"wait timeout")
-	flags.StringVar(
-		&p.Options.Kubeconfig,
-		"kubeconfig",
-		"",
-		"Path to kubeconfig associated with site being managed")
 	flags.BoolVar(
 		&p.Options.Progress,
 		"progress",

cmd/phase/testdata/TestRunGoldenOutput/run-with-help.golden

@@ -12,6 +12,5 @@ airshipctl phase run ephemeral-control-plane
 Flags:
       --dry-run                 simulate phase execution
   -h, --help                    help for run
-      --kubeconfig string       Path to kubeconfig associated with site being managed
       --progress                show progress
       --wait-timeout duration   wait timeout

docs/source/cli/airshipctl_phase.md

@@ -28,5 +28,5 @@ such as getting list and applying specific one.
 * [airshipctl phase render](airshipctl_phase_render.md)	 - Render phase documents from model
 * [airshipctl phase run](airshipctl_phase_run.md)	 - Run phase
 * [airshipctl phase tree](airshipctl_phase_tree.md)	 - Tree view of kustomize entrypoints of phase
-* [airshipctl phase validate](airshipctl_phase_validate.md)	 - Validate phase
+* [airshipctl phase validate](airshipctl_phase_validate.md)	 - Assert that a phase is valid
 

docs/source/cli/airshipctl_phase_run.md

@@ -24,7 +24,6 @@ airshipctl phase run ephemeral-control-plane
 ```
       --dry-run                 simulate phase execution
   -h, --help                    help for run
-      --kubeconfig string       Path to kubeconfig associated with site being managed
       --progress                show progress
       --wait-timeout duration   wait timeout
 ```

docs/source/cli/airshipctl_phase_validate.md

@@ -4,7 +4,8 @@ Assert that a phase is valid
 
 ### Synopsis
 
-Command which would validate that the phase contains the required documents to run the phase
+Command which would validate that the phase contains the required documents to run the phase.
+
 
 ```
 airshipctl phase validate PHASE_NAME [flags]
@@ -22,7 +23,7 @@ airshipctl phase validate initinfra
 ### Options
 
 ```
-  -h, --help                    help for run
+  -h, --help   help for validate
 ```
 
 ### Options inherited from parent commands

manifests/phases/cluster-map.yaml

@@ -8,4 +8,25 @@ metadata:
 map:
   target-cluster:
     parent: ephemeral-cluster
-  ephemeral-cluster: {}
+    kubeconfigSources:
+      - type: "filesystem"
+        filesystem:
+          path: ~/.airship/kubeconfig
+          contextName: target-cluster
+      - type: "clusterAPI"
+        clusterAPI:
+          clusterNamespacedName:
+            name: target-cluster
+            namespace: default
+      - type: "bundle"
+        bundle:
+          contextName: target-cluster
+  ephemeral-cluster:
+    kubeconfigSources:
+      - type: "filesystem"
+        filesystem:
+          path: ~/.airship/kubeconfig
+          contextName: ephemeral-cluster
+      - type: "bundle"
+        bundle:
+          contextName: ephemeral-cluster

pkg/api/v1alpha1/cluster_map_types.go

@@ -32,19 +32,50 @@ type ClusterMap struct {
 type Cluster struct {
 	// Parent is a key in ClusterMap.Map that identifies the name of the parent(management) cluster
 	Parent string `json:"parent,omitempty"`
-	// DynamicKubeConfig kubeconfig allows to get kubeconfig from parent cluster, instead
-	// expecting it to be in document bundle. Parent kubeconfig will be used to get kubeconfig
-	DynamicKubeConfig bool `json:"dynamicKubeConf,omitempty"`
 	// KubeconfigContext is the context in kubeconfig, default is equals to clusterMap key
-	KubeconfigContext string `json:"kubeconfigContext,omitempty"`
+	Sources []KubeconfigSource `json:"kubeconfigSources"`
-	// ClusterAPIRef references to Cluster API cluster resources
-	ClusterAPIRef ClusterAPIRef `json:"clusterAPIRef,omitempty"`
 }
 
-// ClusterAPIRef will be used to find cluster object in kubernetes parent cluster
+// KubeconfigSource describes source of the kubeconfig
-type ClusterAPIRef struct {
+type KubeconfigSource struct {
-	Name      string
+	Type       KubeconfigSourceType       `json:"type"`
-	Namespace string
+	FileSystem KubeconfigSourceFilesystem `json:"filesystem,omitempty"`
+	Bundle     KubeconfigSourceBundle     `json:"bundle,omitempty"`
+	ClusterAPI KubeconfigSourceClusterAPI `json:"clusterAPI,omitempty"`
+}
+
+// KubeconfigSourceType type of source
+type KubeconfigSourceType string
+
+const (
+	// KubeconfigSourceTypeFilesystem is used when you want kubeconfig to be taken from local filesystem
+	KubeconfigSourceTypeFilesystem KubeconfigSourceType = "filesystem"
+	// KubeconfigSourceTypeBundle use config document bundle to get kubeconfig
+	KubeconfigSourceTypeBundle KubeconfigSourceType = "bundle"
+	// KubeconfigSourceTypeClusterAPI use ClusterAPI to get kubeconfig, parent cluster must be specified
+	KubeconfigSourceTypeClusterAPI KubeconfigSourceType = "clusterAPI"
+)
+
+// KubeconfigSourceFilesystem get kubeconfig from filesystem path
+type KubeconfigSourceFilesystem struct {
+	Path    string `json:"path,omitempty"`
+	Context string `json:"contextName,omitempty"`
+}
+
+// KubeconfigSourceClusterAPI get kubeconfig from clusterAPI parent cluster
+type KubeconfigSourceClusterAPI struct {
+	NamespacedName `json:"clusterNamespacedName,omitempty"`
+}
+
+// KubeconfigSourceBundle get kubeconfig from bundle
+type KubeconfigSourceBundle struct {
+	Context string `json:"contextName,omitempty"`
+}
+
+// NamespacedName is a name combined with namespace to uniquely identify objects
+type NamespacedName struct {
+	Name      string `json:"name,omitempty"`
+	Namespace string `json:"namespace,omitempty"`
 }
 
 // DefaultClusterMap can be used to safely unmarshal ClusterMap object without nil pointers

pkg/api/v1alpha1/zz_generated.deepcopy.go

@@ -209,7 +209,11 @@ func (in *BootstrapContainer) DeepCopy() *BootstrapContainer {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Cluster) DeepCopyInto(out *Cluster) {
 	*out = *in
-	out.ClusterAPIRef = in.ClusterAPIRef
+	if in.Sources != nil {
+		in, out := &in.Sources, &out.Sources
+		*out = make([]KubeconfigSource, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster.
@@ -222,21 +226,6 @@ func (in *Cluster) DeepCopy() *Cluster {
 	return out
 }
 
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *ClusterAPIRef) DeepCopyInto(out *ClusterAPIRef) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAPIRef.
-func (in *ClusterAPIRef) DeepCopy() *ClusterAPIRef {
-	if in == nil {
-		return nil
-	}
-	out := new(ClusterAPIRef)
-	in.DeepCopyInto(out)
-	return out
-}
-
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ClusterMap) DeepCopyInto(out *ClusterMap) {
 	*out = *in
@@ -252,7 +241,7 @@ func (in *ClusterMap) DeepCopyInto(out *ClusterMap) {
 			} else {
 				in, out := &val, &outVal
 				*out = new(Cluster)
-				**out = **in
+				(*in).DeepCopyInto(*out)
 			}
 			(*out)[key] = outVal
 		}
@@ -568,6 +557,70 @@ func (in *KubeConfig) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigSource) DeepCopyInto(out *KubeconfigSource) {
+	*out = *in
+	out.FileSystem = in.FileSystem
+	out.Bundle = in.Bundle
+	out.ClusterAPI = in.ClusterAPI
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSource.
+func (in *KubeconfigSource) DeepCopy() *KubeconfigSource {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigSource)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigSourceBundle) DeepCopyInto(out *KubeconfigSourceBundle) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceBundle.
+func (in *KubeconfigSourceBundle) DeepCopy() *KubeconfigSourceBundle {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigSourceBundle)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigSourceClusterAPI) DeepCopyInto(out *KubeconfigSourceClusterAPI) {
+	*out = *in
+	out.NamespacedName = in.NamespacedName
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceClusterAPI.
+func (in *KubeconfigSourceClusterAPI) DeepCopy() *KubeconfigSourceClusterAPI {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigSourceClusterAPI)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *KubeconfigSourceFilesystem) DeepCopyInto(out *KubeconfigSourceFilesystem) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceFilesystem.
+func (in *KubeconfigSourceFilesystem) DeepCopy() *KubeconfigSourceFilesystem {
+	if in == nil {
+		return nil
+	}
+	out := new(KubeconfigSourceFilesystem)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *KubernetesApply) DeepCopyInto(out *KubernetesApply) {
 	*out = *in
@@ -609,6 +662,21 @@ func (in *MoveOptions) DeepCopy() *MoveOptions {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *NamespacedName) DeepCopyInto(out *NamespacedName) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedName.
+func (in *NamespacedName) DeepCopy() *NamespacedName {
+	if in == nil {
+		return nil
+	}
+	out := new(NamespacedName)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Phase) DeepCopyInto(out *Phase) {
 	*out = *in

pkg/cluster/clustermap/errors.go

@@ -37,5 +37,5 @@ type ErrClusterNotInMap struct {
 }
 
 func (e ErrClusterNotInMap) Error() string {
-	return fmt.Sprintf("cluster %s is not defined in in cluster map %v", e.Child, e.Map)
+	return fmt.Sprintf("cluster '%s' is not defined in cluster map %v", e.Child, e.Map)
 }

pkg/cluster/clustermap/map.go

@@ -16,7 +16,6 @@ package clustermap
 
 import (
 	"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
-	"opendev.org/airship/airshipctl/pkg/log"
 )
 
 // DefaultClusterAPIObjNamespace is a default namespace used for cluster-api cluster object
@@ -28,9 +27,8 @@ const DefaultClusterAPIObjNamespace = "default"
 type ClusterMap interface {
 	ParentCluster(string) (string, error)
 	AllClusters() []string
-	DynamicKubeConfig(string) bool
 	ClusterKubeconfigContext(string) (string, error)
-	ClusterAPIRef(string) (ClusterAPIRef, error)
+	Sources(string) ([]v1alpha1.KubeconfigSource, error)
 }
 
 // clusterMap allows to view clusters and relationship between them
@@ -57,16 +55,6 @@ func (cm clusterMap) ParentCluster(child string) (string, error) {
 	return currentCluster.Parent, nil
 }
 
-// DynamicKubeConfig check if dynamic kubeconfig is enabled for the child cluster
-func (cm clusterMap) DynamicKubeConfig(child string) bool {
-	childCluster, exist := cm.apiMap.Map[child]
-	if !exist {
-		log.Debugf("cluster %s is not defined in cluster map %v", child, cm.apiMap)
-		return false
-	}
-	return childCluster.DynamicKubeConfig
-}
-
 // AllClusters returns all clusters in a map
 func (cm clusterMap) AllClusters() []string {
 	clusters := []string{}
@@ -76,49 +64,21 @@ func (cm clusterMap) AllClusters() []string {
 	return clusters
 }
 
-// ClusterAPIRef helps to find corresponding cluster-api Cluster object in kubernetes cluster
-type ClusterAPIRef struct {
-	Name      string
-	Namespace string
-}
-
-// ClusterAPIRef maps a clusterapi name and namespace for a given cluster
-func (cm clusterMap) ClusterAPIRef(clusterName string) (ClusterAPIRef, error) {
-	clstr, ok := cm.apiMap.Map[clusterName]
-	if !ok {
-		return ClusterAPIRef{}, ErrClusterNotInMap{Child: clusterName, Map: cm.apiMap}
-	}
-
-	name := clstr.ClusterAPIRef.Name
-	namespace := clstr.ClusterAPIRef.Namespace
-
-	if name == "" {
-		name = clusterName
-	}
-
-	if namespace == "" {
-		namespace = DefaultClusterAPIObjNamespace
-	}
-
-	return ClusterAPIRef{
-		Name:      name,
-		Namespace: namespace,
-	}, nil
-}
-
 // ClusterKubeconfigContext returns name of the context in kubeconfig corresponding to a given cluster
 func (cm clusterMap) ClusterKubeconfigContext(clusterName string) (string, error) {
-	cluster, exists := cm.apiMap.Map[clusterName]
+	_, exists := cm.apiMap.Map[clusterName]
 
 	if !exists {
 		return "", ErrClusterNotInMap{Map: cm.apiMap, Child: clusterName}
 	}
 
-	kubeContext := cluster.KubeconfigContext
+	return clusterName, nil
-	// if kubeContext is still empty, set it to clusterName
+}
-	if cluster.KubeconfigContext == "" {
+
-		kubeContext = clusterName
+func (cm clusterMap) Sources(clusterName string) ([]v1alpha1.KubeconfigSource, error) {
-	}
+	cluster, ok := cm.apiMap.Map[clusterName]
-
+	if !ok {
-	return kubeContext, nil
+		return nil, ErrClusterNotInMap{Child: clusterName, Map: cm.apiMap}
+	}
+	return cluster.Sources, nil
 }

pkg/cluster/clustermap/map_test.go

@@ -28,7 +28,6 @@ func TestClusterMap(t *testing.T) {
 	targetCluster := "target"
 	ephemeraCluster := "ephemeral"
 	workloadCluster := "workload"
-	workloadClusterKubeconfigContext := "different-workload-context"
 	workloadClusterNoParent := "workload without parent"
 	workloadClusterAPIRefName := "workload-cluster-api"
 	workloadClusterAPIRefNamespace := "some-namespace"
@@ -36,20 +35,33 @@ func TestClusterMap(t *testing.T) {
 		Map: map[string]*v1alpha1.Cluster{
 			targetCluster: {
 				Parent: ephemeraCluster,
-				DynamicKubeConfig: false,
+				Sources: []v1alpha1.KubeconfigSource{
+					{
+						Type: v1alpha1.KubeconfigSourceTypeBundle,
+					},
+				},
 			},
 			ephemeraCluster: {},
 			workloadCluster: {
 				Parent: targetCluster,
-				DynamicKubeConfig: true,
+				Sources: []v1alpha1.KubeconfigSource{
-				KubeconfigContext: workloadClusterKubeconfigContext,
+					{
-				ClusterAPIRef: v1alpha1.ClusterAPIRef{
+						Type: v1alpha1.KubeconfigSourceTypeClusterAPI,
+						ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{
+							NamespacedName: v1alpha1.NamespacedName{
 								Name:      workloadClusterAPIRefName,
 								Namespace: workloadClusterAPIRefNamespace,
 							},
 						},
+					},
+				},
+			},
 			workloadClusterNoParent: {
-				DynamicKubeConfig: true,
+				Sources: []v1alpha1.KubeconfigSource{
+					{
+						Type: v1alpha1.KubeconfigSourceTypeClusterAPI,
+					},
+				},
 			},
 		},
 	}
@@ -69,16 +81,6 @@ func TestClusterMap(t *testing.T) {
 		assert.Equal(t, "", parent)
 	})
 
-	t.Run("not dynamic kubeconf target", func(t *testing.T) {
-		dynamic := cMap.DynamicKubeConfig(targetCluster)
-		assert.False(t, dynamic)
-	})
-
-	t.Run("dynamic kubeconf workload", func(t *testing.T) {
-		dynamic := cMap.DynamicKubeConfig(workloadCluster)
-		assert.True(t, dynamic)
-	})
-
 	t.Run("target parent", func(t *testing.T) {
 		parent, err := cMap.ParentCluster(workloadCluster)
 		assert.NoError(t, err)
@@ -97,12 +99,6 @@ func TestClusterMap(t *testing.T) {
 	})
 
 	t.Run("kubeconfig context", func(t *testing.T) {
-		kubeContext, err := cMap.ClusterKubeconfigContext(workloadCluster)
-		assert.NoError(t, err)
-		assert.Equal(t, workloadClusterKubeconfigContext, kubeContext)
-	})
-
-	t.Run("kubeconfig default context", func(t *testing.T) {
 		kubeContext, err := cMap.ClusterKubeconfigContext(targetCluster)
 		assert.NoError(t, err)
 		assert.Equal(t, targetCluster, kubeContext)
@@ -113,22 +109,15 @@ func TestClusterMap(t *testing.T) {
 		assert.Error(t, err)
 	})
 
-	t.Run("ClusterAPI ref name and namespace defaults", func(t *testing.T) {
+	t.Run("sources match", func(t *testing.T) {
-		ref, err := cMap.ClusterAPIRef(workloadClusterNoParent)
+		sources, err := cMap.Sources(workloadCluster)
 		assert.NoError(t, err)
-		assert.Equal(t, clustermap.DefaultClusterAPIObjNamespace, ref.Namespace)
+		expectedSources := apiMap.Map[workloadCluster].Sources
-		assert.Equal(t, workloadClusterNoParent, ref.Name)
+		assert.Equal(t, expectedSources, sources)
 	})
 
-	t.Run("ClusterAPI ref name and namespace", func(t *testing.T) {
+	t.Run("sources no cluster found", func(t *testing.T) {
-		ref, err := cMap.ClusterAPIRef(workloadCluster)
+		_, err := cMap.Sources("does not exist")
-		assert.NoError(t, err)
-		assert.Equal(t, workloadClusterAPIRefNamespace, ref.Namespace)
-		assert.Equal(t, workloadClusterAPIRefName, ref.Name)
-	})
-
-	t.Run("ClusterAPI ref error", func(t *testing.T) {
-		_, err := cMap.ClusterAPIRef("doesn't exist")
 		assert.Error(t, err)
 	})
 }

pkg/k8s/kubeconfig/builder.go

@@ -15,8 +15,7 @@
 package kubeconfig
 
 import (
-	"bytes"
+	"fmt"
-	"path/filepath"
 
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/clientcmd/api"
@@ -24,10 +23,8 @@ import (
 	"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
 	"opendev.org/airship/airshipctl/pkg/cluster/clustermap"
 	"opendev.org/airship/airshipctl/pkg/clusterctl/client"
-	"opendev.org/airship/airshipctl/pkg/config"
 	"opendev.org/airship/airshipctl/pkg/fs"
 	"opendev.org/airship/airshipctl/pkg/log"
-	"opendev.org/airship/airshipctl/pkg/util"
 )
 
 // KubeconfigDefaultFileName is a default name for kubeconfig
@@ -35,13 +32,14 @@ const KubeconfigDefaultFileName = "kubeconfig"
 
 // NewBuilder returns instance of kubeconfig builder.
 func NewBuilder() *Builder {
-	return &Builder{}
+	return &Builder{
+		siteKubeconf: emptyConfig(),
+	}
 }
 
 // Builder is an object that allows to build a kubeconfig based on various provided sources
 // such as path to kubeconfig, path to bundle that should contain kubeconfig and parent cluster
 type Builder struct {
-	path        string
 	bundlePath  string
 	clusterName string
 	root        string
@@ -49,12 +47,7 @@ type Builder struct {
 	clusterMap       clustermap.ClusterMap
 	clusterctlClient client.Interface
 	fs               fs.FileSystem
-}
+	siteKubeconf     *api.Config
-
-// WithPath allows to set path to prexisting kubeconfig
-func (b *Builder) WithPath(filePath string) *Builder {
-	b.path = filePath
-	return b
 }
 
 // WithBundle allows to set path to bundle that should contain kubeconfig api object
@@ -94,43 +87,116 @@ func (b *Builder) WithFilesytem(fs fs.FileSystem) *Builder {
 	return b
 }
 
-// Build builds a kubeconfig interface to be used
+// Build site kubeconfig, ignores, but logs, errors that happen when building individual
+// kubeconfigs. We need this behavior because, some clusters may not yet be deployed
+// and their kubeconfig is inaccessible yet, but will be accessible at later phases
+// If builder can't build kubeconfig for specific cluster, its context will not be present
+// in final kubeconfig. User of kubeconfig, will receive error stating that context doesn't exist
 func (b *Builder) Build() Interface {
-	switch {
+	return NewKubeConfig(b.build, InjectFileSystem(b.fs), InjectTempRoot(b.root))
-	case b.path != "":
+}
-		return NewKubeConfig(FromFile(b.path, b.fs), InjectFilePath(b.path, b.fs), InjectTempRoot(b.root))
+
-	case b.fromParent():
+func (b *Builder) build() ([]byte, error) {
-		// TODO consider adding various drivers to source kubeconfig from
+	for _, clusterID := range b.clusterMap.AllClusters() {
-		// Also consider accumulating different kubeconfigs, and returning one single
+		log.Printf("Getting kubeconfig for cluster '%s'", clusterID)
-		// large file, so that every executor has access to all parent clusters.
+		// buildOne merges context into site kubeconfig
-		return NewKubeConfig(b.buildClusterctlFromParent, InjectTempRoot(b.root), InjectFileSystem(b.fs))
+		_, _, err := b.buildOne(clusterID)
-	case b.bundlePath != "":
+		if IsErrAllSourcesFailedErr(err) {
-		return NewKubeConfig(FromBundle(b.bundlePath), InjectTempRoot(b.root), InjectFileSystem(b.fs))
+			log.Printf("All kubeconfig sources failed for cluster '%s', error '%v', skipping it",
+				clusterID, err)
+			continue
+		} else if err != nil {
+			return nil, err
+		}
+	}
+	// Set current context to clustername if it was provided
+	if b.clusterName != "" {
+		kubeContext, err := b.clusterMap.ClusterKubeconfigContext(b.clusterName)
+		if err != nil {
+			return nil, err
+		}
+		b.siteKubeconf.CurrentContext = kubeContext
+	}
+	return clientcmd.Write(*b.siteKubeconf)
+}
+
+func (b *Builder) buildOne(clusterID string) (string, *api.Config, error) {
+	destContext, err := b.clusterMap.ClusterKubeconfigContext(clusterID)
+	if err != nil {
+		return "", nil, err
+	}
+
+	// use already built kubeconfig context, to avoid doing work multiple times
+	built, oneKubeconf := b.alreadyBuilt(destContext)
+	if built {
+		log.Printf("kubeconfig for cluster '%s' is already built, using it", clusterID)
+		return destContext, oneKubeconf, nil
+	}
+
+	sources, err := b.clusterMap.Sources(clusterID)
+	if err != nil {
+		return "", nil, err
+	}
+
+	for _, source := range sources {
+		oneKubeconf, sourceErr := b.trySource(clusterID, destContext, source)
+		if sourceErr == nil {
+			// Merge source context into site kubeconfig
+			log.Printf("Merging kubecontext for cluster '%s', into site kubeconfig", clusterID)
+			if err = mergeContextAPI(destContext, destContext, b.siteKubeconf, oneKubeconf); err != nil {
+				return "", nil, err
+			}
+			return destContext, oneKubeconf, err
+		}
+		// if error, log it and ignore it. missing problem with one kubeconfig should not
+		// effect other clusters, which don't depend on it. If they do depend on it, their calls
+		// will fail because the context will be missing. Combitation with a log message will make
+		// it clear where the problem is.
+		log.Printf("received error while trying kubeconfig source for cluster '%s', source type '%s', error '%v'",
+			clusterID, source.Type, sourceErr)
+	}
+	// return empty not nil kubeconfig without error.
+	return "", nil, &ErrAllSourcesFailed{ClusterName: clusterID}
+}
+
+func (b *Builder) trySource(clusterID, dstContext string, source v1alpha1.KubeconfigSource) (*api.Config, error) {
+	var getter KubeSourceFunc
+	// TODO add sourceContext defaults
+	var sourceContext string
+	switch source.Type {
+	case v1alpha1.KubeconfigSourceTypeFilesystem:
+		getter = FromFile(source.FileSystem.Path, b.fs)
+		sourceContext = source.FileSystem.Context
+	case v1alpha1.KubeconfigSourceTypeBundle:
+		getter = FromBundle(b.bundlePath)
+		sourceContext = source.Bundle.Context
+	case v1alpha1.KubeconfigSourceTypeClusterAPI:
+		getter = b.fromClusterAPI(clusterID, source.ClusterAPI)
 	default:
-		// return default path to kubeconfig file in airship workdir
+		// TODO add validation for fast fails to clustermap interface instead of this
-		path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, KubeconfigDefaultFileName)
+		return nil, &ErrUknownKubeconfigSourceType{Type: string(source.Type)}
-		return NewKubeConfig(FromFile(path, b.fs), InjectFilePath(path, b.fs), InjectTempRoot(b.root))
 	}
+	kubeBytes, err := getter()
+	if err != nil {
+		return nil, err
+	}
+	return extractContext(dstContext, sourceContext, kubeBytes)
 }
 
-// fromParent checks if we should get kubeconfig from parent cluster secret
+func (b *Builder) fromClusterAPI(clusterName string, ref v1alpha1.KubeconfigSourceClusterAPI) KubeSourceFunc {
-func (b *Builder) fromParent() bool {
+	return func() ([]byte, error) {
-	if b.clusterMap == nil {
+		log.Printf("Getting kubeconfig from cluster API for cluster '%s'", clusterName)
-		return false
+		parentCluster, err := b.clusterMap.ParentCluster(clusterName)
-	}
-	return b.clusterMap.DynamicKubeConfig(b.clusterName)
-}
-
-func (b *Builder) buildClusterctlFromParent() ([]byte, error) {
-	currentCluster := b.clusterName
-	log.Printf("current cluster name is '%s'",
-		currentCluster)
-	parentCluster, err := b.clusterMap.ParentCluster(currentCluster)
 		if err != nil {
 			return nil, err
 		}
 
-	parentKubeconfig := b.WithClusterName(parentCluster).Build()
+		parentContext, parentKubeconf, err := b.buildOne(parentCluster)
+		if err != nil {
+			return nil, err
+		}
+
+		parentKubeconfig := NewKubeConfig(FromConfig(parentKubeconf), InjectFileSystem(b.fs))
 
 		f, cleanup, err := parentKubeconfig.GetFile()
 		if err != nil {
@@ -138,16 +204,6 @@ func (b *Builder) buildClusterctlFromParent() ([]byte, error) {
 		}
 		defer cleanup()
 
-	parentCtx, err := b.clusterMap.ClusterKubeconfigContext(parentCluster)
-	if err != nil {
-		return nil, err
-	}
-
-	clusterAPIRef, err := b.clusterMap.ClusterAPIRef(currentCluster)
-	if err != nil {
-		return nil, err
-	}
-
 		if b.clusterctlClient == nil {
 			b.clusterctlClient, err = client.NewClient("", log.DebugEnabled(), v1alpha1.DefaultClusterctl())
 			if err != nil {
@@ -156,67 +212,92 @@ func (b *Builder) buildClusterctlFromParent() ([]byte, error) {
 		}
 
 		log.Printf("Getting child kubeconfig from parent, parent context '%s', parent kubeconfing '%s'",
-		parentCtx, f)
+			parentContext, f)
-
+		return FromSecret(b.clusterctlClient, &client.GetKubeconfigOptions{
-	stringChild, err := b.clusterctlClient.GetKubeconfig(&client.GetKubeconfigOptions{
 			ParentKubeconfigPath:    f,
-		ParentKubeconfigContext: parentCtx,
+			ParentKubeconfigContext: parentContext,
-		ManagedClusterNamespace: clusterAPIRef.Namespace,
+			ManagedClusterNamespace: ref.Namespace,
-		ManagedClusterName:      clusterAPIRef.Name,
+			ManagedClusterName:      ref.Name,
-	})
+		})()
-	if err != nil {
-		return nil, err
 	}
-
-	buf := bytes.NewBuffer([]byte{})
-
-	err = parentKubeconfig.Write(buf)
-	if err != nil {
-		return nil, err
-	}
-
-	parentObj, err := clientcmd.Load(buf.Bytes())
-	if err != nil {
-		return nil, err
-	}
-
-	childObj, err := clientcmd.Load([]byte(stringChild))
-	if err != nil {
-		return nil, err
-	}
-
-	childCtx, err := b.clusterMap.ClusterKubeconfigContext(currentCluster)
-	if err != nil {
-		return nil, err
-	}
-	log.Printf("Merging '%s' cluster kubeconfig into '%s' cluster kubeconfig",
-		currentCluster, parentCluster)
-
-	return b.mergeOneContext(childCtx, parentObj, childObj)
 }
 
-// merges two kubeconfigs,
+func (b *Builder) alreadyBuilt(clusterContext string) (bool, *api.Config) {
-func (b *Builder) mergeOneContext(contextOverride string, dst, src *api.Config) ([]byte, error) {
+	kubeconfBytes, err := clientcmd.Write(*b.siteKubeconf)
-	for key, content := range src.AuthInfos {
+	if err != nil {
-		dst.AuthInfos[key] = content
+		log.Debugf("Received error when converting kubeconfig to bytes, ignoring kubeconfig. Error: %v", err)
+		return false, nil
 	}
 
-	for key, content := range src.Clusters {
+	// resulting and existing context names must be the same, otherwise error will be returned
-		dst.Clusters[key] = content
+	clusterKubeconfig, err := extractContext(clusterContext, clusterContext, kubeconfBytes)
+	if err != nil {
+		log.Debugf("Received error when extacting context, ignoring kubeconfig. Error: %v", err)
+		return false, nil
 	}
 
-	if len(src.Contexts) != 1 {
+	return true, clusterKubeconfig
-		return nil, &ErrClusterctlKubeconfigWrongContextsCount{
+}
-			ContextCount: len(src.Contexts),
+
-		}
+func extractContext(destContext, sourceContext string, src []byte) (*api.Config, error) {
-	}
+	srcKubeconf, err := clientcmd.Load(src)
-
+	if err != nil {
-	for key, content := range src.Contexts {
+		return nil, err
-		if contextOverride == "" {
+	}
-			contextOverride = key
+	dstKubeconf := emptyConfig()
-		}
+	return dstKubeconf, mergeContextAPI(destContext, sourceContext, dstKubeconf, srcKubeconf)
-		dst.Contexts[contextOverride] = content
+}
-	}
+
-
+// merges two kubeconfigs
-	return clientcmd.Write(*dst)
+func mergeContextAPI(destContext, sourceContext string, dst, src *api.Config) error {
+	if len(src.Contexts) > 1 && sourceContext == "" {
+		// When more than one context, we don't know which to choose
+		return &ErrKubeconfigMergeFailed{
+			Message: "kubeconfig has multiple contexts, don't know which to choose, " +
+				"please specify contextName in clusterMap cluster kubeconfig source",
+		}
+	}
+
+	var context *api.Context
+	context, exists := src.Contexts[sourceContext]
+	switch {
+	case exists:
+	case sourceContext == "" && len(src.Contexts) == 1:
+		for _, context = range src.Contexts {
+			log.Debugf("Using context '%v' to merge kubeconfig", context)
+		}
+	default:
+		return &ErrKubeconfigMergeFailed{
+			Message: fmt.Sprintf("source context '%s' does not exist in source kubeconfig", sourceContext),
+		}
+	}
+	dst.Contexts[destContext] = context
+
+	// TODO design logic to make authinfo keys unique, they can overlap, or human error can occur
+	user, exists := src.AuthInfos[context.AuthInfo]
+	if !exists {
+		return &ErrKubeconfigMergeFailed{
+			Message: fmt.Sprintf("user '%s' does not exist in source kubeconfig", context.AuthInfo),
+		}
+	}
+	dst.AuthInfos[context.AuthInfo] = user
+
+	// TODO design logic to make cluster keys unique, they can overlap, or human error can occur
+	cluster, exists := src.Clusters[context.Cluster]
+	if !exists {
+		return &ErrKubeconfigMergeFailed{
+			Message: fmt.Sprintf("cluster '%s' does not exist in source kubeconfig", context.Cluster),
+		}
+	}
+	dst.Clusters[context.Cluster] = cluster
+
+	return nil
+}
+
+func emptyConfig() *api.Config {
+	return &api.Config{
+		Contexts:  make(map[string]*api.Context),
+		AuthInfos: make(map[string]*api.AuthInfo),
+		Clusters:  make(map[string]*api.Cluster),
+	}
 }

pkg/k8s/kubeconfig/builder_test.go

@@ -16,22 +16,19 @@ package kubeconfig_test
 
 import (
 	"bytes"
-	"fmt"
-	"io/ioutil"
 	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
+	"k8s.io/client-go/tools/clientcmd"
 
 	"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
 	"opendev.org/airship/airshipctl/pkg/cluster/clustermap"
 	"opendev.org/airship/airshipctl/pkg/clusterctl/client"
-	"opendev.org/airship/airshipctl/pkg/config"
 	"opendev.org/airship/airshipctl/pkg/fs"
 	"opendev.org/airship/airshipctl/pkg/k8s/kubeconfig"
-	"opendev.org/airship/airshipctl/pkg/util"
 	"opendev.org/airship/airshipctl/testutil/clusterctl"
 	testfs "opendev.org/airship/airshipctl/testutil/fs"
 )
@@ -77,59 +74,17 @@ users:
     client-key-data: c29tZWNlcnQK`
 )
 
-func TestBuilder(t *testing.T) {
-	t.Run("Only bundle", func(t *testing.T) {
-		builder := kubeconfig.NewBuilder().WithBundle("testdata")
-		kube := builder.Build()
-		require.NotNil(t, kube)
-		buf := bytes.NewBuffer([]byte{})
-		err := kube.Write(buf)
-		require.NoError(t, err)
-		// check that kubeconfig contains expected cluster string
-		assert.Contains(t, buf.String(), "parent_parent_context")
-	})
-
-	t.Run("Only filepath", func(t *testing.T) {
-		builder := kubeconfig.NewBuilder().WithPath("testdata/kubeconfig")
-		kube := builder.Build()
-		require.NotNil(t, kube)
-		buf := bytes.NewBuffer([]byte{})
-		err := kube.Write(buf)
-		require.NoError(t, err)
-		// check that kubeconfig contains expected cluster string
-		assert.Contains(t, buf.String(), "dummycluster_ephemeral")
-	})
-
-	t.Run("No current cluster, fall to default", func(t *testing.T) {
-		clusterMap := &v1alpha1.ClusterMap{}
-		builder := kubeconfig.NewBuilder().
-			WithClusterMap(clustermap.NewClusterMap(clusterMap)).
-			WithClusterName("some-cluster")
-		kube := builder.Build()
-		// We should get a default value for cluster since we don't have some-cluster set
-		actualPath, cleanup, err := kube.GetFile()
-		require.NoError(t, err)
-		defer cleanup()
-		path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, kubeconfig.KubeconfigDefaultFileName)
-		assert.Equal(t, path, actualPath)
-	})
-
-	t.Run("Default source", func(t *testing.T) {
-		builder := kubeconfig.NewBuilder()
-		kube := builder.Build()
-		// When ClusterMap is specified, but it doesn't have cluster-name defined, and no
-		// other sources provided,
-		actualPath, cleanup, err := kube.GetFile()
-		require.NoError(t, err)
-		defer cleanup()
-		path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, kubeconfig.KubeconfigDefaultFileName)
-		assert.Equal(t, path, actualPath)
-	})
-}
-
 func TestBuilderClusterctl(t *testing.T) {
-	childCluster := "child"
+	childClusterID := "child"
-	parentCluster := "parent"
+	parentClusterID := "parent"
+	parentParentClusterID := "parent-parent"
+	// these are values in kubeconfig.cluster
+	parentCluster := "parent_cluster"
+	parentParentCluster := "parent_parent_cluster"
+	childCluster := "child_cluster"
+	parentUser := "parent_admin"
+	parentParentUser := "parent_parent_admin"
+	childUser := "child_user"
 	testBundlePath := "testdata"
 	kubeconfigPath := filepath.Join(testBundlePath, "kubeconfig-12341234")
 
@@ -137,79 +92,114 @@ func TestBuilderClusterctl(t *testing.T) {
 		name                 string
 		errString            string
 		requestedClusterName string
-		parentClusterName    string
 		tempRoot             string
+
+		expectedContexts, expectedClusters, expectedAuthInfos []string
 		clusterMap                                            clustermap.ClusterMap
 		clusterctlClient                                      client.Interface
 		fs                                                    fs.FileSystem
 	}{
 		{
-			name:                 "error no parent context",
+			name:                 "success cluster-api not reachable",
-			errString:            fmt.Sprintf("context \"%s\" does not exist", parentCluster),
+			requestedClusterName: childClusterID,
-			parentClusterName:    parentCluster,
+			expectedContexts:     []string{parentClusterID},
-			requestedClusterName: childCluster,
+			expectedClusters:     []string{parentParentCluster},
+			expectedAuthInfos:    []string{parentParentUser},
 			clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{
 				Map: map[string]*v1alpha1.Cluster{
-					childCluster: {
+					childClusterID: {
-						Parent:            parentCluster,
+						Parent: parentClusterID,
-						DynamicKubeConfig: true,
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeClusterAPI,
+							},
+						},
+					},
+					parentClusterID: {
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeBundle,
+								Bundle: v1alpha1.KubeconfigSourceBundle{
+									Context: "parent_parent_context",
+								},
+							},
 						},
-					parentCluster: {
-						DynamicKubeConfig: false,
 					},
 				},
 			}),
 		},
 		{
-			name:                 "error dynamic but no parrent",
+			name:              "success two clusters",
-			parentClusterName:    parentCluster,
+			expectedContexts:  []string{parentClusterID, parentParentClusterID},
-			requestedClusterName: childCluster,
+			expectedClusters:  []string{"dummycluster_ephemeral", parentParentCluster},
-			errString:            "failed to find a parent",
+			expectedAuthInfos: []string{"kubernetes-admin", parentParentUser},
 			clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{
 				Map: map[string]*v1alpha1.Cluster{
-					childCluster: {
+					parentParentClusterID: {
-						DynamicKubeConfig: true,
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeBundle,
+								Bundle: v1alpha1.KubeconfigSourceBundle{
+									Context: "parent_parent_context",
+								},
+							},
+						},
+					},
+					parentClusterID: {
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeFilesystem,
+								FileSystem: v1alpha1.KubeconfigSourceFilesystem{
+									Path:    "testdata/kubeconfig",
+									Context: "dummy_cluster",
+								},
+							},
+						},
 					},
 				},
 			}),
 		},
 		{
-			name:                 "error write temp parent",
+			name:              "success three clusters cluster-api",
-			parentClusterName:    parentCluster,
+			expectedContexts:  []string{parentClusterID, childClusterID, parentParentClusterID},
-			requestedClusterName: childCluster,
+			expectedClusters:  []string{parentCluster, parentParentCluster, childCluster},
-			tempRoot:             "does not exist anywhere",
+			expectedAuthInfos: []string{parentUser, parentParentUser, childUser},
-			errString:            "no such file or directory",
 			clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{
 				Map: map[string]*v1alpha1.Cluster{
-					childCluster: {
+					childClusterID: {
-						Parent:            parentCluster,
+						Parent: parentClusterID,
-						DynamicKubeConfig: true,
+						Sources: []v1alpha1.KubeconfigSource{
-					},
-					parentCluster: {
-						DynamicKubeConfig: false,
-						KubeconfigContext: "dummy_cluster",
-					},
-				},
-			}),
-		},
 							{
-			name:                 "success",
+								Type: v1alpha1.KubeconfigSourceTypeClusterAPI,
-			parentClusterName:    parentCluster,
+								ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{
-			requestedClusterName: childCluster,
+									NamespacedName: v1alpha1.NamespacedName{
-			clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{
+										Name:      childClusterID,
-				Map: map[string]*v1alpha1.Cluster{
+										Namespace: "default",
-					childCluster: {
+									},
-						Parent:            parentCluster,
+								},
-						DynamicKubeConfig: true,
+							},
+						},
+					},
+					parentClusterID: {
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeClusterAPI,
+								ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{
+									NamespacedName: v1alpha1.NamespacedName{
+										Name:      parentClusterID,
+										Namespace: "default",
+									},
+								},
+							},
+						},
+						Parent: parentParentClusterID,
+					},
+					parentParentClusterID: {
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeBundle,
 							},
-					parentCluster: {
-						DynamicKubeConfig: true,
-						KubeconfigContext: "parent-custom",
-						Parent:            "parent-parent",
 						},
-					"parent-parent": {
-						DynamicKubeConfig: false,
-						KubeconfigContext: "parent_parent_context",
 					},
 				},
 			}),
@@ -230,19 +220,38 @@ func TestBuilderClusterctl(t *testing.T) {
 				}
 				c.On("GetKubeconfig", &client.GetKubeconfigOptions{
 					ParentKubeconfigPath:    kubeconfigPath,
-					ParentKubeconfigContext: "parent-custom",
+					ParentKubeconfigContext: parentClusterID,
 					ManagedClusterNamespace: clustermap.DefaultClusterAPIObjNamespace,
-					ManagedClusterName:      childCluster,
+					ManagedClusterName:      childClusterID,
 				}).Once().Return(testKubeconfigString, nil)
 				c.On("GetKubeconfig", &client.GetKubeconfigOptions{
 					ParentKubeconfigPath:    kubeconfigPath,
-					ParentKubeconfigContext: "parent_parent_context",
+					ParentKubeconfigContext: parentParentClusterID,
 					ManagedClusterNamespace: clustermap.DefaultClusterAPIObjNamespace,
-					ManagedClusterName:      parentCluster,
+					ManagedClusterName:      parentClusterID,
 				}).Once().Return(testKubeconfigStringSecond, nil)
 				return c
 			}(),
 		},
+		{
+			name:                 "error requested cluster doesn't exist",
+			errString:            "is not defined in cluster map",
+			requestedClusterName: "non-existent-cluster",
+			clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{
+				Map: map[string]*v1alpha1.Cluster{
+					parentParentClusterID: {
+						Sources: []v1alpha1.KubeconfigSource{
+							{
+								Type: v1alpha1.KubeconfigSourceTypeBundle,
+								Bundle: v1alpha1.KubeconfigSourceBundle{
+									Context: "parent_parent_context",
+								},
+							},
+						},
+					},
+				},
+			}),
+		},
 	}
 
 	for _, tt := range tests {
@@ -268,16 +277,35 @@ func TestBuilderClusterctl(t *testing.T) {
 				assert.Contains(t, err.Error(), tt.errString)
 				assert.Equal(t, "", filePath)
 			} else {
-				assert.NoError(t, err)
+				require.NoError(t, err)
 				assert.NotEqual(t, "", filePath)
 				assert.NotNil(t, cleanup)
 				buf := bytes.NewBuffer([]byte{})
 				err := kube.Write(buf)
 				require.NoError(t, err)
-				result, err := ioutil.ReadFile("testdata/result-kubeconf")
+				compareResults(t, tt.expectedContexts, tt.expectedClusters, tt.expectedAuthInfos, buf.Bytes())
-				require.NoError(t, err)
-				assert.Equal(t, result, buf.Bytes())
 			}
 		})
 	}
 }
+
+func compareResults(t *testing.T, contexts, clusters, authInfos []string, kubeconfBytes []byte) {
+	t.Helper()
+	resultKubeconf, err := clientcmd.Load(kubeconfBytes)
+	require.NoError(t, err)
+
+	assert.Len(t, resultKubeconf.Contexts, len(contexts))
+	for _, name := range contexts {
+		assert.Contains(t, resultKubeconf.Contexts, name)
+	}
+
+	assert.Len(t, resultKubeconf.AuthInfos, len(authInfos))
+	for _, name := range authInfos {
+		assert.Contains(t, resultKubeconf.AuthInfos, name)
+	}
+
+	assert.Len(t, resultKubeconf.Clusters, len(clusters))
+	for _, name := range clusters {
+		assert.Contains(t, resultKubeconf.Clusters, name)
+	}
+}

pkg/k8s/kubeconfig/errors.go

@@ -16,21 +16,35 @@ package kubeconfig
 
 import "fmt"
 
-// ErrKubeConfigPathEmpty returned when kubeconfig path is not specified
+// ErrAllSourcesFailed returned when kubeconfig path is not specified
-type ErrKubeConfigPathEmpty struct {
+type ErrAllSourcesFailed struct {
+	ClusterName string
 }
 
-func (e *ErrKubeConfigPathEmpty) Error() string {
+func (e *ErrAllSourcesFailed) Error() string {
-	return "kubeconfig path is not defined"
+	return fmt.Sprintf("all kubeconfig sources failed for cluster '%s'", e.ClusterName)
 }
 
-// ErrClusterctlKubeconfigWrongContextsCount is returned when clusterctl client returns
+// ErrKubeconfigMergeFailed is returned when builder doesn't know which context to merge
-// multiple or no contexts in kubeconfig for the child cluster
+type ErrKubeconfigMergeFailed struct {
-type ErrClusterctlKubeconfigWrongContextsCount struct {
+	Message string
-	ContextCount int
 }
 
-func (e *ErrClusterctlKubeconfigWrongContextsCount) Error() string {
+func (e *ErrKubeconfigMergeFailed) Error() string {
-	return fmt.Sprintf("clusterctl client returned '%d' contexts in kubeconfig "+
+	return fmt.Sprintf("failed merging kubeconfig: %s", e.Message)
-		"context count must exactly one", e.ContextCount)
+}
+
+// IsErrAllSourcesFailedErr returns true if error is of type ErrAllSourcesFailedErr
+func IsErrAllSourcesFailedErr(err error) bool {
+	_, ok := err.(*ErrAllSourcesFailed)
+	return ok
+}
+
+// ErrUknownKubeconfigSourceType returned type of kubeconfig source is unknown
+type ErrUknownKubeconfigSourceType struct {
+	Type string
+}
+
+func (e *ErrUknownKubeconfigSourceType) Error() string {
+	return fmt.Sprintf("unknown source type %s", e.Type)
 }

pkg/k8s/kubeconfig/kubeconfig.go

@@ -18,12 +18,15 @@ import (
 	"io"
 	"log"
 
+	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/client-go/tools/clientcmd/api"
 	"sigs.k8s.io/yaml"
 
 	"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
 	"opendev.org/airship/airshipctl/pkg/clusterctl/client"
 	"opendev.org/airship/airshipctl/pkg/document"
 	"opendev.org/airship/airshipctl/pkg/fs"
+	"opendev.org/airship/airshipctl/pkg/util"
 )
 
 const (
@@ -110,10 +113,14 @@ func FromSecret(c client.Interface, o *client.GetKubeconfigOptions) KubeSourceFu
 // FromFile returns KubeSource type, uses path to kubeconfig on FS as source to construct kubeconfig object
 func FromFile(path string, fSys fs.FileSystem) KubeSourceFunc {
 	return func() ([]byte, error) {
+		expandedPath, err := util.ExpandTilde(path)
+		if err != nil {
+			return nil, err
+		}
 		if fSys == nil {
 			fSys = fs.NewDocumentFs()
 		}
-		return fSys.ReadFile(path)
+		return fSys.ReadFile(expandedPath)
 	}
 }
 
@@ -144,6 +151,13 @@ func FromBundle(root string) KubeSourceFunc {
 	}
 }
 
+// FromConfig returns KubeSource type, write passed config as bytes
+func FromConfig(cfg *api.Config) KubeSourceFunc {
+	return func() ([]byte, error) {
+		return clientcmd.Write(*cfg)
+	}
+}
+
 // InjectFileSystem sets fileSystem to be used, mostly to be used for tests
 func InjectFileSystem(fSys fs.FileSystem) Option {
 	return func(k *kubeConfig) {
@@ -193,6 +207,7 @@ func (k *kubeConfig) WriteTempFile(root string) (string, Cleanup, error) {
 	}
 	file, err := k.fileSystem.TempFile(root, KubeconfigPrefix)
 	if err != nil {
+		log.Printf("Failed to write temporary file, error %v", err)
 		return "", nil, err
 	}
 	defer file.Close()

pkg/k8s/kubeconfig/testdata/kubeconfig.yaml

@@ -15,7 +15,6 @@ config:
       cluster: parent_parent_cluster
       user: parent_parent_admin
     name: parent_parent_context
-  current-context: dummy_cluster
   preferences: {}
   users:
   - name: parent_parent_admin

pkg/phase/client.go

@@ -103,7 +103,6 @@ func (p *phase) Executor() (ifc.Executor, error) {
 		WithBundle(p.helper.PhaseBundleRoot()).
 		WithClusterMap(cMap).
 		WithClusterName(p.apiObj.ClusterName).
-		WithPath(p.kubeconfig).
 		WithTempRoot(wd).
 		WithClusterctClient(cctlClient).
 		Build()

pkg/phase/command_test.go

@@ -18,12 +18,14 @@ import (
 	"bytes"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
 	"opendev.org/airship/airshipctl/pkg/config"
+	"opendev.org/airship/airshipctl/pkg/log"
 	"opendev.org/airship/airshipctl/pkg/phase"
 )
 
@@ -296,6 +298,7 @@ func TestPlanListCommand(t *testing.T) {
 }
 
 func TestPlanRunCommand(t *testing.T) {
+	log.Init(true, os.Stdout)
 	testErr := fmt.Errorf(testFactoryErr)
 	testCases := []struct {
 		name        string
@@ -343,8 +346,7 @@ func TestPlanRunCommand(t *testing.T) {
 				}
 				return conf, nil
 			},
-			expectedErr: `Error events received on channel, errors are:
+			expectedErr: `context "ephemeral-cluster" does not exist`,
-[document filtered by selector [Group="airshipit.org", Version="v1alpha1", Kind="KubeConfig"] found no documents]`,
 		},
 	}
 	for _, tc := range testCases {
@@ -361,7 +363,7 @@ func TestPlanRunCommand(t *testing.T) {
 			err := cmd.RunE()
 			if tt.expectedErr != "" {
 				require.Error(t, err)
-				assert.Equal(t, tt.expectedErr, err.Error())
+				assert.Contains(t, err.Error(), tt.expectedErr)
 			} else {
 				assert.NoError(t, err)
 			}

pkg/phase/executors/k8s_applier.go

@@ -117,7 +117,7 @@ func (e *KubeApplierExecutor) prepareApplier(ch chan events.Event) (*k8sapplier.
 	}
 	// set up cleanup only if all calls up to here were successful
 	e.cleanup = cleanup
-	log.Debugf("Using kubeconfig at '%s' and context '%s'", path, context)
+	log.Printf("Using kubeconfig at '%s' and context '%s'", path, context)
 	factory := utils.FactoryFromKubeConfig(path, context)
 	return k8sapplier.NewApplier(ch, factory), bundle, nil
 }

pkg/phase/executors/k8s_applier_test.go

@@ -178,7 +178,7 @@ func TestKubeApplierExecutorRun(t *testing.T) {
 		},
 		{
 			name:          "error cluster not defined",
-			containsErr:   "cluster  is not defined in in cluster map",
+			containsErr:   "is not defined in cluster map",
 			helper:        makeDefaultHelper(t, "../../k8s/applier/testdata", defaultMetadataPath),
 			bundleFactory: testBundleFactory("../../k8s/applier/testdata/source_bundle"),
 			kubeconf:      testKubeconfig(testValidKubeconfig),

pkg/phase/helper_test.go

@@ -319,9 +319,19 @@ func TestHelperClusterMapAPI(t *testing.T) {
 				Map: map[string]*airshipv1.Cluster{
 					"target": {
 						Parent: "ephemeral",
-						DynamicKubeConfig: false,
+						Sources: []airshipv1.KubeconfigSource{
+							{
+								Type: airshipv1.KubeconfigSourceTypeBundle,
+							},
+						},
+					},
+					"ephemeral": {
+						Sources: []airshipv1.KubeconfigSource{
+							{
+								Type: airshipv1.KubeconfigSourceTypeBundle,
+							},
+						},
 					},
-					"ephemeral": {},
 				},
 			},
 			config: testConfig,

pkg/phase/testdata/phases/cluster-map.yaml

@@ -7,5 +7,9 @@ metadata:
   name: main-map
 map:
   target-cluster:
-    parent: ephemeral-cluster
+    parent: ephemeral
-  ephemeral-cluster: {}
+    kubeconfigSources:
+    - type: bundle
+  ephemeral-cluster:
+    kubeconfigSources:
+    - type: bundle

pkg/phase/testdata/valid_site/phases/cluster_map.yaml

@@ -5,5 +5,8 @@ metadata:
 map:
   target:
     parent: ephemeral
-    dynamicKubeConf: false
+    kubeconfigSources:
-  ephemeral: {}
+    - type: bundle
+  ephemeral:
+    kubeconfigSources:
+    - type: bundle

tools/deployment/26_deploy_capi_ephemeral_node.sh

@@ -54,7 +54,7 @@ if [ "$PROVIDER" = "metal3" ]; then
     airshipctl phase run clusterctl-init-ephemeral --debug
 else
     echo "Deploy cluster-api components to ephemeral node"
-    airshipctl phase run clusterctl-init-ephemeral --debug --kubeconfig ${KUBECONFIG}
+    airshipctl phase run clusterctl-init-ephemeral --debug
     kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get pods -A
 fi
 

tools/document/validate_site_docs.sh

@@ -24,6 +24,8 @@ set -xe
 : ${SITE:="test-workload"}
 : ${CONTEXT:="kind-airship"}
 : ${AIRSHIPKUBECONFIG:="${HOME}/.airship/kubeconfig"}
+: ${AIRSHIPKUBECONFIG_BACKUP:="${AIRSHIPKUBECONFIG}-backup"}
+
 
 : ${KUBECTL:="/usr/local/bin/kubectl"}
 TMP=$(mktemp -d)
@@ -37,11 +39,10 @@ else
 fi
 
 : ${AIRSHIPCONFIG:="${TMP}/config"}
-: ${KUBECONFIG:="${TMP}/kubeconfig"}
 : ${AIRSHIPCTL:="${AIRSHIPCTL_DEFAULT}"}
-ACTL="${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} --kubeconfig ${KUBECONFIG}"
+ACTL="${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG}"
 
-export KUBECONFIG
+export KUBECONFIG="${AIRSHIPKUBECONFIG}"
 
 # TODO: use `airshipctl config` to do this once all the needed knobs are exposed
 # The non-default parts are to set the targetPath appropriately,
@@ -83,9 +84,20 @@ EOL
 function cleanup() {
   ${KIND} delete cluster --name $CLUSTER
   rm -rf ${TMP}
+
+  if [ -f "${AIRSHIPKUBECONFIG_BACKUP}" ]; then
+    echo "Restoring a backup copy of kubeconfig"
+    cp "${AIRSHIPKUBECONFIG_BACKUP}" "${AIRSHIPKUBECONFIG}"
+  fi
 }
 trap cleanup EXIT
 
+if [ -f "${AIRSHIPKUBECONFIG}" ]; then
+  echo "Making a backup copy of kubeconfig"
+  cp "${AIRSHIPKUBECONFIG}" "${AIRSHIPKUBECONFIG_BACKUP}"
+fi
+
+
 generate_airshipconf "default"
 
 phase_plans=$(airshipctl --airshipconf ${AIRSHIPCONFIG} plan list | grep "PhasePlan" | awk -F '/' '{print $2}' | awk '{print $1}')
@@ -96,8 +108,6 @@ for plan in $phase_plans; do
   for cluster in $cluster_list; do
     echo -e "\n**** Rendering phases for cluster: ${cluster}"
 
-    # Since we'll be mucking with the kubeconfig - make a copy of it and muck with the copy
-    cp ${AIRSHIPKUBECONFIG} ${KUBECONFIG}
     export CLUSTER="${cluster}"
 
     # Start a fresh, empty kind cluster for validating documents
@@ -120,13 +130,14 @@ for plan in $phase_plans; do
       # e.g., load CRDs from initinfra first, so they're present when validating later phases
       ${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} phase render ${phase} -s executor -k CustomResourceDefinition >${TMP}/${phase}-crds.yaml
       if [ -s ${TMP}/${phase}-crds.yaml ]; then
-        ${KUBECTL} --context ${CLUSTER} --kubeconfig ${KUBECONFIG} apply -f ${TMP}/${phase}-crds.yaml
+        ${KUBECTL} --context ${CLUSTER} apply -f ${TMP}/${phase}-crds.yaml
       fi
 
       # step 2: dry-run the entire phase
       ${ACTL} phase run --dry-run ${phase}
     done
-
+    # Delete cluster kubeconfig
+    rm ${KUBECONFIG}
     ${KIND} delete cluster --name $CLUSTER
   done
 done