From 6207e2c24d48c5d87f64e33b8dc3ede02ede9f51 Mon Sep 17 00:00:00 2001 From: Kostiantyn Kalynovskyi Date: Fri, 5 Mar 2021 06:12:21 +0000 Subject: [PATCH] Add multiple kubeconifg sources to ClusterMap Change ClusterMap API object to support multiple kubeconfig sources for a cluster. If one kubeconfig source fails, kubeconfig builder will not fail and move on to the next one. This behaviour will allow to support cases when ephemeral cluster is not accesible anymore or when target cluster is not yet accessible. For more information please read issue #460 in airshipctl github Relates-To: #460 Related-To: #460 Change-Id: I7cd32f78cd7c4ad8814eac357424c24216f40d76 --- cmd/phase/run.go | 5 - .../TestRunGoldenOutput/run-with-help.golden | 1 - docs/source/cli/airshipctl_phase.md | 2 +- docs/source/cli/airshipctl_phase_run.md | 1 - docs/source/cli/airshipctl_phase_validate.md | 5 +- manifests/phases/cluster-map.yaml | 23 +- pkg/api/v1alpha1/cluster_map_types.go | 51 ++- pkg/api/v1alpha1/zz_generated.deepcopy.go | 102 +++++- pkg/cluster/clustermap/errors.go | 2 +- pkg/cluster/clustermap/map.go | 62 +--- pkg/cluster/clustermap/map_test.go | 67 ++-- pkg/k8s/kubeconfig/builder.go | 317 +++++++++++------- pkg/k8s/kubeconfig/builder_test.go | 258 +++++++------- pkg/k8s/kubeconfig/errors.go | 36 +- pkg/k8s/kubeconfig/kubeconfig.go | 17 +- pkg/k8s/kubeconfig/testdata/kubeconfig.yaml | 1 - pkg/phase/client.go | 1 - pkg/phase/command_test.go | 8 +- pkg/phase/executors/k8s_applier.go | 2 +- pkg/phase/executors/k8s_applier_test.go | 2 +- pkg/phase/helper_test.go | 16 +- pkg/phase/testdata/phases/cluster-map.yaml | 8 +- .../valid_site/phases/cluster_map.yaml | 7 +- .../26_deploy_capi_ephemeral_node.sh | 2 +- tools/document/validate_site_docs.sh | 25 +- 25 files changed, 625 insertions(+), 396 deletions(-) diff --git a/cmd/phase/run.go b/cmd/phase/run.go index 345522255..540a0b247 100644 --- a/cmd/phase/run.go +++ b/cmd/phase/run.go @@ -60,11 +60,6 @@ func NewRunCommand(cfgFactory config.Factory) *cobra.Command { "wait-timeout", 0, "wait timeout") - flags.StringVar( - &p.Options.Kubeconfig, - "kubeconfig", - "", - "Path to kubeconfig associated with site being managed") flags.BoolVar( &p.Options.Progress, "progress", diff --git a/cmd/phase/testdata/TestRunGoldenOutput/run-with-help.golden b/cmd/phase/testdata/TestRunGoldenOutput/run-with-help.golden index e13f4e122..f1b50eae8 100644 --- a/cmd/phase/testdata/TestRunGoldenOutput/run-with-help.golden +++ b/cmd/phase/testdata/TestRunGoldenOutput/run-with-help.golden @@ -12,6 +12,5 @@ airshipctl phase run ephemeral-control-plane Flags: --dry-run simulate phase execution -h, --help help for run - --kubeconfig string Path to kubeconfig associated with site being managed --progress show progress --wait-timeout duration wait timeout diff --git a/docs/source/cli/airshipctl_phase.md b/docs/source/cli/airshipctl_phase.md index 635aa3948..71342604e 100644 --- a/docs/source/cli/airshipctl_phase.md +++ b/docs/source/cli/airshipctl_phase.md @@ -28,5 +28,5 @@ such as getting list and applying specific one. * [airshipctl phase render](airshipctl_phase_render.md) - Render phase documents from model * [airshipctl phase run](airshipctl_phase_run.md) - Run phase * [airshipctl phase tree](airshipctl_phase_tree.md) - Tree view of kustomize entrypoints of phase -* [airshipctl phase validate](airshipctl_phase_validate.md) - Validate phase +* [airshipctl phase validate](airshipctl_phase_validate.md) - Assert that a phase is valid diff --git a/docs/source/cli/airshipctl_phase_run.md b/docs/source/cli/airshipctl_phase_run.md index 9c0fdb081..29d1dae0e 100644 --- a/docs/source/cli/airshipctl_phase_run.md +++ b/docs/source/cli/airshipctl_phase_run.md @@ -24,7 +24,6 @@ airshipctl phase run ephemeral-control-plane ``` --dry-run simulate phase execution -h, --help help for run - --kubeconfig string Path to kubeconfig associated with site being managed --progress show progress --wait-timeout duration wait timeout ``` diff --git a/docs/source/cli/airshipctl_phase_validate.md b/docs/source/cli/airshipctl_phase_validate.md index 687af7e9e..4afe0ec48 100644 --- a/docs/source/cli/airshipctl_phase_validate.md +++ b/docs/source/cli/airshipctl_phase_validate.md @@ -4,7 +4,8 @@ Assert that a phase is valid ### Synopsis -Command which would validate that the phase contains the required documents to run the phase +Command which would validate that the phase contains the required documents to run the phase. + ``` airshipctl phase validate PHASE_NAME [flags] @@ -22,7 +23,7 @@ airshipctl phase validate initinfra ### Options ``` - -h, --help help for run + -h, --help help for validate ``` ### Options inherited from parent commands diff --git a/manifests/phases/cluster-map.yaml b/manifests/phases/cluster-map.yaml index d53fe7e30..add586144 100644 --- a/manifests/phases/cluster-map.yaml +++ b/manifests/phases/cluster-map.yaml @@ -8,4 +8,25 @@ metadata: map: target-cluster: parent: ephemeral-cluster - ephemeral-cluster: {} \ No newline at end of file + kubeconfigSources: + - type: "filesystem" + filesystem: + path: ~/.airship/kubeconfig + contextName: target-cluster + - type: "clusterAPI" + clusterAPI: + clusterNamespacedName: + name: target-cluster + namespace: default + - type: "bundle" + bundle: + contextName: target-cluster + ephemeral-cluster: + kubeconfigSources: + - type: "filesystem" + filesystem: + path: ~/.airship/kubeconfig + contextName: ephemeral-cluster + - type: "bundle" + bundle: + contextName: ephemeral-cluster diff --git a/pkg/api/v1alpha1/cluster_map_types.go b/pkg/api/v1alpha1/cluster_map_types.go index 6b954f0a9..990d99db4 100644 --- a/pkg/api/v1alpha1/cluster_map_types.go +++ b/pkg/api/v1alpha1/cluster_map_types.go @@ -32,19 +32,50 @@ type ClusterMap struct { type Cluster struct { // Parent is a key in ClusterMap.Map that identifies the name of the parent(management) cluster Parent string `json:"parent,omitempty"` - // DynamicKubeConfig kubeconfig allows to get kubeconfig from parent cluster, instead - // expecting it to be in document bundle. Parent kubeconfig will be used to get kubeconfig - DynamicKubeConfig bool `json:"dynamicKubeConf,omitempty"` // KubeconfigContext is the context in kubeconfig, default is equals to clusterMap key - KubeconfigContext string `json:"kubeconfigContext,omitempty"` - // ClusterAPIRef references to Cluster API cluster resources - ClusterAPIRef ClusterAPIRef `json:"clusterAPIRef,omitempty"` + Sources []KubeconfigSource `json:"kubeconfigSources"` } -// ClusterAPIRef will be used to find cluster object in kubernetes parent cluster -type ClusterAPIRef struct { - Name string - Namespace string +// KubeconfigSource describes source of the kubeconfig +type KubeconfigSource struct { + Type KubeconfigSourceType `json:"type"` + FileSystem KubeconfigSourceFilesystem `json:"filesystem,omitempty"` + Bundle KubeconfigSourceBundle `json:"bundle,omitempty"` + ClusterAPI KubeconfigSourceClusterAPI `json:"clusterAPI,omitempty"` +} + +// KubeconfigSourceType type of source +type KubeconfigSourceType string + +const ( + // KubeconfigSourceTypeFilesystem is used when you want kubeconfig to be taken from local filesystem + KubeconfigSourceTypeFilesystem KubeconfigSourceType = "filesystem" + // KubeconfigSourceTypeBundle use config document bundle to get kubeconfig + KubeconfigSourceTypeBundle KubeconfigSourceType = "bundle" + // KubeconfigSourceTypeClusterAPI use ClusterAPI to get kubeconfig, parent cluster must be specified + KubeconfigSourceTypeClusterAPI KubeconfigSourceType = "clusterAPI" +) + +// KubeconfigSourceFilesystem get kubeconfig from filesystem path +type KubeconfigSourceFilesystem struct { + Path string `json:"path,omitempty"` + Context string `json:"contextName,omitempty"` +} + +// KubeconfigSourceClusterAPI get kubeconfig from clusterAPI parent cluster +type KubeconfigSourceClusterAPI struct { + NamespacedName `json:"clusterNamespacedName,omitempty"` +} + +// KubeconfigSourceBundle get kubeconfig from bundle +type KubeconfigSourceBundle struct { + Context string `json:"contextName,omitempty"` +} + +// NamespacedName is a name combined with namespace to uniquely identify objects +type NamespacedName struct { + Name string `json:"name,omitempty"` + Namespace string `json:"namespace,omitempty"` } // DefaultClusterMap can be used to safely unmarshal ClusterMap object without nil pointers diff --git a/pkg/api/v1alpha1/zz_generated.deepcopy.go b/pkg/api/v1alpha1/zz_generated.deepcopy.go index 744145839..cd23690eb 100644 --- a/pkg/api/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/api/v1alpha1/zz_generated.deepcopy.go @@ -209,7 +209,11 @@ func (in *BootstrapContainer) DeepCopy() *BootstrapContainer { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Cluster) DeepCopyInto(out *Cluster) { *out = *in - out.ClusterAPIRef = in.ClusterAPIRef + if in.Sources != nil { + in, out := &in.Sources, &out.Sources + *out = make([]KubeconfigSource, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Cluster. @@ -222,21 +226,6 @@ func (in *Cluster) DeepCopy() *Cluster { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *ClusterAPIRef) DeepCopyInto(out *ClusterAPIRef) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterAPIRef. -func (in *ClusterAPIRef) DeepCopy() *ClusterAPIRef { - if in == nil { - return nil - } - out := new(ClusterAPIRef) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ClusterMap) DeepCopyInto(out *ClusterMap) { *out = *in @@ -252,7 +241,7 @@ func (in *ClusterMap) DeepCopyInto(out *ClusterMap) { } else { in, out := &val, &outVal *out = new(Cluster) - **out = **in + (*in).DeepCopyInto(*out) } (*out)[key] = outVal } @@ -568,6 +557,70 @@ func (in *KubeConfig) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigSource) DeepCopyInto(out *KubeconfigSource) { + *out = *in + out.FileSystem = in.FileSystem + out.Bundle = in.Bundle + out.ClusterAPI = in.ClusterAPI +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSource. +func (in *KubeconfigSource) DeepCopy() *KubeconfigSource { + if in == nil { + return nil + } + out := new(KubeconfigSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigSourceBundle) DeepCopyInto(out *KubeconfigSourceBundle) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceBundle. +func (in *KubeconfigSourceBundle) DeepCopy() *KubeconfigSourceBundle { + if in == nil { + return nil + } + out := new(KubeconfigSourceBundle) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigSourceClusterAPI) DeepCopyInto(out *KubeconfigSourceClusterAPI) { + *out = *in + out.NamespacedName = in.NamespacedName +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceClusterAPI. +func (in *KubeconfigSourceClusterAPI) DeepCopy() *KubeconfigSourceClusterAPI { + if in == nil { + return nil + } + out := new(KubeconfigSourceClusterAPI) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *KubeconfigSourceFilesystem) DeepCopyInto(out *KubeconfigSourceFilesystem) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new KubeconfigSourceFilesystem. +func (in *KubeconfigSourceFilesystem) DeepCopy() *KubeconfigSourceFilesystem { + if in == nil { + return nil + } + out := new(KubeconfigSourceFilesystem) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *KubernetesApply) DeepCopyInto(out *KubernetesApply) { *out = *in @@ -609,6 +662,21 @@ func (in *MoveOptions) DeepCopy() *MoveOptions { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *NamespacedName) DeepCopyInto(out *NamespacedName) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamespacedName. +func (in *NamespacedName) DeepCopy() *NamespacedName { + if in == nil { + return nil + } + out := new(NamespacedName) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Phase) DeepCopyInto(out *Phase) { *out = *in diff --git a/pkg/cluster/clustermap/errors.go b/pkg/cluster/clustermap/errors.go index fa48847fc..080851008 100644 --- a/pkg/cluster/clustermap/errors.go +++ b/pkg/cluster/clustermap/errors.go @@ -37,5 +37,5 @@ type ErrClusterNotInMap struct { } func (e ErrClusterNotInMap) Error() string { - return fmt.Sprintf("cluster %s is not defined in in cluster map %v", e.Child, e.Map) + return fmt.Sprintf("cluster '%s' is not defined in cluster map %v", e.Child, e.Map) } diff --git a/pkg/cluster/clustermap/map.go b/pkg/cluster/clustermap/map.go index 46d1ffd9e..4d0d4971f 100644 --- a/pkg/cluster/clustermap/map.go +++ b/pkg/cluster/clustermap/map.go @@ -16,7 +16,6 @@ package clustermap import ( "opendev.org/airship/airshipctl/pkg/api/v1alpha1" - "opendev.org/airship/airshipctl/pkg/log" ) // DefaultClusterAPIObjNamespace is a default namespace used for cluster-api cluster object @@ -28,9 +27,8 @@ const DefaultClusterAPIObjNamespace = "default" type ClusterMap interface { ParentCluster(string) (string, error) AllClusters() []string - DynamicKubeConfig(string) bool ClusterKubeconfigContext(string) (string, error) - ClusterAPIRef(string) (ClusterAPIRef, error) + Sources(string) ([]v1alpha1.KubeconfigSource, error) } // clusterMap allows to view clusters and relationship between them @@ -57,16 +55,6 @@ func (cm clusterMap) ParentCluster(child string) (string, error) { return currentCluster.Parent, nil } -// DynamicKubeConfig check if dynamic kubeconfig is enabled for the child cluster -func (cm clusterMap) DynamicKubeConfig(child string) bool { - childCluster, exist := cm.apiMap.Map[child] - if !exist { - log.Debugf("cluster %s is not defined in cluster map %v", child, cm.apiMap) - return false - } - return childCluster.DynamicKubeConfig -} - // AllClusters returns all clusters in a map func (cm clusterMap) AllClusters() []string { clusters := []string{} @@ -76,49 +64,21 @@ func (cm clusterMap) AllClusters() []string { return clusters } -// ClusterAPIRef helps to find corresponding cluster-api Cluster object in kubernetes cluster -type ClusterAPIRef struct { - Name string - Namespace string -} - -// ClusterAPIRef maps a clusterapi name and namespace for a given cluster -func (cm clusterMap) ClusterAPIRef(clusterName string) (ClusterAPIRef, error) { - clstr, ok := cm.apiMap.Map[clusterName] - if !ok { - return ClusterAPIRef{}, ErrClusterNotInMap{Child: clusterName, Map: cm.apiMap} - } - - name := clstr.ClusterAPIRef.Name - namespace := clstr.ClusterAPIRef.Namespace - - if name == "" { - name = clusterName - } - - if namespace == "" { - namespace = DefaultClusterAPIObjNamespace - } - - return ClusterAPIRef{ - Name: name, - Namespace: namespace, - }, nil -} - // ClusterKubeconfigContext returns name of the context in kubeconfig corresponding to a given cluster func (cm clusterMap) ClusterKubeconfigContext(clusterName string) (string, error) { - cluster, exists := cm.apiMap.Map[clusterName] + _, exists := cm.apiMap.Map[clusterName] if !exists { return "", ErrClusterNotInMap{Map: cm.apiMap, Child: clusterName} } - kubeContext := cluster.KubeconfigContext - // if kubeContext is still empty, set it to clusterName - if cluster.KubeconfigContext == "" { - kubeContext = clusterName - } - - return kubeContext, nil + return clusterName, nil +} + +func (cm clusterMap) Sources(clusterName string) ([]v1alpha1.KubeconfigSource, error) { + cluster, ok := cm.apiMap.Map[clusterName] + if !ok { + return nil, ErrClusterNotInMap{Child: clusterName, Map: cm.apiMap} + } + return cluster.Sources, nil } diff --git a/pkg/cluster/clustermap/map_test.go b/pkg/cluster/clustermap/map_test.go index 90b8e6693..f5a31d334 100644 --- a/pkg/cluster/clustermap/map_test.go +++ b/pkg/cluster/clustermap/map_test.go @@ -28,28 +28,40 @@ func TestClusterMap(t *testing.T) { targetCluster := "target" ephemeraCluster := "ephemeral" workloadCluster := "workload" - workloadClusterKubeconfigContext := "different-workload-context" workloadClusterNoParent := "workload without parent" workloadClusterAPIRefName := "workload-cluster-api" workloadClusterAPIRefNamespace := "some-namespace" apiMap := &v1alpha1.ClusterMap{ Map: map[string]*v1alpha1.Cluster{ targetCluster: { - Parent: ephemeraCluster, - DynamicKubeConfig: false, + Parent: ephemeraCluster, + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeBundle, + }, + }, }, ephemeraCluster: {}, workloadCluster: { - Parent: targetCluster, - DynamicKubeConfig: true, - KubeconfigContext: workloadClusterKubeconfigContext, - ClusterAPIRef: v1alpha1.ClusterAPIRef{ - Name: workloadClusterAPIRefName, - Namespace: workloadClusterAPIRefNamespace, + Parent: targetCluster, + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeClusterAPI, + ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{ + NamespacedName: v1alpha1.NamespacedName{ + Name: workloadClusterAPIRefName, + Namespace: workloadClusterAPIRefNamespace, + }, + }, + }, }, }, workloadClusterNoParent: { - DynamicKubeConfig: true, + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeClusterAPI, + }, + }, }, }, } @@ -69,16 +81,6 @@ func TestClusterMap(t *testing.T) { assert.Equal(t, "", parent) }) - t.Run("not dynamic kubeconf target", func(t *testing.T) { - dynamic := cMap.DynamicKubeConfig(targetCluster) - assert.False(t, dynamic) - }) - - t.Run("dynamic kubeconf workload", func(t *testing.T) { - dynamic := cMap.DynamicKubeConfig(workloadCluster) - assert.True(t, dynamic) - }) - t.Run("target parent", func(t *testing.T) { parent, err := cMap.ParentCluster(workloadCluster) assert.NoError(t, err) @@ -97,12 +99,6 @@ func TestClusterMap(t *testing.T) { }) t.Run("kubeconfig context", func(t *testing.T) { - kubeContext, err := cMap.ClusterKubeconfigContext(workloadCluster) - assert.NoError(t, err) - assert.Equal(t, workloadClusterKubeconfigContext, kubeContext) - }) - - t.Run("kubeconfig default context", func(t *testing.T) { kubeContext, err := cMap.ClusterKubeconfigContext(targetCluster) assert.NoError(t, err) assert.Equal(t, targetCluster, kubeContext) @@ -113,22 +109,15 @@ func TestClusterMap(t *testing.T) { assert.Error(t, err) }) - t.Run("ClusterAPI ref name and namespace defaults", func(t *testing.T) { - ref, err := cMap.ClusterAPIRef(workloadClusterNoParent) + t.Run("sources match", func(t *testing.T) { + sources, err := cMap.Sources(workloadCluster) assert.NoError(t, err) - assert.Equal(t, clustermap.DefaultClusterAPIObjNamespace, ref.Namespace) - assert.Equal(t, workloadClusterNoParent, ref.Name) + expectedSources := apiMap.Map[workloadCluster].Sources + assert.Equal(t, expectedSources, sources) }) - t.Run("ClusterAPI ref name and namespace", func(t *testing.T) { - ref, err := cMap.ClusterAPIRef(workloadCluster) - assert.NoError(t, err) - assert.Equal(t, workloadClusterAPIRefNamespace, ref.Namespace) - assert.Equal(t, workloadClusterAPIRefName, ref.Name) - }) - - t.Run("ClusterAPI ref error", func(t *testing.T) { - _, err := cMap.ClusterAPIRef("doesn't exist") + t.Run("sources no cluster found", func(t *testing.T) { + _, err := cMap.Sources("does not exist") assert.Error(t, err) }) } diff --git a/pkg/k8s/kubeconfig/builder.go b/pkg/k8s/kubeconfig/builder.go index 7f66fe671..2dc76b1a6 100644 --- a/pkg/k8s/kubeconfig/builder.go +++ b/pkg/k8s/kubeconfig/builder.go @@ -15,8 +15,7 @@ package kubeconfig import ( - "bytes" - "path/filepath" + "fmt" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/clientcmd/api" @@ -24,10 +23,8 @@ import ( "opendev.org/airship/airshipctl/pkg/api/v1alpha1" "opendev.org/airship/airshipctl/pkg/cluster/clustermap" "opendev.org/airship/airshipctl/pkg/clusterctl/client" - "opendev.org/airship/airshipctl/pkg/config" "opendev.org/airship/airshipctl/pkg/fs" "opendev.org/airship/airshipctl/pkg/log" - "opendev.org/airship/airshipctl/pkg/util" ) // KubeconfigDefaultFileName is a default name for kubeconfig @@ -35,13 +32,14 @@ const KubeconfigDefaultFileName = "kubeconfig" // NewBuilder returns instance of kubeconfig builder. func NewBuilder() *Builder { - return &Builder{} + return &Builder{ + siteKubeconf: emptyConfig(), + } } // Builder is an object that allows to build a kubeconfig based on various provided sources // such as path to kubeconfig, path to bundle that should contain kubeconfig and parent cluster type Builder struct { - path string bundlePath string clusterName string root string @@ -49,12 +47,7 @@ type Builder struct { clusterMap clustermap.ClusterMap clusterctlClient client.Interface fs fs.FileSystem -} - -// WithPath allows to set path to prexisting kubeconfig -func (b *Builder) WithPath(filePath string) *Builder { - b.path = filePath - return b + siteKubeconf *api.Config } // WithBundle allows to set path to bundle that should contain kubeconfig api object @@ -94,129 +87,217 @@ func (b *Builder) WithFilesytem(fs fs.FileSystem) *Builder { return b } -// Build builds a kubeconfig interface to be used +// Build site kubeconfig, ignores, but logs, errors that happen when building individual +// kubeconfigs. We need this behavior because, some clusters may not yet be deployed +// and their kubeconfig is inaccessible yet, but will be accessible at later phases +// If builder can't build kubeconfig for specific cluster, its context will not be present +// in final kubeconfig. User of kubeconfig, will receive error stating that context doesn't exist func (b *Builder) Build() Interface { - switch { - case b.path != "": - return NewKubeConfig(FromFile(b.path, b.fs), InjectFilePath(b.path, b.fs), InjectTempRoot(b.root)) - case b.fromParent(): - // TODO consider adding various drivers to source kubeconfig from - // Also consider accumulating different kubeconfigs, and returning one single - // large file, so that every executor has access to all parent clusters. - return NewKubeConfig(b.buildClusterctlFromParent, InjectTempRoot(b.root), InjectFileSystem(b.fs)) - case b.bundlePath != "": - return NewKubeConfig(FromBundle(b.bundlePath), InjectTempRoot(b.root), InjectFileSystem(b.fs)) - default: - // return default path to kubeconfig file in airship workdir - path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, KubeconfigDefaultFileName) - return NewKubeConfig(FromFile(path, b.fs), InjectFilePath(path, b.fs), InjectTempRoot(b.root)) - } + return NewKubeConfig(b.build, InjectFileSystem(b.fs), InjectTempRoot(b.root)) } -// fromParent checks if we should get kubeconfig from parent cluster secret -func (b *Builder) fromParent() bool { - if b.clusterMap == nil { - return false - } - return b.clusterMap.DynamicKubeConfig(b.clusterName) -} - -func (b *Builder) buildClusterctlFromParent() ([]byte, error) { - currentCluster := b.clusterName - log.Printf("current cluster name is '%s'", - currentCluster) - parentCluster, err := b.clusterMap.ParentCluster(currentCluster) - if err != nil { - return nil, err - } - - parentKubeconfig := b.WithClusterName(parentCluster).Build() - - f, cleanup, err := parentKubeconfig.GetFile() - if err != nil { - return nil, err - } - defer cleanup() - - parentCtx, err := b.clusterMap.ClusterKubeconfigContext(parentCluster) - if err != nil { - return nil, err - } - - clusterAPIRef, err := b.clusterMap.ClusterAPIRef(currentCluster) - if err != nil { - return nil, err - } - - if b.clusterctlClient == nil { - b.clusterctlClient, err = client.NewClient("", log.DebugEnabled(), v1alpha1.DefaultClusterctl()) - if err != nil { +func (b *Builder) build() ([]byte, error) { + for _, clusterID := range b.clusterMap.AllClusters() { + log.Printf("Getting kubeconfig for cluster '%s'", clusterID) + // buildOne merges context into site kubeconfig + _, _, err := b.buildOne(clusterID) + if IsErrAllSourcesFailedErr(err) { + log.Printf("All kubeconfig sources failed for cluster '%s', error '%v', skipping it", + clusterID, err) + continue + } else if err != nil { return nil, err } } - - log.Printf("Getting child kubeconfig from parent, parent context '%s', parent kubeconfing '%s'", - parentCtx, f) - - stringChild, err := b.clusterctlClient.GetKubeconfig(&client.GetKubeconfigOptions{ - ParentKubeconfigPath: f, - ParentKubeconfigContext: parentCtx, - ManagedClusterNamespace: clusterAPIRef.Namespace, - ManagedClusterName: clusterAPIRef.Name, - }) - if err != nil { - return nil, err + // Set current context to clustername if it was provided + if b.clusterName != "" { + kubeContext, err := b.clusterMap.ClusterKubeconfigContext(b.clusterName) + if err != nil { + return nil, err + } + b.siteKubeconf.CurrentContext = kubeContext } - - buf := bytes.NewBuffer([]byte{}) - - err = parentKubeconfig.Write(buf) - if err != nil { - return nil, err - } - - parentObj, err := clientcmd.Load(buf.Bytes()) - if err != nil { - return nil, err - } - - childObj, err := clientcmd.Load([]byte(stringChild)) - if err != nil { - return nil, err - } - - childCtx, err := b.clusterMap.ClusterKubeconfigContext(currentCluster) - if err != nil { - return nil, err - } - log.Printf("Merging '%s' cluster kubeconfig into '%s' cluster kubeconfig", - currentCluster, parentCluster) - - return b.mergeOneContext(childCtx, parentObj, childObj) + return clientcmd.Write(*b.siteKubeconf) } -// merges two kubeconfigs, -func (b *Builder) mergeOneContext(contextOverride string, dst, src *api.Config) ([]byte, error) { - for key, content := range src.AuthInfos { - dst.AuthInfos[key] = content +func (b *Builder) buildOne(clusterID string) (string, *api.Config, error) { + destContext, err := b.clusterMap.ClusterKubeconfigContext(clusterID) + if err != nil { + return "", nil, err } - for key, content := range src.Clusters { - dst.Clusters[key] = content + // use already built kubeconfig context, to avoid doing work multiple times + built, oneKubeconf := b.alreadyBuilt(destContext) + if built { + log.Printf("kubeconfig for cluster '%s' is already built, using it", clusterID) + return destContext, oneKubeconf, nil } - if len(src.Contexts) != 1 { - return nil, &ErrClusterctlKubeconfigWrongContextsCount{ - ContextCount: len(src.Contexts), + sources, err := b.clusterMap.Sources(clusterID) + if err != nil { + return "", nil, err + } + + for _, source := range sources { + oneKubeconf, sourceErr := b.trySource(clusterID, destContext, source) + if sourceErr == nil { + // Merge source context into site kubeconfig + log.Printf("Merging kubecontext for cluster '%s', into site kubeconfig", clusterID) + if err = mergeContextAPI(destContext, destContext, b.siteKubeconf, oneKubeconf); err != nil { + return "", nil, err + } + return destContext, oneKubeconf, err + } + // if error, log it and ignore it. missing problem with one kubeconfig should not + // effect other clusters, which don't depend on it. If they do depend on it, their calls + // will fail because the context will be missing. Combitation with a log message will make + // it clear where the problem is. + log.Printf("received error while trying kubeconfig source for cluster '%s', source type '%s', error '%v'", + clusterID, source.Type, sourceErr) + } + // return empty not nil kubeconfig without error. + return "", nil, &ErrAllSourcesFailed{ClusterName: clusterID} +} + +func (b *Builder) trySource(clusterID, dstContext string, source v1alpha1.KubeconfigSource) (*api.Config, error) { + var getter KubeSourceFunc + // TODO add sourceContext defaults + var sourceContext string + switch source.Type { + case v1alpha1.KubeconfigSourceTypeFilesystem: + getter = FromFile(source.FileSystem.Path, b.fs) + sourceContext = source.FileSystem.Context + case v1alpha1.KubeconfigSourceTypeBundle: + getter = FromBundle(b.bundlePath) + sourceContext = source.Bundle.Context + case v1alpha1.KubeconfigSourceTypeClusterAPI: + getter = b.fromClusterAPI(clusterID, source.ClusterAPI) + default: + // TODO add validation for fast fails to clustermap interface instead of this + return nil, &ErrUknownKubeconfigSourceType{Type: string(source.Type)} + } + kubeBytes, err := getter() + if err != nil { + return nil, err + } + return extractContext(dstContext, sourceContext, kubeBytes) +} + +func (b *Builder) fromClusterAPI(clusterName string, ref v1alpha1.KubeconfigSourceClusterAPI) KubeSourceFunc { + return func() ([]byte, error) { + log.Printf("Getting kubeconfig from cluster API for cluster '%s'", clusterName) + parentCluster, err := b.clusterMap.ParentCluster(clusterName) + if err != nil { + return nil, err + } + + parentContext, parentKubeconf, err := b.buildOne(parentCluster) + if err != nil { + return nil, err + } + + parentKubeconfig := NewKubeConfig(FromConfig(parentKubeconf), InjectFileSystem(b.fs)) + + f, cleanup, err := parentKubeconfig.GetFile() + if err != nil { + return nil, err + } + defer cleanup() + + if b.clusterctlClient == nil { + b.clusterctlClient, err = client.NewClient("", log.DebugEnabled(), v1alpha1.DefaultClusterctl()) + if err != nil { + return nil, err + } + } + + log.Printf("Getting child kubeconfig from parent, parent context '%s', parent kubeconfing '%s'", + parentContext, f) + return FromSecret(b.clusterctlClient, &client.GetKubeconfigOptions{ + ParentKubeconfigPath: f, + ParentKubeconfigContext: parentContext, + ManagedClusterNamespace: ref.Namespace, + ManagedClusterName: ref.Name, + })() + } +} + +func (b *Builder) alreadyBuilt(clusterContext string) (bool, *api.Config) { + kubeconfBytes, err := clientcmd.Write(*b.siteKubeconf) + if err != nil { + log.Debugf("Received error when converting kubeconfig to bytes, ignoring kubeconfig. Error: %v", err) + return false, nil + } + + // resulting and existing context names must be the same, otherwise error will be returned + clusterKubeconfig, err := extractContext(clusterContext, clusterContext, kubeconfBytes) + if err != nil { + log.Debugf("Received error when extacting context, ignoring kubeconfig. Error: %v", err) + return false, nil + } + + return true, clusterKubeconfig +} + +func extractContext(destContext, sourceContext string, src []byte) (*api.Config, error) { + srcKubeconf, err := clientcmd.Load(src) + if err != nil { + return nil, err + } + dstKubeconf := emptyConfig() + return dstKubeconf, mergeContextAPI(destContext, sourceContext, dstKubeconf, srcKubeconf) +} + +// merges two kubeconfigs +func mergeContextAPI(destContext, sourceContext string, dst, src *api.Config) error { + if len(src.Contexts) > 1 && sourceContext == "" { + // When more than one context, we don't know which to choose + return &ErrKubeconfigMergeFailed{ + Message: "kubeconfig has multiple contexts, don't know which to choose, " + + "please specify contextName in clusterMap cluster kubeconfig source", } } - for key, content := range src.Contexts { - if contextOverride == "" { - contextOverride = key + var context *api.Context + context, exists := src.Contexts[sourceContext] + switch { + case exists: + case sourceContext == "" && len(src.Contexts) == 1: + for _, context = range src.Contexts { + log.Debugf("Using context '%v' to merge kubeconfig", context) + } + default: + return &ErrKubeconfigMergeFailed{ + Message: fmt.Sprintf("source context '%s' does not exist in source kubeconfig", sourceContext), } - dst.Contexts[contextOverride] = content } + dst.Contexts[destContext] = context - return clientcmd.Write(*dst) + // TODO design logic to make authinfo keys unique, they can overlap, or human error can occur + user, exists := src.AuthInfos[context.AuthInfo] + if !exists { + return &ErrKubeconfigMergeFailed{ + Message: fmt.Sprintf("user '%s' does not exist in source kubeconfig", context.AuthInfo), + } + } + dst.AuthInfos[context.AuthInfo] = user + + // TODO design logic to make cluster keys unique, they can overlap, or human error can occur + cluster, exists := src.Clusters[context.Cluster] + if !exists { + return &ErrKubeconfigMergeFailed{ + Message: fmt.Sprintf("cluster '%s' does not exist in source kubeconfig", context.Cluster), + } + } + dst.Clusters[context.Cluster] = cluster + + return nil +} + +func emptyConfig() *api.Config { + return &api.Config{ + Contexts: make(map[string]*api.Context), + AuthInfos: make(map[string]*api.AuthInfo), + Clusters: make(map[string]*api.Cluster), + } } diff --git a/pkg/k8s/kubeconfig/builder_test.go b/pkg/k8s/kubeconfig/builder_test.go index eadd8416a..a28f10c7f 100644 --- a/pkg/k8s/kubeconfig/builder_test.go +++ b/pkg/k8s/kubeconfig/builder_test.go @@ -16,22 +16,19 @@ package kubeconfig_test import ( "bytes" - "fmt" - "io/ioutil" "path/filepath" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "k8s.io/client-go/tools/clientcmd" "opendev.org/airship/airshipctl/pkg/api/v1alpha1" "opendev.org/airship/airshipctl/pkg/cluster/clustermap" "opendev.org/airship/airshipctl/pkg/clusterctl/client" - "opendev.org/airship/airshipctl/pkg/config" "opendev.org/airship/airshipctl/pkg/fs" "opendev.org/airship/airshipctl/pkg/k8s/kubeconfig" - "opendev.org/airship/airshipctl/pkg/util" "opendev.org/airship/airshipctl/testutil/clusterctl" testfs "opendev.org/airship/airshipctl/testutil/fs" ) @@ -77,59 +74,17 @@ users: client-key-data: c29tZWNlcnQK` ) -func TestBuilder(t *testing.T) { - t.Run("Only bundle", func(t *testing.T) { - builder := kubeconfig.NewBuilder().WithBundle("testdata") - kube := builder.Build() - require.NotNil(t, kube) - buf := bytes.NewBuffer([]byte{}) - err := kube.Write(buf) - require.NoError(t, err) - // check that kubeconfig contains expected cluster string - assert.Contains(t, buf.String(), "parent_parent_context") - }) - - t.Run("Only filepath", func(t *testing.T) { - builder := kubeconfig.NewBuilder().WithPath("testdata/kubeconfig") - kube := builder.Build() - require.NotNil(t, kube) - buf := bytes.NewBuffer([]byte{}) - err := kube.Write(buf) - require.NoError(t, err) - // check that kubeconfig contains expected cluster string - assert.Contains(t, buf.String(), "dummycluster_ephemeral") - }) - - t.Run("No current cluster, fall to default", func(t *testing.T) { - clusterMap := &v1alpha1.ClusterMap{} - builder := kubeconfig.NewBuilder(). - WithClusterMap(clustermap.NewClusterMap(clusterMap)). - WithClusterName("some-cluster") - kube := builder.Build() - // We should get a default value for cluster since we don't have some-cluster set - actualPath, cleanup, err := kube.GetFile() - require.NoError(t, err) - defer cleanup() - path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, kubeconfig.KubeconfigDefaultFileName) - assert.Equal(t, path, actualPath) - }) - - t.Run("Default source", func(t *testing.T) { - builder := kubeconfig.NewBuilder() - kube := builder.Build() - // When ClusterMap is specified, but it doesn't have cluster-name defined, and no - // other sources provided, - actualPath, cleanup, err := kube.GetFile() - require.NoError(t, err) - defer cleanup() - path := filepath.Join(util.UserHomeDir(), config.AirshipConfigDir, kubeconfig.KubeconfigDefaultFileName) - assert.Equal(t, path, actualPath) - }) -} - func TestBuilderClusterctl(t *testing.T) { - childCluster := "child" - parentCluster := "parent" + childClusterID := "child" + parentClusterID := "parent" + parentParentClusterID := "parent-parent" + // these are values in kubeconfig.cluster + parentCluster := "parent_cluster" + parentParentCluster := "parent_parent_cluster" + childCluster := "child_cluster" + parentUser := "parent_admin" + parentParentUser := "parent_parent_admin" + childUser := "child_user" testBundlePath := "testdata" kubeconfigPath := filepath.Join(testBundlePath, "kubeconfig-12341234") @@ -137,79 +92,114 @@ func TestBuilderClusterctl(t *testing.T) { name string errString string requestedClusterName string - parentClusterName string tempRoot string - clusterMap clustermap.ClusterMap - clusterctlClient client.Interface - fs fs.FileSystem + + expectedContexts, expectedClusters, expectedAuthInfos []string + clusterMap clustermap.ClusterMap + clusterctlClient client.Interface + fs fs.FileSystem }{ { - name: "error no parent context", - errString: fmt.Sprintf("context \"%s\" does not exist", parentCluster), - parentClusterName: parentCluster, - requestedClusterName: childCluster, + name: "success cluster-api not reachable", + requestedClusterName: childClusterID, + expectedContexts: []string{parentClusterID}, + expectedClusters: []string{parentParentCluster}, + expectedAuthInfos: []string{parentParentUser}, clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{ Map: map[string]*v1alpha1.Cluster{ - childCluster: { - Parent: parentCluster, - DynamicKubeConfig: true, + childClusterID: { + Parent: parentClusterID, + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeClusterAPI, + }, + }, }, - parentCluster: { - DynamicKubeConfig: false, + parentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeBundle, + Bundle: v1alpha1.KubeconfigSourceBundle{ + Context: "parent_parent_context", + }, + }, + }, }, }, }), }, { - name: "error dynamic but no parrent", - parentClusterName: parentCluster, - requestedClusterName: childCluster, - errString: "failed to find a parent", + name: "success two clusters", + expectedContexts: []string{parentClusterID, parentParentClusterID}, + expectedClusters: []string{"dummycluster_ephemeral", parentParentCluster}, + expectedAuthInfos: []string{"kubernetes-admin", parentParentUser}, clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{ Map: map[string]*v1alpha1.Cluster{ - childCluster: { - DynamicKubeConfig: true, + parentParentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeBundle, + Bundle: v1alpha1.KubeconfigSourceBundle{ + Context: "parent_parent_context", + }, + }, + }, + }, + parentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeFilesystem, + FileSystem: v1alpha1.KubeconfigSourceFilesystem{ + Path: "testdata/kubeconfig", + Context: "dummy_cluster", + }, + }, + }, }, }, }), }, { - name: "error write temp parent", - parentClusterName: parentCluster, - requestedClusterName: childCluster, - tempRoot: "does not exist anywhere", - errString: "no such file or directory", + name: "success three clusters cluster-api", + expectedContexts: []string{parentClusterID, childClusterID, parentParentClusterID}, + expectedClusters: []string{parentCluster, parentParentCluster, childCluster}, + expectedAuthInfos: []string{parentUser, parentParentUser, childUser}, clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{ Map: map[string]*v1alpha1.Cluster{ - childCluster: { - Parent: parentCluster, - DynamicKubeConfig: true, + childClusterID: { + Parent: parentClusterID, + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeClusterAPI, + ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{ + NamespacedName: v1alpha1.NamespacedName{ + Name: childClusterID, + Namespace: "default", + }, + }, + }, + }, }, - parentCluster: { - DynamicKubeConfig: false, - KubeconfigContext: "dummy_cluster", + parentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeClusterAPI, + ClusterAPI: v1alpha1.KubeconfigSourceClusterAPI{ + NamespacedName: v1alpha1.NamespacedName{ + Name: parentClusterID, + Namespace: "default", + }, + }, + }, + }, + Parent: parentParentClusterID, }, - }, - }), - }, - { - name: "success", - parentClusterName: parentCluster, - requestedClusterName: childCluster, - clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{ - Map: map[string]*v1alpha1.Cluster{ - childCluster: { - Parent: parentCluster, - DynamicKubeConfig: true, - }, - parentCluster: { - DynamicKubeConfig: true, - KubeconfigContext: "parent-custom", - Parent: "parent-parent", - }, - "parent-parent": { - DynamicKubeConfig: false, - KubeconfigContext: "parent_parent_context", + parentParentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeBundle, + }, + }, }, }, }), @@ -230,19 +220,38 @@ func TestBuilderClusterctl(t *testing.T) { } c.On("GetKubeconfig", &client.GetKubeconfigOptions{ ParentKubeconfigPath: kubeconfigPath, - ParentKubeconfigContext: "parent-custom", + ParentKubeconfigContext: parentClusterID, ManagedClusterNamespace: clustermap.DefaultClusterAPIObjNamespace, - ManagedClusterName: childCluster, + ManagedClusterName: childClusterID, }).Once().Return(testKubeconfigString, nil) c.On("GetKubeconfig", &client.GetKubeconfigOptions{ ParentKubeconfigPath: kubeconfigPath, - ParentKubeconfigContext: "parent_parent_context", + ParentKubeconfigContext: parentParentClusterID, ManagedClusterNamespace: clustermap.DefaultClusterAPIObjNamespace, - ManagedClusterName: parentCluster, + ManagedClusterName: parentClusterID, }).Once().Return(testKubeconfigStringSecond, nil) return c }(), }, + { + name: "error requested cluster doesn't exist", + errString: "is not defined in cluster map", + requestedClusterName: "non-existent-cluster", + clusterMap: clustermap.NewClusterMap(&v1alpha1.ClusterMap{ + Map: map[string]*v1alpha1.Cluster{ + parentParentClusterID: { + Sources: []v1alpha1.KubeconfigSource{ + { + Type: v1alpha1.KubeconfigSourceTypeBundle, + Bundle: v1alpha1.KubeconfigSourceBundle{ + Context: "parent_parent_context", + }, + }, + }, + }, + }, + }), + }, } for _, tt := range tests { @@ -268,16 +277,35 @@ func TestBuilderClusterctl(t *testing.T) { assert.Contains(t, err.Error(), tt.errString) assert.Equal(t, "", filePath) } else { - assert.NoError(t, err) + require.NoError(t, err) assert.NotEqual(t, "", filePath) assert.NotNil(t, cleanup) buf := bytes.NewBuffer([]byte{}) err := kube.Write(buf) require.NoError(t, err) - result, err := ioutil.ReadFile("testdata/result-kubeconf") - require.NoError(t, err) - assert.Equal(t, result, buf.Bytes()) + compareResults(t, tt.expectedContexts, tt.expectedClusters, tt.expectedAuthInfos, buf.Bytes()) } }) } } + +func compareResults(t *testing.T, contexts, clusters, authInfos []string, kubeconfBytes []byte) { + t.Helper() + resultKubeconf, err := clientcmd.Load(kubeconfBytes) + require.NoError(t, err) + + assert.Len(t, resultKubeconf.Contexts, len(contexts)) + for _, name := range contexts { + assert.Contains(t, resultKubeconf.Contexts, name) + } + + assert.Len(t, resultKubeconf.AuthInfos, len(authInfos)) + for _, name := range authInfos { + assert.Contains(t, resultKubeconf.AuthInfos, name) + } + + assert.Len(t, resultKubeconf.Clusters, len(clusters)) + for _, name := range clusters { + assert.Contains(t, resultKubeconf.Clusters, name) + } +} diff --git a/pkg/k8s/kubeconfig/errors.go b/pkg/k8s/kubeconfig/errors.go index 78d5a8e0d..12ad3a68c 100644 --- a/pkg/k8s/kubeconfig/errors.go +++ b/pkg/k8s/kubeconfig/errors.go @@ -16,21 +16,35 @@ package kubeconfig import "fmt" -// ErrKubeConfigPathEmpty returned when kubeconfig path is not specified -type ErrKubeConfigPathEmpty struct { +// ErrAllSourcesFailed returned when kubeconfig path is not specified +type ErrAllSourcesFailed struct { + ClusterName string } -func (e *ErrKubeConfigPathEmpty) Error() string { - return "kubeconfig path is not defined" +func (e *ErrAllSourcesFailed) Error() string { + return fmt.Sprintf("all kubeconfig sources failed for cluster '%s'", e.ClusterName) } -// ErrClusterctlKubeconfigWrongContextsCount is returned when clusterctl client returns -// multiple or no contexts in kubeconfig for the child cluster -type ErrClusterctlKubeconfigWrongContextsCount struct { - ContextCount int +// ErrKubeconfigMergeFailed is returned when builder doesn't know which context to merge +type ErrKubeconfigMergeFailed struct { + Message string } -func (e *ErrClusterctlKubeconfigWrongContextsCount) Error() string { - return fmt.Sprintf("clusterctl client returned '%d' contexts in kubeconfig "+ - "context count must exactly one", e.ContextCount) +func (e *ErrKubeconfigMergeFailed) Error() string { + return fmt.Sprintf("failed merging kubeconfig: %s", e.Message) +} + +// IsErrAllSourcesFailedErr returns true if error is of type ErrAllSourcesFailedErr +func IsErrAllSourcesFailedErr(err error) bool { + _, ok := err.(*ErrAllSourcesFailed) + return ok +} + +// ErrUknownKubeconfigSourceType returned type of kubeconfig source is unknown +type ErrUknownKubeconfigSourceType struct { + Type string +} + +func (e *ErrUknownKubeconfigSourceType) Error() string { + return fmt.Sprintf("unknown source type %s", e.Type) } diff --git a/pkg/k8s/kubeconfig/kubeconfig.go b/pkg/k8s/kubeconfig/kubeconfig.go index 57f1d19e0..05745b036 100644 --- a/pkg/k8s/kubeconfig/kubeconfig.go +++ b/pkg/k8s/kubeconfig/kubeconfig.go @@ -18,12 +18,15 @@ import ( "io" "log" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/client-go/tools/clientcmd/api" "sigs.k8s.io/yaml" "opendev.org/airship/airshipctl/pkg/api/v1alpha1" "opendev.org/airship/airshipctl/pkg/clusterctl/client" "opendev.org/airship/airshipctl/pkg/document" "opendev.org/airship/airshipctl/pkg/fs" + "opendev.org/airship/airshipctl/pkg/util" ) const ( @@ -110,10 +113,14 @@ func FromSecret(c client.Interface, o *client.GetKubeconfigOptions) KubeSourceFu // FromFile returns KubeSource type, uses path to kubeconfig on FS as source to construct kubeconfig object func FromFile(path string, fSys fs.FileSystem) KubeSourceFunc { return func() ([]byte, error) { + expandedPath, err := util.ExpandTilde(path) + if err != nil { + return nil, err + } if fSys == nil { fSys = fs.NewDocumentFs() } - return fSys.ReadFile(path) + return fSys.ReadFile(expandedPath) } } @@ -144,6 +151,13 @@ func FromBundle(root string) KubeSourceFunc { } } +// FromConfig returns KubeSource type, write passed config as bytes +func FromConfig(cfg *api.Config) KubeSourceFunc { + return func() ([]byte, error) { + return clientcmd.Write(*cfg) + } +} + // InjectFileSystem sets fileSystem to be used, mostly to be used for tests func InjectFileSystem(fSys fs.FileSystem) Option { return func(k *kubeConfig) { @@ -193,6 +207,7 @@ func (k *kubeConfig) WriteTempFile(root string) (string, Cleanup, error) { } file, err := k.fileSystem.TempFile(root, KubeconfigPrefix) if err != nil { + log.Printf("Failed to write temporary file, error %v", err) return "", nil, err } defer file.Close() diff --git a/pkg/k8s/kubeconfig/testdata/kubeconfig.yaml b/pkg/k8s/kubeconfig/testdata/kubeconfig.yaml index 437b0fcde..e7053e4a5 100644 --- a/pkg/k8s/kubeconfig/testdata/kubeconfig.yaml +++ b/pkg/k8s/kubeconfig/testdata/kubeconfig.yaml @@ -15,7 +15,6 @@ config: cluster: parent_parent_cluster user: parent_parent_admin name: parent_parent_context - current-context: dummy_cluster preferences: {} users: - name: parent_parent_admin diff --git a/pkg/phase/client.go b/pkg/phase/client.go index 79e329a43..a3aaec6f1 100644 --- a/pkg/phase/client.go +++ b/pkg/phase/client.go @@ -103,7 +103,6 @@ func (p *phase) Executor() (ifc.Executor, error) { WithBundle(p.helper.PhaseBundleRoot()). WithClusterMap(cMap). WithClusterName(p.apiObj.ClusterName). - WithPath(p.kubeconfig). WithTempRoot(wd). WithClusterctClient(cctlClient). Build() diff --git a/pkg/phase/command_test.go b/pkg/phase/command_test.go index 39b93e93a..04302ca76 100644 --- a/pkg/phase/command_test.go +++ b/pkg/phase/command_test.go @@ -18,12 +18,14 @@ import ( "bytes" "fmt" "io/ioutil" + "os" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "opendev.org/airship/airshipctl/pkg/config" + "opendev.org/airship/airshipctl/pkg/log" "opendev.org/airship/airshipctl/pkg/phase" ) @@ -296,6 +298,7 @@ func TestPlanListCommand(t *testing.T) { } func TestPlanRunCommand(t *testing.T) { + log.Init(true, os.Stdout) testErr := fmt.Errorf(testFactoryErr) testCases := []struct { name string @@ -343,8 +346,7 @@ func TestPlanRunCommand(t *testing.T) { } return conf, nil }, - expectedErr: `Error events received on channel, errors are: -[document filtered by selector [Group="airshipit.org", Version="v1alpha1", Kind="KubeConfig"] found no documents]`, + expectedErr: `context "ephemeral-cluster" does not exist`, }, } for _, tc := range testCases { @@ -361,7 +363,7 @@ func TestPlanRunCommand(t *testing.T) { err := cmd.RunE() if tt.expectedErr != "" { require.Error(t, err) - assert.Equal(t, tt.expectedErr, err.Error()) + assert.Contains(t, err.Error(), tt.expectedErr) } else { assert.NoError(t, err) } diff --git a/pkg/phase/executors/k8s_applier.go b/pkg/phase/executors/k8s_applier.go index 53083e739..74bb8f826 100644 --- a/pkg/phase/executors/k8s_applier.go +++ b/pkg/phase/executors/k8s_applier.go @@ -117,7 +117,7 @@ func (e *KubeApplierExecutor) prepareApplier(ch chan events.Event) (*k8sapplier. } // set up cleanup only if all calls up to here were successful e.cleanup = cleanup - log.Debugf("Using kubeconfig at '%s' and context '%s'", path, context) + log.Printf("Using kubeconfig at '%s' and context '%s'", path, context) factory := utils.FactoryFromKubeConfig(path, context) return k8sapplier.NewApplier(ch, factory), bundle, nil } diff --git a/pkg/phase/executors/k8s_applier_test.go b/pkg/phase/executors/k8s_applier_test.go index 4117b8525..155f693f1 100644 --- a/pkg/phase/executors/k8s_applier_test.go +++ b/pkg/phase/executors/k8s_applier_test.go @@ -178,7 +178,7 @@ func TestKubeApplierExecutorRun(t *testing.T) { }, { name: "error cluster not defined", - containsErr: "cluster is not defined in in cluster map", + containsErr: "is not defined in cluster map", helper: makeDefaultHelper(t, "../../k8s/applier/testdata", defaultMetadataPath), bundleFactory: testBundleFactory("../../k8s/applier/testdata/source_bundle"), kubeconf: testKubeconfig(testValidKubeconfig), diff --git a/pkg/phase/helper_test.go b/pkg/phase/helper_test.go index 5e427f662..0d41a552e 100644 --- a/pkg/phase/helper_test.go +++ b/pkg/phase/helper_test.go @@ -318,10 +318,20 @@ func TestHelperClusterMapAPI(t *testing.T) { }, Map: map[string]*airshipv1.Cluster{ "target": { - Parent: "ephemeral", - DynamicKubeConfig: false, + Parent: "ephemeral", + Sources: []airshipv1.KubeconfigSource{ + { + Type: airshipv1.KubeconfigSourceTypeBundle, + }, + }, + }, + "ephemeral": { + Sources: []airshipv1.KubeconfigSource{ + { + Type: airshipv1.KubeconfigSourceTypeBundle, + }, + }, }, - "ephemeral": {}, }, }, config: testConfig, diff --git a/pkg/phase/testdata/phases/cluster-map.yaml b/pkg/phase/testdata/phases/cluster-map.yaml index d53fe7e30..0ec889670 100755 --- a/pkg/phase/testdata/phases/cluster-map.yaml +++ b/pkg/phase/testdata/phases/cluster-map.yaml @@ -7,5 +7,9 @@ metadata: name: main-map map: target-cluster: - parent: ephemeral-cluster - ephemeral-cluster: {} \ No newline at end of file + parent: ephemeral + kubeconfigSources: + - type: bundle + ephemeral-cluster: + kubeconfigSources: + - type: bundle diff --git a/pkg/phase/testdata/valid_site/phases/cluster_map.yaml b/pkg/phase/testdata/valid_site/phases/cluster_map.yaml index 89c410b06..47e720022 100644 --- a/pkg/phase/testdata/valid_site/phases/cluster_map.yaml +++ b/pkg/phase/testdata/valid_site/phases/cluster_map.yaml @@ -5,5 +5,8 @@ metadata: map: target: parent: ephemeral - dynamicKubeConf: false - ephemeral: {} \ No newline at end of file + kubeconfigSources: + - type: bundle + ephemeral: + kubeconfigSources: + - type: bundle diff --git a/tools/deployment/26_deploy_capi_ephemeral_node.sh b/tools/deployment/26_deploy_capi_ephemeral_node.sh index 7fd9d584e..5db08047d 100755 --- a/tools/deployment/26_deploy_capi_ephemeral_node.sh +++ b/tools/deployment/26_deploy_capi_ephemeral_node.sh @@ -54,7 +54,7 @@ if [ "$PROVIDER" = "metal3" ]; then airshipctl phase run clusterctl-init-ephemeral --debug else echo "Deploy cluster-api components to ephemeral node" - airshipctl phase run clusterctl-init-ephemeral --debug --kubeconfig ${KUBECONFIG} + airshipctl phase run clusterctl-init-ephemeral --debug kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get pods -A fi diff --git a/tools/document/validate_site_docs.sh b/tools/document/validate_site_docs.sh index fc67ee626..a4e6c03c0 100755 --- a/tools/document/validate_site_docs.sh +++ b/tools/document/validate_site_docs.sh @@ -24,6 +24,8 @@ set -xe : ${SITE:="test-workload"} : ${CONTEXT:="kind-airship"} : ${AIRSHIPKUBECONFIG:="${HOME}/.airship/kubeconfig"} +: ${AIRSHIPKUBECONFIG_BACKUP:="${AIRSHIPKUBECONFIG}-backup"} + : ${KUBECTL:="/usr/local/bin/kubectl"} TMP=$(mktemp -d) @@ -37,11 +39,10 @@ else fi : ${AIRSHIPCONFIG:="${TMP}/config"} -: ${KUBECONFIG:="${TMP}/kubeconfig"} : ${AIRSHIPCTL:="${AIRSHIPCTL_DEFAULT}"} -ACTL="${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} --kubeconfig ${KUBECONFIG}" +ACTL="${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG}" -export KUBECONFIG +export KUBECONFIG="${AIRSHIPKUBECONFIG}" # TODO: use `airshipctl config` to do this once all the needed knobs are exposed # The non-default parts are to set the targetPath appropriately, @@ -83,9 +84,20 @@ EOL function cleanup() { ${KIND} delete cluster --name $CLUSTER rm -rf ${TMP} + + if [ -f "${AIRSHIPKUBECONFIG_BACKUP}" ]; then + echo "Restoring a backup copy of kubeconfig" + cp "${AIRSHIPKUBECONFIG_BACKUP}" "${AIRSHIPKUBECONFIG}" + fi } trap cleanup EXIT +if [ -f "${AIRSHIPKUBECONFIG}" ]; then + echo "Making a backup copy of kubeconfig" + cp "${AIRSHIPKUBECONFIG}" "${AIRSHIPKUBECONFIG_BACKUP}" +fi + + generate_airshipconf "default" phase_plans=$(airshipctl --airshipconf ${AIRSHIPCONFIG} plan list | grep "PhasePlan" | awk -F '/' '{print $2}' | awk '{print $1}') @@ -96,8 +108,6 @@ for plan in $phase_plans; do for cluster in $cluster_list; do echo -e "\n**** Rendering phases for cluster: ${cluster}" - # Since we'll be mucking with the kubeconfig - make a copy of it and muck with the copy - cp ${AIRSHIPKUBECONFIG} ${KUBECONFIG} export CLUSTER="${cluster}" # Start a fresh, empty kind cluster for validating documents @@ -120,13 +130,14 @@ for plan in $phase_plans; do # e.g., load CRDs from initinfra first, so they're present when validating later phases ${AIRSHIPCTL} --airshipconf ${AIRSHIPCONFIG} phase render ${phase} -s executor -k CustomResourceDefinition >${TMP}/${phase}-crds.yaml if [ -s ${TMP}/${phase}-crds.yaml ]; then - ${KUBECTL} --context ${CLUSTER} --kubeconfig ${KUBECONFIG} apply -f ${TMP}/${phase}-crds.yaml + ${KUBECTL} --context ${CLUSTER} apply -f ${TMP}/${phase}-crds.yaml fi # step 2: dry-run the entire phase ${ACTL} phase run --dry-run ${phase} done - + # Delete cluster kubeconfig + rm ${KUBECONFIG} ${KIND} delete cluster --name $CLUSTER done done