From a26d0c61077871134f4a8a00b38a42162e6675e2 Mon Sep 17 00:00:00 2001 From: Ruslan Aliev Date: Thu, 19 Sep 2024 10:44:00 -0500 Subject: [PATCH] Replace existing CRDs as a preupgrade action Change-Id: I9ede06dd05576d98925a17e179c78b53527c6c57 Signed-off-by: Ruslan Aliev --- .zuul.yaml | 3 +- api/v1/armadachart_types.go | 5 +- .../armada.airshipit.org_armadacharts.yaml | 2 + go.mod | 14 +-- go.sum | 28 ++--- pkg/controller/armadachart_controller.go | 104 +++++++++++++++++- 6 files changed, 131 insertions(+), 25 deletions(-) diff --git a/.zuul.yaml b/.zuul.yaml index 21621be..40a2344 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -14,7 +14,8 @@ check: jobs: - armada-operator-docker-build-gate-ubuntu_focal - - armada-operator-airskiff-deployment-focal + # disable for now until jammy jobs are configured + #- armada-operator-airskiff-deployment-focal gate: jobs: diff --git a/api/v1/armadachart_types.go b/api/v1/armadachart_types.go index 66cc4ce..141688f 100644 --- a/api/v1/armadachart_types.go +++ b/api/v1/armadachart_types.go @@ -104,8 +104,9 @@ type ArmadaChartUpgrade struct { } type ArmadaChartPreUpgrade struct { - Cleanup bool `json:"cleanup,omitempty"` - Delete []ArmadaChartDeleteResource `json:"delete,omitempty"` + Cleanup bool `json:"cleanup,omitempty"` + Delete []ArmadaChartDeleteResource `json:"delete,omitempty"` + UpdateCRD bool `json:"update_crd,omitempty"` } // ArmadaChartDeleteResource defines the delete options of ArmadaChart diff --git a/config/crd/bases/armada.airshipit.org_armadacharts.yaml b/config/crd/bases/armada.airshipit.org_armadacharts.yaml index 829122f..7ff52b0 100644 --- a/config/crd/bases/armada.airshipit.org_armadacharts.yaml +++ b/config/crd/bases/armada.airshipit.org_armadacharts.yaml @@ -94,6 +94,8 @@ spec: type: string type: object type: array + update_crd: + type: boolean type: object type: object values: diff --git a/go.mod b/go.mod index 0371752..2296f42 100644 --- a/go.mod +++ b/go.mod @@ -9,13 +9,13 @@ require ( github.com/onsi/ginkgo/v2 v2.14.0 github.com/onsi/gomega v1.30.0 helm.sh/helm/v3 v3.14.2 - k8s.io/api v0.29.0 - k8s.io/apiextensions-apiserver v0.29.0 - k8s.io/apimachinery v0.29.0 + k8s.io/api v0.29.2 + k8s.io/apiextensions-apiserver v0.29.2 + k8s.io/apimachinery v0.29.2 k8s.io/cli-runtime v0.29.0 - k8s.io/client-go v0.29.0 + k8s.io/client-go v0.29.2 k8s.io/klog/v2 v2.110.1 - sigs.k8s.io/controller-runtime v0.17.2 + sigs.k8s.io/controller-runtime v0.17.6 ) require ( @@ -147,8 +147,8 @@ require ( gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/apiserver v0.29.0 // indirect - k8s.io/component-base v0.29.0 // indirect + k8s.io/apiserver v0.29.2 // indirect + k8s.io/component-base v0.29.2 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect k8s.io/kubectl v0.29.0 // indirect k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect diff --git a/go.sum b/go.sum index ff664b6..156e6f7 100644 --- a/go.sum +++ b/go.sum @@ -565,20 +565,20 @@ helm.sh/helm/v3 v3.14.2 h1:V71fv+NGZv0icBlr+in1MJXuUIHCiPG1hW9gEBISTIA= helm.sh/helm/v3 v3.14.2/go.mod h1:2itvvDv2WSZXTllknfQo6j7u3VVgMAvm8POCDgYH424= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A= -k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA= -k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0= -k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc= -k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o= -k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis= -k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o= -k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM= +k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A= +k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0= +k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg= +k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8= +k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8= +k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU= +k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ= +k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ= k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4= k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk= -k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8= -k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38= -k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s= -k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M= +k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg= +k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA= +k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8= +k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM= k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0= k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo= k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780= @@ -589,8 +589,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY= oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324= -sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0= -sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s= +sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw= +sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0= diff --git a/pkg/controller/armadachart_controller.go b/pkg/controller/armadachart_controller.go index 2ca4bf8..1d1ced3 100644 --- a/pkg/controller/armadachart_controller.go +++ b/pkg/controller/armadachart_controller.go @@ -17,10 +17,15 @@ limitations under the License. package controller import ( + "bytes" "context" "errors" "fmt" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/cli-runtime/pkg/resource" + "io" + apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -35,10 +40,12 @@ import ( "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" "helm.sh/helm/v3/pkg/chartutil" + helmkube "helm.sh/helm/v3/pkg/kube" "helm.sh/helm/v3/pkg/release" "helm.sh/helm/v3/pkg/storage/driver" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + apiextension "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/apimachinery/pkg/runtime" "k8s.io/cli-runtime/pkg/genericclioptions" "k8s.io/client-go/rest" @@ -116,6 +123,12 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request) return requeueRequired(ac, result, err) } +type rootScoped struct{} + +func (*rootScoped) Name() apimeta.RESTScopeName { + return apimeta.RESTScopeNameRoot +} + func (r *ArmadaChartReconciler) reconcile(ctx context.Context, ac armadav1.ArmadaChart) (armadav1.ArmadaChart, ctrl.Result, error) { log := ctrl.LoggerFrom(ctx) @@ -231,6 +244,91 @@ func (r *ArmadaChartReconciler) reconcileChart(ctx context.Context, } } } + + if ac.Spec.Upgrade.PreUpgrade.UpdateCRD { + kc := helmkube.New(gettr) + allCRDs := make(helmkube.ResourceList, 0) + for _, CRDObj := range chrt.CRDObjects() { + res, err := kc.Build(bytes.NewBuffer(CRDObj.File.Data), false) + if err != nil { + log.Info(fmt.Sprintf("failed to parse CustomResourceDefinitions from %s: %s", CRDObj.Name, err)) + continue + } + allCRDs = append(allCRDs, res...) + } + clientSet, err := apiextension.NewForConfig(restCfg) + if err != nil { + log.Info(fmt.Sprintf("could not create Kubernetes client set for API extensions: %s", err)) + return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err + } + csapi := clientSet.ApiextensionsV1().CustomResourceDefinitions() + var totalItems []*resource.Info + original := make(helmkube.ResourceList, 0) + for _, r := range allCRDs { + if o, err := csapi.Get(context.TODO(), r.Name, v1.GetOptions{}); err == nil && o != nil { + o.GetResourceVersion() + original = append(original, &resource.Info{ + Client: clientSet.ApiextensionsV1().RESTClient(), + Mapping: &apimeta.RESTMapping{ + Resource: schema.GroupVersionResource{ + Group: "apiextensions.k8s.io", + Version: r.Mapping.GroupVersionKind.Version, + Resource: "customresourcedefinition", + }, + GroupVersionKind: schema.GroupVersionKind{ + Kind: "CustomResourceDefinition", + Group: "apiextensions.k8s.io", + Version: r.Mapping.GroupVersionKind.Version, + }, + Scope: &rootScoped{}, + }, + Namespace: o.ObjectMeta.Namespace, + Name: o.ObjectMeta.Name, + Object: o, + ResourceVersion: o.ObjectMeta.ResourceVersion, + }) + } else if !apierrors.IsNotFound(err) { + log.Info(fmt.Sprintf("failed to get CustomResourceDefinition %s: %s", r.Name, err)) + continue + } + } + + if rr, err := kc.Update(original, allCRDs, true); err != nil { + log.Info(fmt.Sprintf("failed to update CustomResourceDefinition(s): %s", err)) + return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err + } else { + if rr != nil { + if rr.Created != nil { + totalItems = append(totalItems, rr.Created...) + } + if rr.Updated != nil { + totalItems = append(totalItems, rr.Updated...) + } + if rr.Deleted != nil { + totalItems = append(totalItems, rr.Deleted...) + } + } + } + if len(totalItems) > 0 { + // Give time for the CRD to be recognized. + if err := kc.Wait(totalItems, 60*time.Second); err != nil { + log.Info(fmt.Sprintf("failed to wait for CustomResourceDefinition(s): %s", err)) + return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err + } + log.Info(fmt.Sprintf("successfully applied %d CustomResourceDefinition(s)", len(totalItems))) + + // Clear the RESTMapper cache, since it will not have the new CRDs. + // Helm does further invalidation of the client at a later stage + // when it gathers the server capabilities. + if m, err := gettr.ToRESTMapper(); err == nil { + if rm, ok := m.(apimeta.ResettableRESTMapper); ok { + log.Info("clearing REST mapper cache") + rm.Reset() + } + } + } + } + if ac.Spec.Upgrade.PreUpgrade.Cleanup { getter, err := r.buildRESTClientGetter(ctx, ac) if err != nil { @@ -476,8 +574,12 @@ func isUpdateRequired(ctx context.Context, release *release.Release, chrt *chart log.Info("There are chart values diffs found") log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty())) return true - } + case !cmp.Equal(release.Chart.CRDObjects(), chrt.CRDObjects(), cmpopts.EquateEmpty()): + log.Info("There are chart CRD diffs found") + log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty())) + return true + } return false }