Replace existing CRDs as a preupgrade action

Change-Id: I9ede06dd05576d98925a17e179c78b53527c6c57
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

.zuul.yaml

@@ -14,7 +14,8 @@
     check:
       jobs:
         - armada-operator-docker-build-gate-ubuntu_focal
-        - armada-operator-airskiff-deployment-focal
+        # disable for now until jammy jobs are configured
+        #- armada-operator-airskiff-deployment-focal
 
     gate:
       jobs:

api/v1/armadachart_types.go

@@ -106,6 +106,7 @@ type ArmadaChartUpgrade struct {
 type ArmadaChartPreUpgrade struct {
 	Cleanup   bool                        `json:"cleanup,omitempty"`
 	Delete    []ArmadaChartDeleteResource `json:"delete,omitempty"`
+	UpdateCRD bool                        `json:"update_crd,omitempty"`
 }
 
 // ArmadaChartDeleteResource defines the delete options of ArmadaChart

config/crd/bases/armada.airshipit.org_armadacharts.yaml

@@ -94,6 +94,8 @@ spec:
                               type: string
                           type: object
                         type: array
+                      update_crd:
+                        type: boolean
                     type: object
                 type: object
               values:

go.mod

@@ -9,13 +9,13 @@ require (
 	github.com/onsi/ginkgo/v2 v2.14.0
 	github.com/onsi/gomega v1.30.0
 	helm.sh/helm/v3 v3.14.2
-	k8s.io/api v0.29.0
+	k8s.io/api v0.29.2
-	k8s.io/apiextensions-apiserver v0.29.0
+	k8s.io/apiextensions-apiserver v0.29.2
-	k8s.io/apimachinery v0.29.0
+	k8s.io/apimachinery v0.29.2
 	k8s.io/cli-runtime v0.29.0
-	k8s.io/client-go v0.29.0
+	k8s.io/client-go v0.29.2
 	k8s.io/klog/v2 v2.110.1
-	sigs.k8s.io/controller-runtime v0.17.2
+	sigs.k8s.io/controller-runtime v0.17.6
 )
 
 require (
@@ -147,8 +147,8 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	k8s.io/apiserver v0.29.0 // indirect
+	k8s.io/apiserver v0.29.2 // indirect
-	k8s.io/component-base v0.29.0 // indirect
+	k8s.io/component-base v0.29.2 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
 	k8s.io/kubectl v0.29.0 // indirect
 	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect

go.sum

@@ -565,20 +565,20 @@ helm.sh/helm/v3 v3.14.2 h1:V71fv+NGZv0icBlr+in1MJXuUIHCiPG1hW9gEBISTIA=
 helm.sh/helm/v3 v3.14.2/go.mod h1:2itvvDv2WSZXTllknfQo6j7u3VVgMAvm8POCDgYH424=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
-k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
+k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A=
-k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
+k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0=
-k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0=
+k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg=
-k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc=
+k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8=
-k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
+k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8=
-k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
+k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
-k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o=
+k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ=
-k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM=
+k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ=
 k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4=
 k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk=
-k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
+k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg=
-k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
+k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA=
-k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s=
+k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8=
-k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
+k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM=
 k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
 k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
 k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
@@ -589,8 +589,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn
 k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
 oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
 oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
-sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0=
+sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw=
-sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
+sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
 sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
 sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=

pkg/controller/armadachart_controller.go

@@ -17,10 +17,15 @@ limitations under the License.
 package controller
 
 import (
+	"bytes"
 	"context"
 	"errors"
 	"fmt"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/cli-runtime/pkg/resource"
+
 	"io"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
@@ -35,10 +40,12 @@ import (
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	"helm.sh/helm/v3/pkg/chartutil"
+	helmkube "helm.sh/helm/v3/pkg/kube"
 	"helm.sh/helm/v3/pkg/release"
 	"helm.sh/helm/v3/pkg/storage/driver"
 	batchv1 "k8s.io/api/batch/v1"
 	corev1 "k8s.io/api/core/v1"
+	apiextension "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
 	"k8s.io/client-go/rest"
@@ -116,6 +123,12 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	return requeueRequired(ac, result, err)
 }
 
+type rootScoped struct{}
+
+func (*rootScoped) Name() apimeta.RESTScopeName {
+	return apimeta.RESTScopeNameRoot
+}
+
 func (r *ArmadaChartReconciler) reconcile(ctx context.Context, ac armadav1.ArmadaChart) (armadav1.ArmadaChart, ctrl.Result, error) {
 	log := ctrl.LoggerFrom(ctx)
 
@@ -231,6 +244,91 @@ func (r *ArmadaChartReconciler) reconcileChart(ctx context.Context,
 				}
 			}
 		}
+
+		if ac.Spec.Upgrade.PreUpgrade.UpdateCRD {
+			kc := helmkube.New(gettr)
+			allCRDs := make(helmkube.ResourceList, 0)
+			for _, CRDObj := range chrt.CRDObjects() {
+				res, err := kc.Build(bytes.NewBuffer(CRDObj.File.Data), false)
+				if err != nil {
+					log.Info(fmt.Sprintf("failed to parse CustomResourceDefinitions from %s: %s", CRDObj.Name, err))
+					continue
+				}
+				allCRDs = append(allCRDs, res...)
+			}
+			clientSet, err := apiextension.NewForConfig(restCfg)
+			if err != nil {
+				log.Info(fmt.Sprintf("could not create Kubernetes client set for API extensions: %s", err))
+				return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
+			}
+			csapi := clientSet.ApiextensionsV1().CustomResourceDefinitions()
+			var totalItems []*resource.Info
+			original := make(helmkube.ResourceList, 0)
+			for _, r := range allCRDs {
+				if o, err := csapi.Get(context.TODO(), r.Name, v1.GetOptions{}); err == nil && o != nil {
+					o.GetResourceVersion()
+					original = append(original, &resource.Info{
+						Client: clientSet.ApiextensionsV1().RESTClient(),
+						Mapping: &apimeta.RESTMapping{
+							Resource: schema.GroupVersionResource{
+								Group:    "apiextensions.k8s.io",
+								Version:  r.Mapping.GroupVersionKind.Version,
+								Resource: "customresourcedefinition",
+							},
+							GroupVersionKind: schema.GroupVersionKind{
+								Kind:    "CustomResourceDefinition",
+								Group:   "apiextensions.k8s.io",
+								Version: r.Mapping.GroupVersionKind.Version,
+							},
+							Scope: &rootScoped{},
+						},
+						Namespace:       o.ObjectMeta.Namespace,
+						Name:            o.ObjectMeta.Name,
+						Object:          o,
+						ResourceVersion: o.ObjectMeta.ResourceVersion,
+					})
+				} else if !apierrors.IsNotFound(err) {
+					log.Info(fmt.Sprintf("failed to get CustomResourceDefinition %s: %s", r.Name, err))
+					continue
+				}
+			}
+
+			if rr, err := kc.Update(original, allCRDs, true); err != nil {
+				log.Info(fmt.Sprintf("failed to update CustomResourceDefinition(s): %s", err))
+				return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
+			} else {
+				if rr != nil {
+					if rr.Created != nil {
+						totalItems = append(totalItems, rr.Created...)
+					}
+					if rr.Updated != nil {
+						totalItems = append(totalItems, rr.Updated...)
+					}
+					if rr.Deleted != nil {
+						totalItems = append(totalItems, rr.Deleted...)
+					}
+				}
+			}
+			if len(totalItems) > 0 {
+				// Give time for the CRD to be recognized.
+				if err := kc.Wait(totalItems, 60*time.Second); err != nil {
+					log.Info(fmt.Sprintf("failed to wait for CustomResourceDefinition(s): %s", err))
+					return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
+				}
+				log.Info(fmt.Sprintf("successfully applied %d CustomResourceDefinition(s)", len(totalItems)))
+
+				// Clear the RESTMapper cache, since it will not have the new CRDs.
+				// Helm does further invalidation of the client at a later stage
+				// when it gathers the server capabilities.
+				if m, err := gettr.ToRESTMapper(); err == nil {
+					if rm, ok := m.(apimeta.ResettableRESTMapper); ok {
+						log.Info("clearing REST mapper cache")
+						rm.Reset()
+					}
+				}
+			}
+		}
+
 		if ac.Spec.Upgrade.PreUpgrade.Cleanup {
 			getter, err := r.buildRESTClientGetter(ctx, ac)
 			if err != nil {
@@ -476,8 +574,12 @@ func isUpdateRequired(ctx context.Context, release *release.Release, chrt *chart
 		log.Info("There are chart values diffs found")
 		log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty()))
 		return true
-	}
 
+	case !cmp.Equal(release.Chart.CRDObjects(), chrt.CRDObjects(), cmpopts.EquateEmpty()):
+		log.Info("There are chart CRD diffs found")
+		log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty()))
+		return true
+	}
 	return false
 }