Replace existing CRDs as a preupgrade action
Change-Id: I9ede06dd05576d98925a17e179c78b53527c6c57 Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
This commit is contained in:
Ruslan Aliev
parent
1e4cf3171b
commit
a26d0c6107
@ -14,7 +14,8 @@
|
||||
check:
|
||||
jobs:
|
||||
- armada-operator-docker-build-gate-ubuntu_focal
|
||||
- armada-operator-airskiff-deployment-focal
|
||||
# disable for now until jammy jobs are configured
|
||||
#- armada-operator-airskiff-deployment-focal
|
||||
|
||||
gate:
|
||||
jobs:
|
||||
|
||||
@ -104,8 +104,9 @@ type ArmadaChartUpgrade struct {
|
||||
}
|
||||
|
||||
type ArmadaChartPreUpgrade struct {
|
||||
Cleanup bool `json:"cleanup,omitempty"`
|
||||
Delete []ArmadaChartDeleteResource `json:"delete,omitempty"`
|
||||
Cleanup bool `json:"cleanup,omitempty"`
|
||||
Delete []ArmadaChartDeleteResource `json:"delete,omitempty"`
|
||||
UpdateCRD bool `json:"update_crd,omitempty"`
|
||||
}
|
||||
|
||||
// ArmadaChartDeleteResource defines the delete options of ArmadaChart
|
||||
|
||||
@ -94,6 +94,8 @@ spec:
|
||||
type: string
|
||||
type: object
|
||||
type: array
|
||||
update_crd:
|
||||
type: boolean
|
||||
type: object
|
||||
type: object
|
||||
values:
|
||||
|
||||
14
go.mod
14
go.mod
@ -9,13 +9,13 @@ require (
|
||||
github.com/onsi/ginkgo/v2 v2.14.0
|
||||
github.com/onsi/gomega v1.30.0
|
||||
helm.sh/helm/v3 v3.14.2
|
||||
k8s.io/api v0.29.0
|
||||
k8s.io/apiextensions-apiserver v0.29.0
|
||||
k8s.io/apimachinery v0.29.0
|
||||
k8s.io/api v0.29.2
|
||||
k8s.io/apiextensions-apiserver v0.29.2
|
||||
k8s.io/apimachinery v0.29.2
|
||||
k8s.io/cli-runtime v0.29.0
|
||||
k8s.io/client-go v0.29.0
|
||||
k8s.io/client-go v0.29.2
|
||||
k8s.io/klog/v2 v2.110.1
|
||||
sigs.k8s.io/controller-runtime v0.17.2
|
||||
sigs.k8s.io/controller-runtime v0.17.6
|
||||
)
|
||||
|
||||
require (
|
||||
@ -147,8 +147,8 @@ require (
|
||||
gopkg.in/inf.v0 v0.9.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/apiserver v0.29.0 // indirect
|
||||
k8s.io/component-base v0.29.0 // indirect
|
||||
k8s.io/apiserver v0.29.2 // indirect
|
||||
k8s.io/component-base v0.29.2 // indirect
|
||||
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect
|
||||
k8s.io/kubectl v0.29.0 // indirect
|
||||
k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
|
||||
|
||||
28
go.sum
28
go.sum
@ -565,20 +565,20 @@ helm.sh/helm/v3 v3.14.2 h1:V71fv+NGZv0icBlr+in1MJXuUIHCiPG1hW9gEBISTIA=
|
||||
helm.sh/helm/v3 v3.14.2/go.mod h1:2itvvDv2WSZXTllknfQo6j7u3VVgMAvm8POCDgYH424=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
k8s.io/api v0.29.0 h1:NiCdQMY1QOp1H8lfRyeEf8eOwV6+0xA6XEE44ohDX2A=
|
||||
k8s.io/api v0.29.0/go.mod h1:sdVmXoz2Bo/cb77Pxi71IPTSErEW32xa4aXwKH7gfBA=
|
||||
k8s.io/apiextensions-apiserver v0.29.0 h1:0VuspFG7Hj+SxyF/Z/2T0uFbI5gb5LRgEyUVE3Q4lV0=
|
||||
k8s.io/apiextensions-apiserver v0.29.0/go.mod h1:TKmpy3bTS0mr9pylH0nOt/QzQRrW7/h7yLdRForMZwc=
|
||||
k8s.io/apimachinery v0.29.0 h1:+ACVktwyicPz0oc6MTMLwa2Pw3ouLAfAon1wPLtG48o=
|
||||
k8s.io/apimachinery v0.29.0/go.mod h1:eVBxQ/cwiJxH58eK/jd/vAk4mrxmVlnpBH5J2GbMeis=
|
||||
k8s.io/apiserver v0.29.0 h1:Y1xEMjJkP+BIi0GSEv1BBrf1jLU9UPfAnnGGbbDdp7o=
|
||||
k8s.io/apiserver v0.29.0/go.mod h1:31n78PsRKPmfpee7/l9NYEv67u6hOL6AfcE761HapDM=
|
||||
k8s.io/api v0.29.2 h1:hBC7B9+MU+ptchxEqTNW2DkUosJpp1P+Wn6YncZ474A=
|
||||
k8s.io/api v0.29.2/go.mod h1:sdIaaKuU7P44aoyyLlikSLayT6Vb7bvJNCX105xZXY0=
|
||||
k8s.io/apiextensions-apiserver v0.29.2 h1:UK3xB5lOWSnhaCk0RFZ0LUacPZz9RY4wi/yt2Iu+btg=
|
||||
k8s.io/apiextensions-apiserver v0.29.2/go.mod h1:aLfYjpA5p3OwtqNXQFkhJ56TB+spV8Gc4wfMhUA3/b8=
|
||||
k8s.io/apimachinery v0.29.2 h1:EWGpfJ856oj11C52NRCHuU7rFDwxev48z+6DSlGNsV8=
|
||||
k8s.io/apimachinery v0.29.2/go.mod h1:6HVkd1FwxIagpYrHSwJlQqZI3G9LfYWRPAkUvLnXTKU=
|
||||
k8s.io/apiserver v0.29.2 h1:+Z9S0dSNr+CjnVXQePG8TcBWHr3Q7BmAr7NraHvsMiQ=
|
||||
k8s.io/apiserver v0.29.2/go.mod h1:B0LieKVoyU7ykQvPFm7XSdIHaCHSzCzQWPFa5bqbeMQ=
|
||||
k8s.io/cli-runtime v0.29.0 h1:q2kC3cex4rOBLfPOnMSzV2BIrrQlx97gxHJs21KxKS4=
|
||||
k8s.io/cli-runtime v0.29.0/go.mod h1:VKudXp3X7wR45L+nER85YUzOQIru28HQpXr0mTdeCrk=
|
||||
k8s.io/client-go v0.29.0 h1:KmlDtFcrdUzOYrBhXHgKw5ycWzc3ryPX5mQe0SkG3y8=
|
||||
k8s.io/client-go v0.29.0/go.mod h1:yLkXH4HKMAywcrD82KMSmfYg2DlE8mepPR4JGSo5n38=
|
||||
k8s.io/component-base v0.29.0 h1:T7rjd5wvLnPBV1vC4zWd/iWRbV8Mdxs+nGaoaFzGw3s=
|
||||
k8s.io/component-base v0.29.0/go.mod h1:sADonFTQ9Zc9yFLghpDpmNXEdHyQmFIGbiuZbqAXQ1M=
|
||||
k8s.io/client-go v0.29.2 h1:FEg85el1TeZp+/vYJM7hkDlSTFZ+c5nnK44DJ4FyoRg=
|
||||
k8s.io/client-go v0.29.2/go.mod h1:knlvFZE58VpqbQpJNbCbctTVXcd35mMyAAwBdpt4jrA=
|
||||
k8s.io/component-base v0.29.2 h1:lpiLyuvPA9yV1aQwGLENYyK7n/8t6l3nn3zAtFTJYe8=
|
||||
k8s.io/component-base v0.29.2/go.mod h1:BfB3SLrefbZXiBfbM+2H1dlat21Uewg/5qtKOl8degM=
|
||||
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
|
||||
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
|
||||
k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 h1:aVUu9fTY98ivBPKR9Y5w/AuzbMm96cd3YHRTU83I780=
|
||||
@ -589,8 +589,8 @@ k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSn
|
||||
k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0=
|
||||
oras.land/oras-go v1.2.4 h1:djpBY2/2Cs1PV87GSJlxv4voajVOMZxqqtq9AB8YNvY=
|
||||
oras.land/oras-go v1.2.4/go.mod h1:DYcGfb3YF1nKjcezfX2SNlDAeQFKSXmf+qrFmrh4324=
|
||||
sigs.k8s.io/controller-runtime v0.17.2 h1:FwHwD1CTUemg0pW2otk7/U5/i5m2ymzvOXdbeGOUvw0=
|
||||
sigs.k8s.io/controller-runtime v0.17.2/go.mod h1:+MngTvIQQQhfXtwfdGw/UOQ/aIaqsYywfCINOtwMO/s=
|
||||
sigs.k8s.io/controller-runtime v0.17.6 h1:12IXsozEsIXWAMRpgRlYS1jjAHQXHtWEOMdULh3DbEw=
|
||||
sigs.k8s.io/controller-runtime v0.17.6/go.mod h1:N0jpP5Lo7lMTF9aL56Z/B2oWBJjey6StQM0jRbKQXtY=
|
||||
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo=
|
||||
sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0=
|
||||
sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 h1:XX3Ajgzov2RKUdc5jW3t5jwY7Bo7dcRm+tFxT+NfgY0=
|
||||
|
||||
@ -17,10 +17,15 @@ limitations under the License.
|
||||
package controller
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"k8s.io/cli-runtime/pkg/resource"
|
||||
|
||||
"io"
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
apimeta "k8s.io/apimachinery/pkg/api/meta"
|
||||
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/wait"
|
||||
@ -35,10 +40,12 @@ import (
|
||||
"helm.sh/helm/v3/pkg/chart"
|
||||
"helm.sh/helm/v3/pkg/chart/loader"
|
||||
"helm.sh/helm/v3/pkg/chartutil"
|
||||
helmkube "helm.sh/helm/v3/pkg/kube"
|
||||
"helm.sh/helm/v3/pkg/release"
|
||||
"helm.sh/helm/v3/pkg/storage/driver"
|
||||
batchv1 "k8s.io/api/batch/v1"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
apiextension "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
|
||||
"k8s.io/apimachinery/pkg/runtime"
|
||||
"k8s.io/cli-runtime/pkg/genericclioptions"
|
||||
"k8s.io/client-go/rest"
|
||||
@ -116,6 +123,12 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
|
||||
return requeueRequired(ac, result, err)
|
||||
}
|
||||
|
||||
type rootScoped struct{}
|
||||
|
||||
func (*rootScoped) Name() apimeta.RESTScopeName {
|
||||
return apimeta.RESTScopeNameRoot
|
||||
}
|
||||
|
||||
func (r *ArmadaChartReconciler) reconcile(ctx context.Context, ac armadav1.ArmadaChart) (armadav1.ArmadaChart, ctrl.Result, error) {
|
||||
log := ctrl.LoggerFrom(ctx)
|
||||
|
||||
@ -231,6 +244,91 @@ func (r *ArmadaChartReconciler) reconcileChart(ctx context.Context,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ac.Spec.Upgrade.PreUpgrade.UpdateCRD {
|
||||
kc := helmkube.New(gettr)
|
||||
allCRDs := make(helmkube.ResourceList, 0)
|
||||
for _, CRDObj := range chrt.CRDObjects() {
|
||||
res, err := kc.Build(bytes.NewBuffer(CRDObj.File.Data), false)
|
||||
if err != nil {
|
||||
log.Info(fmt.Sprintf("failed to parse CustomResourceDefinitions from %s: %s", CRDObj.Name, err))
|
||||
continue
|
||||
}
|
||||
allCRDs = append(allCRDs, res...)
|
||||
}
|
||||
clientSet, err := apiextension.NewForConfig(restCfg)
|
||||
if err != nil {
|
||||
log.Info(fmt.Sprintf("could not create Kubernetes client set for API extensions: %s", err))
|
||||
return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
|
||||
}
|
||||
csapi := clientSet.ApiextensionsV1().CustomResourceDefinitions()
|
||||
var totalItems []*resource.Info
|
||||
original := make(helmkube.ResourceList, 0)
|
||||
for _, r := range allCRDs {
|
||||
if o, err := csapi.Get(context.TODO(), r.Name, v1.GetOptions{}); err == nil && o != nil {
|
||||
o.GetResourceVersion()
|
||||
original = append(original, &resource.Info{
|
||||
Client: clientSet.ApiextensionsV1().RESTClient(),
|
||||
Mapping: &apimeta.RESTMapping{
|
||||
Resource: schema.GroupVersionResource{
|
||||
Group: "apiextensions.k8s.io",
|
||||
Version: r.Mapping.GroupVersionKind.Version,
|
||||
Resource: "customresourcedefinition",
|
||||
},
|
||||
GroupVersionKind: schema.GroupVersionKind{
|
||||
Kind: "CustomResourceDefinition",
|
||||
Group: "apiextensions.k8s.io",
|
||||
Version: r.Mapping.GroupVersionKind.Version,
|
||||
},
|
||||
Scope: &rootScoped{},
|
||||
},
|
||||
Namespace: o.ObjectMeta.Namespace,
|
||||
Name: o.ObjectMeta.Name,
|
||||
Object: o,
|
||||
ResourceVersion: o.ObjectMeta.ResourceVersion,
|
||||
})
|
||||
} else if !apierrors.IsNotFound(err) {
|
||||
log.Info(fmt.Sprintf("failed to get CustomResourceDefinition %s: %s", r.Name, err))
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if rr, err := kc.Update(original, allCRDs, true); err != nil {
|
||||
log.Info(fmt.Sprintf("failed to update CustomResourceDefinition(s): %s", err))
|
||||
return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
|
||||
} else {
|
||||
if rr != nil {
|
||||
if rr.Created != nil {
|
||||
totalItems = append(totalItems, rr.Created...)
|
||||
}
|
||||
if rr.Updated != nil {
|
||||
totalItems = append(totalItems, rr.Updated...)
|
||||
}
|
||||
if rr.Deleted != nil {
|
||||
totalItems = append(totalItems, rr.Deleted...)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(totalItems) > 0 {
|
||||
// Give time for the CRD to be recognized.
|
||||
if err := kc.Wait(totalItems, 60*time.Second); err != nil {
|
||||
log.Info(fmt.Sprintf("failed to wait for CustomResourceDefinition(s): %s", err))
|
||||
return armadav1.ArmadaChartNotReady(ac, "CRDUpdateFailed", err.Error()), err
|
||||
}
|
||||
log.Info(fmt.Sprintf("successfully applied %d CustomResourceDefinition(s)", len(totalItems)))
|
||||
|
||||
// Clear the RESTMapper cache, since it will not have the new CRDs.
|
||||
// Helm does further invalidation of the client at a later stage
|
||||
// when it gathers the server capabilities.
|
||||
if m, err := gettr.ToRESTMapper(); err == nil {
|
||||
if rm, ok := m.(apimeta.ResettableRESTMapper); ok {
|
||||
log.Info("clearing REST mapper cache")
|
||||
rm.Reset()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if ac.Spec.Upgrade.PreUpgrade.Cleanup {
|
||||
getter, err := r.buildRESTClientGetter(ctx, ac)
|
||||
if err != nil {
|
||||
@ -476,8 +574,12 @@ func isUpdateRequired(ctx context.Context, release *release.Release, chrt *chart
|
||||
log.Info("There are chart values diffs found")
|
||||
log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty()))
|
||||
return true
|
||||
}
|
||||
|
||||
case !cmp.Equal(release.Chart.CRDObjects(), chrt.CRDObjects(), cmpopts.EquateEmpty()):
|
||||
log.Info("There are chart CRD diffs found")
|
||||
log.Info(cmp.Diff(release.Config, vals.AsMap(), cmpopts.EquateEmpty()))
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
|
||||
Loading…
Reference in New Issue
Block a user