diff --git a/manifests/function/baremetal-operator/replacements/ironic-env-vars.yaml b/manifests/function/baremetal-operator/replacements/ironic-env-vars.yaml index 594bc7f87..e8c580fcd 100644 --- a/manifests/function/baremetal-operator/replacements/ironic-env-vars.yaml +++ b/manifests/function/baremetal-operator/replacements/ironic-env-vars.yaml @@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: baremetal-operator-env-vars-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: # Replace the proxy vars - source: diff --git a/manifests/function/baremetal-operator/replacements/networking.yaml b/manifests/function/baremetal-operator/replacements/networking.yaml index 2045d1d46..fb615e201 100644 --- a/manifests/function/baremetal-operator/replacements/networking.yaml +++ b/manifests/function/baremetal-operator/replacements/networking.yaml @@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: baremetal-operator-networking-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: # Replace the pod & service networks - source: diff --git a/manifests/function/cni/calico/v3/replacements/versions.yaml b/manifests/function/cni/calico/v3/replacements/versions.yaml index 8fb009903..3b3141c94 100644 --- a/manifests/function/cni/calico/v3/replacements/versions.yaml +++ b/manifests/function/cni/calico/v3/replacements/versions.yaml @@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: calico-v3-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: - source: objref: diff --git a/manifests/function/ephemeral/replacements/ephemeral-env-vars.yaml b/manifests/function/ephemeral/replacements/ephemeral-env-vars.yaml index 83dcdc50b..ab2895c8e 100644 --- a/manifests/function/ephemeral/replacements/ephemeral-env-vars.yaml +++ b/manifests/function/ephemeral/replacements/ephemeral-env-vars.yaml @@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: ephemeral-env-vars-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: # Replace the proxy vars - source: diff --git a/manifests/function/ephemeral/replacements/networking.yaml b/manifests/function/ephemeral/replacements/networking.yaml index 8cc238657..79d7f4ed9 100644 --- a/manifests/function/ephemeral/replacements/networking.yaml +++ b/manifests/function/ephemeral/replacements/networking.yaml @@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: ephemeral-networking-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: # Substring-replace the ephemeral control plane's info - source: diff --git a/manifests/function/helm-operator/replacements/versions.yaml b/manifests/function/helm-operator/replacements/versions.yaml index ef1befc24..7ee5dee58 100644 --- a/manifests/function/helm-operator/replacements/versions.yaml +++ b/manifests/function/helm-operator/replacements/versions.yaml @@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: helm-operator-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: - source: objref: diff --git a/manifests/function/hwcc/replacements/versions.yaml b/manifests/function/hwcc/replacements/versions.yaml index 421684e6e..9901ac13b 100644 --- a/manifests/function/hwcc/replacements/versions.yaml +++ b/manifests/function/hwcc/replacements/versions.yaml @@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1 kind: ReplacementTransformer metadata: name: hwcc-versions-replacements + annotations: + config.kubernetes.io/function: |- + container: + image: quay.io/airshipit/replacement-transformer:dev replacements: - source: objref: diff --git a/pkg/document/plugin/replacement/errors.go b/pkg/document/plugin/replacement/errors.go index 7dfd8b424..449fcd1c6 100644 --- a/pkg/document/plugin/replacement/errors.go +++ b/pkg/document/plugin/replacement/errors.go @@ -22,23 +22,6 @@ import ( "sigs.k8s.io/kustomize/api/types" ) -// ErrTypeMismatch is returned for type conversion errors. This error -// is raised if JSON path element points to a wrong data structure e.g. -// JSON path 'a.b[x=y]c' considers that there is a list of maps under key 'b' -// therefore ErrTypeMismatch will be returned for following structure -// -// a: -// b: -// - 'some string' -type ErrTypeMismatch struct { - Actual interface{} - Expectation string -} - -func (e ErrTypeMismatch) Error() string { - return fmt.Sprintf("%#v %s", e.Actual, e.Expectation) -} - // ErrBadConfiguration returned in case of plugin misconfiguration type ErrBadConfiguration struct { Msg string @@ -94,15 +77,6 @@ func (e ErrIndexOutOfBound) Error() string { return fmt.Sprintf("array index out of bounds: index %d, length %d", e.Index, e.Length) } -// ErrValueNotFound returned if value specified in fieldRef option was not found -type ErrValueNotFound struct { - ID string -} - -func (e ErrValueNotFound) Error() string { - return fmt.Sprintf("unable to find value identified by %s", e.ID) -} - func printFields(objRef interface{}) string { val := reflect.ValueOf(objRef).Elem() valType := val.Type() diff --git a/pkg/document/plugin/replacement/transformer.go b/pkg/document/plugin/replacement/transformer.go index b5db9b8ed..fa7d45611 100644 --- a/pkg/document/plugin/replacement/transformer.go +++ b/pkg/document/plugin/replacement/transformer.go @@ -1,39 +1,39 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ package replacement import ( "fmt" "io" - "io/ioutil" - "reflect" "regexp" - "strconv" - "strings" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/kustomize/api/k8sdeps/kunstruct" - "sigs.k8s.io/kustomize/api/resmap" - "sigs.k8s.io/kustomize/api/resource" "sigs.k8s.io/kustomize/api/types" "sigs.k8s.io/kustomize/kyaml/yaml" airshipv1 "opendev.org/airship/airshipctl/pkg/api/v1alpha1" "opendev.org/airship/airshipctl/pkg/document/plugin/kyamlutils" plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types" + "opendev.org/airship/airshipctl/pkg/errors" ) var ( - pattern = regexp.MustCompile(`(\S+)\[(\S+)=(\S+)\]`) // substring substitutions are appended to paths as: ...%VARNAME% substringPatternRegex = regexp.MustCompile(`(.+)%(\S+)%$`) ) -const ( - dotReplacer = "$$$$" -) - var _ plugtypes.Plugin = &plugin{} type plugin struct { @@ -63,56 +63,7 @@ func New(obj map[string]interface{}) (plugtypes.Plugin, error) { } func (p *plugin) Run(in io.Reader, out io.Writer) error { - data, err := ioutil.ReadAll(in) - if err != nil { - return err - } - - kf := kunstruct.NewKunstructuredFactoryImpl() - rf := resource.NewFactory(kf) - resources, err := rf.SliceFromBytes(data) - if err != nil { - return err - } - - rm := resmap.New() - for _, r := range resources { - if err = rm.Append(r); err != nil { - return err - } - } - - if err = p.Transform(rm); err != nil { - return err - } - - result, err := rm.AsYaml() - if err != nil { - return err - } - fmt.Fprint(out, string(result)) - return nil -} - -// Transform resources using configured replacements -func (p *plugin) Transform(m resmap.ResMap) error { - var err error - for _, r := range p.Replacements { - var replacement interface{} - if r.Source.ObjRef != nil { - replacement, err = getReplacement(m, r.Source.ObjRef, r.Source.FieldRef) - if err != nil { - return err - } - } - if r.Source.Value != "" { - replacement = r.Source.Value - } - if err = substitute(m, r.Target, replacement); err != nil { - return err - } - } - return nil + return errors.ErrNotImplemented{What: "ReplacementTransformer method Run is deprecated and will be removed"} } func (p *plugin) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) { @@ -229,256 +180,3 @@ func substituteSubstring(tgt *yaml.RNode, fieldRef, substringPattern string, val } return nil } - -func getReplacement(m resmap.ResMap, objRef *types.Target, fieldRef string) (interface{}, error) { - s := types.Selector{ - Gvk: objRef.Gvk, - Name: objRef.Name, - Namespace: objRef.Namespace, - } - resources, err := m.Select(s) - if err != nil { - return nil, err - } - if len(resources) > 1 { - resList := make([]string, len(resources)) - for i := range resources { - resList[i] = resources[i].String() - } - return nil, ErrMultipleResources{ObjRef: objRef} - } - if len(resources) == 0 { - return nil, ErrSourceNotFound{ObjRef: objRef} - } - if fieldRef == "" { - fieldRef = "metadata.name" - } - return resources[0].GetFieldValue(fieldRef) -} - -func substitute(m resmap.ResMap, to *types.ReplTarget, replacement interface{}) error { - resources, err := m.Select(*to.ObjRef) - if err != nil { - return err - } - if len(resources) == 0 { - return ErrTargetNotFound{ObjRef: to.ObjRef} - } - for _, r := range resources { - for _, p := range to.FieldRefs { - // TODO (dukov) rework this using k8s.io/client-go/util/jsonpath - parts := strings.Split(p, "[") - var tmp []string - for _, part := range parts { - if strings.Contains(part, "]") { - filter := strings.Split(part, "]") - filter[0] = strings.ReplaceAll(filter[0], ".", dotReplacer) - part = strings.Join(filter, "]") - } - tmp = append(tmp, part) - } - p = strings.Join(tmp, "[") - // Exclude substring portion from dot replacer - // substring can contain IP or any dot separated string - substringPattern := "" - p, substringPattern = extractSubstringPattern(p) - - pathSlice := strings.Split(p, ".") - // append back the extracted substring - if len(substringPattern) > 0 { - pathSlice[len(pathSlice)-1] = pathSlice[len(pathSlice)-1] + "%" + - substringPattern + "%" - } - for i, part := range pathSlice { - pathSlice[i] = strings.ReplaceAll(part, dotReplacer, ".") - } - if err := updateField(r.Map(), pathSlice, replacement); err != nil { - return err - } - } - } - return nil -} - -func getFirstPathSegment(path string) (field string, key string, value string, isArray bool) { - groups := pattern.FindStringSubmatch(path) - if len(groups) != 4 { - return path, "", "", false - } - return groups[1], groups[2], groups[3], groups[2] != "" -} - -func updateField(m interface{}, pathToField []string, replacement interface{}) error { - if len(pathToField) == 0 { - return nil - } - - switch typedM := m.(type) { - case map[string]interface{}: - return updateMapField(typedM, pathToField, replacement) - case []interface{}: - return updateSliceField(typedM, pathToField, replacement) - default: - return ErrTypeMismatch{Actual: typedM, Expectation: "is not expected be a primitive type"} - } -} - -// Extract the substring pattern (if present) from the target path spec -func extractSubstringPattern(path string) (extractedPath string, substringPattern string) { - groups := substringPatternRegex.FindStringSubmatch(path) - if len(groups) != 3 { - return path, "" - } - return groups[1], groups[2] -} - -// replaces substring in a string if pattern applies -func processString(field string, substringPattern string, replacement string) string { - pattern := regexp.MustCompile(substringPattern) - return pattern.ReplaceAllString(field, replacement) -} - -// replaces substring in any string in the array if pattern applies -func processArray(tgt []string, pattern string, replacement string) []string { - result := make([]string, 0, len(tgt)) - for _, field := range tgt { - result = append(result, processString(field, pattern, replacement)) - } - return result -} - -// converts array of interfaces to array of strings -func convertToStrings(iFaces []interface{}) ([]string, bool) { - result := []string{} - for _, val := range iFaces { - if str, ok := val.(string); ok { - result = append(result, str) - } - } - return result, (len(result) != 0) -} - -// apply a substring substitution based on a pattern -func applySubstringPattern(target interface{}, replacement interface{}, - substringPattern string) (regexedReplacement interface{}, err error) { - // no regex'ing needed if there is no substringPattern - if substringPattern == "" { - return replacement, nil - } - - // if replacement is numeric, convert it to a string for the sake of - // substring substitution. - var replacementString string - switch t := replacement.(type) { - case uint, uint8, uint16, uint32, uint64, int, int8, int16, int32, int64: - replacementString = fmt.Sprint(t) - case string: - replacementString = t - default: - return nil, ErrPatternSubstring{Msg: "pattern-based substitution can only be applied " + - "with string or numeric replacement values"} - } - - switch reflect.TypeOf(target).Kind() { - case reflect.String: - return processString(target.(string), substringPattern, replacementString), nil - case reflect.Slice: - if ifaceArray, ok := target.([]interface{}); ok { - if strArray, ok := convertToStrings(ifaceArray); ok { - return processArray(strArray, substringPattern, replacementString), nil - } - } - if strArray, ok := target.([]string); ok { - return processArray(strArray, substringPattern, replacementString), nil - } - } - return nil, ErrPatternSubstring{ - Msg: "pattern-based substitution can only be applied to string " + - "or array of strings target fields", - } -} - -func updateMapField(m map[string]interface{}, pathToField []string, replacement interface{}) error { - path, key, value, isArray := getFirstPathSegment(pathToField[0]) - - path, substringPattern := extractSubstringPattern(path) - - v, found := m[path] - if !found { - m[path] = make(map[string]interface{}) - v = m[path] - } - - if v == nil { - return ErrTypeMismatch{Actual: v, Expectation: "is not expected be nil"} - } - - if len(pathToField) == 1 { - if !isArray { - renderedRepl, err := applySubstringPattern(m[path], replacement, substringPattern) - if err != nil { - return err - } - m[path] = renderedRepl - return nil - } - switch typedV := v.(type) { - case []interface{}: - for i, item := range typedV { - typedItem, ok := item.(map[string]interface{}) - if !ok { - return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)} - } - if actualValue, ok := typedItem[key]; ok { - if value == actualValue { - typedV[i] = replacement - return nil - } - } - } - default: - return ErrTypeMismatch{Actual: typedV, Expectation: "is not expected be a primitive type"} - } - } - - if isArray { - return updateField(v, pathToField, replacement) - } - return updateField(v, pathToField[1:], replacement) -} - -func updateSliceField(m []interface{}, pathToField []string, replacement interface{}) error { - if len(pathToField) == 0 { - return nil - } - _, key, value, isArray := getFirstPathSegment(pathToField[0]) - - if isArray { - for _, item := range m { - typedItem, ok := item.(map[string]interface{}) - if !ok { - return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)} - } - if actualValue, ok := typedItem[key]; ok { - if value == actualValue { - return updateField(typedItem, pathToField[1:], replacement) - } - } - } - return nil - } - - index, err := strconv.Atoi(pathToField[0]) - if err != nil { - return err - } - - if len(m) < index || index < 0 { - return ErrIndexOutOfBound{Index: index, Length: len(m)} - } - if len(pathToField) == 1 { - m[index] = replacement - return nil - } - return updateField(m[index], pathToField[1:], replacement) -} diff --git a/pkg/document/plugin/replacement/transformer_test.go b/pkg/document/plugin/replacement/transformer_test.go index 1dd2b1e79..7726ced99 100644 --- a/pkg/document/plugin/replacement/transformer_test.go +++ b/pkg/document/plugin/replacement/transformer_test.go @@ -1,12 +1,22 @@ -// Copyright 2019 The Kubernetes Authors. -// SPDX-License-Identifier: Apache-2.0 +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ package replacement_test import ( "bytes" "fmt" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -16,62 +26,8 @@ import ( "sigs.k8s.io/yaml" "opendev.org/airship/airshipctl/pkg/document/plugin/replacement" - plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types" ) -func samplePlugin(t *testing.T) plugtypes.Plugin { - cfg := make(map[string]interface{}) - conf := ` -apiVersion: airshipit.org/v1alpha1 -kind: ReplacementTransformer -metadata: - name: notImportantHere -replacements: -- source: - value: nginx:newtag - target: - objref: - kind: Deployment - fieldrefs: - - spec.template.spec.containers[name=nginx-latest].image` - - err := yaml.Unmarshal([]byte(conf), &cfg) - require.NoError(t, err) - plugin, err := replacement.New(cfg) - require.NoError(t, err) - return plugin -} - -func TestMalformedInput(t *testing.T) { - plugin := samplePlugin(t) - err := plugin.Run(strings.NewReader("--"), &bytes.Buffer{}) - assert.Error(t, err) -} - -func TestDuplicatedResources(t *testing.T) { - plugin := samplePlugin(t) - err := plugin.Run(strings.NewReader(` -apiVersion: v1 -kind: Pod -metadata: - name: pod -spec: - containers: - - name: myapp-container - image: busybox ---- -apiVersion: v1 -kind: Pod -metadata: - name: pod -spec: - containers: - - name: myapp-container - image: busybox - `), &bytes.Buffer{}) - assert.Error(t, err) -} - var testCases = []struct { cfg string in string @@ -804,7 +760,8 @@ spec: containers: - name: myapp-container image: busybox`, - expectedErr: `"some string value" is not expected be a primitive type`, + expectedErr: "wrong Node Kind for labels.somelabel expected: " + + "MappingNode was ScalarNode: value: {'some string value'}", }, { cfg: ` @@ -843,7 +800,8 @@ spec: containers: - name: myapp-container image: busybox`, - expectedErr: `"some string value" is not expected be a primitive type`, + expectedErr: "wrong Node Kind for labels.somelabel expected: " + + "SequenceNode was ScalarNode: value: {'some string value'}", }, { cfg: ` @@ -882,9 +840,8 @@ spec: containers: - name: myapp-container image: busybox`, - expectedErr: "map[string]interface {}{\"containers\":[]interface " + - "{}{map[string]interface {}{\"image\":\"busybox\", \"name\":\"myapp-container\"}}} " + - "is not expected be a primitive type", + expectedErr: "wrong Node Kind for spec expected: " + + "SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n image: busybox}", }, { cfg: ` @@ -962,7 +919,8 @@ spec: containers: - name: myapp-container image: busybox`, - expectedErr: `strconv.Atoi: parsing "notInteger": invalid syntax`, + expectedErr: "wrong Node Kind for spec.containers expected: " + + "MappingNode was SequenceNode: value: {- name: myapp-container\n image: busybox}", }, { cfg: ` @@ -1001,7 +959,8 @@ spec: containers: - image: nginx:TAG name: nginx-latest`, - expectedErr: "pattern-based substitution can only be applied to string or array of strings target fields", + expectedErr: "wrong Node Kind for expected: " + + "ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}", }, { cfg: ` @@ -1058,39 +1017,7 @@ spec: }, } -func TestReplacementTransformer(t *testing.T) { - for _, tc := range testCases { - cfg := make(map[string]interface{}) - err := yaml.Unmarshal([]byte(tc.cfg), &cfg) - require.NoError(t, err) - plugin, err := replacement.New(cfg) - require.NoError(t, err) - - buf := &bytes.Buffer{} - err = plugin.Run(strings.NewReader(tc.in), buf) - errString := "" - if err != nil { - errString = err.Error() - } - assert.Equal(t, tc.expectedErr, errString) - assert.Equal(t, tc.expectedOut, buf.String()) - } -} - func TestExec(t *testing.T) { - // TODO (dukov) Remove this once we migrate to new kustomize plugin approach - // NOTE (dukov) we need this since error format is different for new kustomize plugins - testCases[13].expectedErr = "wrong Node Kind for labels.somelabel expected: " + - "MappingNode was ScalarNode: value: {'some string value'}" - testCases[14].expectedErr = "wrong Node Kind for labels.somelabel expected: " + - "SequenceNode was ScalarNode: value: {'some string value'}" - testCases[15].expectedErr = "wrong Node Kind for spec expected: " + - "SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n image: busybox}" - testCases[17].expectedErr = "wrong Node Kind for spec.containers expected: " + - "MappingNode was SequenceNode: value: {- name: myapp-container\n image: busybox}" - testCases[18].expectedErr = "wrong Node Kind for expected: " + - "ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}" - for i, tc := range testCases { tc := tc t.Run(fmt.Sprintf("Test Case %d", i), func(t *testing.T) {