Merge "Remove legacy code from ReplacementTransformer"

manifests/function/baremetal-operator/replacements/ironic-env-vars.yaml

@@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: baremetal-operator-env-vars-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 # Replace the proxy vars
 - source:

manifests/function/baremetal-operator/replacements/networking.yaml

@@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: baremetal-operator-networking-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 # Replace the pod & service networks
 - source:

manifests/function/cni/calico/v3/replacements/versions.yaml

@@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: calico-v3-versions-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 - source:
     objref:

manifests/function/ephemeral/replacements/ephemeral-env-vars.yaml

@@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: ephemeral-env-vars-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 # Replace the proxy vars
 - source:

manifests/function/ephemeral/replacements/networking.yaml

@@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: ephemeral-networking-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 # Substring-replace the ephemeral control plane's info
 - source:

manifests/function/helm-operator/replacements/versions.yaml

@@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: helm-operator-versions-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 - source:
     objref:

manifests/function/hwcc/replacements/versions.yaml

@@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
 kind: ReplacementTransformer
 metadata:
   name: hwcc-versions-replacements
+  annotations:
+    config.kubernetes.io/function: |-
+      container:
+        image: quay.io/airshipit/replacement-transformer:dev
 replacements:
 - source:
     objref:

pkg/document/plugin/replacement/errors.go

@@ -22,23 +22,6 @@ import (
 	"sigs.k8s.io/kustomize/api/types"
 )
 
-// ErrTypeMismatch is returned for type conversion errors. This error
-// is raised if JSON path element points to a wrong data structure e.g.
-// JSON path 'a.b[x=y]c' considers that there is a list of maps under key 'b'
-// therefore ErrTypeMismatch will be returned for following structure
-//
-// a:
-//   b:
-//     - 'some string'
-type ErrTypeMismatch struct {
-	Actual      interface{}
-	Expectation string
-}
-
-func (e ErrTypeMismatch) Error() string {
-	return fmt.Sprintf("%#v %s", e.Actual, e.Expectation)
-}
-
 // ErrBadConfiguration returned in case of plugin misconfiguration
 type ErrBadConfiguration struct {
 	Msg string
@@ -94,15 +77,6 @@ func (e ErrIndexOutOfBound) Error() string {
 	return fmt.Sprintf("array index out of bounds: index %d, length %d", e.Index, e.Length)
 }
 
-// ErrValueNotFound returned if value specified in fieldRef option was not found
-type ErrValueNotFound struct {
-	ID string
-}
-
-func (e ErrValueNotFound) Error() string {
-	return fmt.Sprintf("unable to find value identified by %s", e.ID)
-}
-
 func printFields(objRef interface{}) string {
 	val := reflect.ValueOf(objRef).Elem()
 	valType := val.Type()

pkg/document/plugin/replacement/transformer.go

@@ -1,39 +1,39 @@
-// Copyright 2019 The Kubernetes Authors.
+/*
-// SPDX-License-Identifier: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
 
 package replacement
 
 import (
 	"fmt"
 	"io"
-	"io/ioutil"
-	"reflect"
 	"regexp"
-	"strconv"
-	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
-	"sigs.k8s.io/kustomize/api/resmap"
-	"sigs.k8s.io/kustomize/api/resource"
 	"sigs.k8s.io/kustomize/api/types"
 	"sigs.k8s.io/kustomize/kyaml/yaml"
 
 	airshipv1 "opendev.org/airship/airshipctl/pkg/api/v1alpha1"
 	"opendev.org/airship/airshipctl/pkg/document/plugin/kyamlutils"
 	plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
+	"opendev.org/airship/airshipctl/pkg/errors"
 )
 
 var (
-	pattern = regexp.MustCompile(`(\S+)\[(\S+)=(\S+)\]`)
 	// substring substitutions are appended to paths as: ...%VARNAME%
 	substringPatternRegex = regexp.MustCompile(`(.+)%(\S+)%$`)
 )
 
-const (
-	dotReplacer = "$$$$"
-)
-
 var _ plugtypes.Plugin = &plugin{}
 
 type plugin struct {
@@ -63,56 +63,7 @@ func New(obj map[string]interface{}) (plugtypes.Plugin, error) {
 }
 
 func (p *plugin) Run(in io.Reader, out io.Writer) error {
-	data, err := ioutil.ReadAll(in)
+	return errors.ErrNotImplemented{What: "ReplacementTransformer method Run is deprecated and will be removed"}
-	if err != nil {
-		return err
-	}
-
-	kf := kunstruct.NewKunstructuredFactoryImpl()
-	rf := resource.NewFactory(kf)
-	resources, err := rf.SliceFromBytes(data)
-	if err != nil {
-		return err
-	}
-
-	rm := resmap.New()
-	for _, r := range resources {
-		if err = rm.Append(r); err != nil {
-			return err
-		}
-	}
-
-	if err = p.Transform(rm); err != nil {
-		return err
-	}
-
-	result, err := rm.AsYaml()
-	if err != nil {
-		return err
-	}
-	fmt.Fprint(out, string(result))
-	return nil
-}
-
-// Transform resources using configured replacements
-func (p *plugin) Transform(m resmap.ResMap) error {
-	var err error
-	for _, r := range p.Replacements {
-		var replacement interface{}
-		if r.Source.ObjRef != nil {
-			replacement, err = getReplacement(m, r.Source.ObjRef, r.Source.FieldRef)
-			if err != nil {
-				return err
-			}
-		}
-		if r.Source.Value != "" {
-			replacement = r.Source.Value
-		}
-		if err = substitute(m, r.Target, replacement); err != nil {
-			return err
-		}
-	}
-	return nil
 }
 
 func (p *plugin) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
@@ -229,256 +180,3 @@ func substituteSubstring(tgt *yaml.RNode, fieldRef, substringPattern string, val
 	}
 	return nil
 }
-
-func getReplacement(m resmap.ResMap, objRef *types.Target, fieldRef string) (interface{}, error) {
-	s := types.Selector{
-		Gvk:       objRef.Gvk,
-		Name:      objRef.Name,
-		Namespace: objRef.Namespace,
-	}
-	resources, err := m.Select(s)
-	if err != nil {
-		return nil, err
-	}
-	if len(resources) > 1 {
-		resList := make([]string, len(resources))
-		for i := range resources {
-			resList[i] = resources[i].String()
-		}
-		return nil, ErrMultipleResources{ObjRef: objRef}
-	}
-	if len(resources) == 0 {
-		return nil, ErrSourceNotFound{ObjRef: objRef}
-	}
-	if fieldRef == "" {
-		fieldRef = "metadata.name"
-	}
-	return resources[0].GetFieldValue(fieldRef)
-}
-
-func substitute(m resmap.ResMap, to *types.ReplTarget, replacement interface{}) error {
-	resources, err := m.Select(*to.ObjRef)
-	if err != nil {
-		return err
-	}
-	if len(resources) == 0 {
-		return ErrTargetNotFound{ObjRef: to.ObjRef}
-	}
-	for _, r := range resources {
-		for _, p := range to.FieldRefs {
-			// TODO (dukov) rework this using k8s.io/client-go/util/jsonpath
-			parts := strings.Split(p, "[")
-			var tmp []string
-			for _, part := range parts {
-				if strings.Contains(part, "]") {
-					filter := strings.Split(part, "]")
-					filter[0] = strings.ReplaceAll(filter[0], ".", dotReplacer)
-					part = strings.Join(filter, "]")
-				}
-				tmp = append(tmp, part)
-			}
-			p = strings.Join(tmp, "[")
-			// Exclude substring portion from dot replacer
-			// substring can contain IP or any dot separated string
-			substringPattern := ""
-			p, substringPattern = extractSubstringPattern(p)
-
-			pathSlice := strings.Split(p, ".")
-			// append back the extracted substring
-			if len(substringPattern) > 0 {
-				pathSlice[len(pathSlice)-1] = pathSlice[len(pathSlice)-1] + "%" +
-					substringPattern + "%"
-			}
-			for i, part := range pathSlice {
-				pathSlice[i] = strings.ReplaceAll(part, dotReplacer, ".")
-			}
-			if err := updateField(r.Map(), pathSlice, replacement); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func getFirstPathSegment(path string) (field string, key string, value string, isArray bool) {
-	groups := pattern.FindStringSubmatch(path)
-	if len(groups) != 4 {
-		return path, "", "", false
-	}
-	return groups[1], groups[2], groups[3], groups[2] != ""
-}
-
-func updateField(m interface{}, pathToField []string, replacement interface{}) error {
-	if len(pathToField) == 0 {
-		return nil
-	}
-
-	switch typedM := m.(type) {
-	case map[string]interface{}:
-		return updateMapField(typedM, pathToField, replacement)
-	case []interface{}:
-		return updateSliceField(typedM, pathToField, replacement)
-	default:
-		return ErrTypeMismatch{Actual: typedM, Expectation: "is not expected be a primitive type"}
-	}
-}
-
-// Extract the substring pattern (if present) from the target path spec
-func extractSubstringPattern(path string) (extractedPath string, substringPattern string) {
-	groups := substringPatternRegex.FindStringSubmatch(path)
-	if len(groups) != 3 {
-		return path, ""
-	}
-	return groups[1], groups[2]
-}
-
-// replaces substring in a string if pattern applies
-func processString(field string, substringPattern string, replacement string) string {
-	pattern := regexp.MustCompile(substringPattern)
-	return pattern.ReplaceAllString(field, replacement)
-}
-
-// replaces substring in any string in the array if pattern applies
-func processArray(tgt []string, pattern string, replacement string) []string {
-	result := make([]string, 0, len(tgt))
-	for _, field := range tgt {
-		result = append(result, processString(field, pattern, replacement))
-	}
-	return result
-}
-
-// converts array of interfaces to array of strings
-func convertToStrings(iFaces []interface{}) ([]string, bool) {
-	result := []string{}
-	for _, val := range iFaces {
-		if str, ok := val.(string); ok {
-			result = append(result, str)
-		}
-	}
-	return result, (len(result) != 0)
-}
-
-// apply a substring substitution based on a pattern
-func applySubstringPattern(target interface{}, replacement interface{},
-	substringPattern string) (regexedReplacement interface{}, err error) {
-	// no regex'ing needed if there is no substringPattern
-	if substringPattern == "" {
-		return replacement, nil
-	}
-
-	// if replacement is numeric, convert it to a string for the sake of
-	// substring substitution.
-	var replacementString string
-	switch t := replacement.(type) {
-	case uint, uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
-		replacementString = fmt.Sprint(t)
-	case string:
-		replacementString = t
-	default:
-		return nil, ErrPatternSubstring{Msg: "pattern-based substitution can only be applied " +
-			"with string or numeric replacement values"}
-	}
-
-	switch reflect.TypeOf(target).Kind() {
-	case reflect.String:
-		return processString(target.(string), substringPattern, replacementString), nil
-	case reflect.Slice:
-		if ifaceArray, ok := target.([]interface{}); ok {
-			if strArray, ok := convertToStrings(ifaceArray); ok {
-				return processArray(strArray, substringPattern, replacementString), nil
-			}
-		}
-		if strArray, ok := target.([]string); ok {
-			return processArray(strArray, substringPattern, replacementString), nil
-		}
-	}
-	return nil, ErrPatternSubstring{
-		Msg: "pattern-based substitution can only be applied to string " +
-			"or array of strings target fields",
-	}
-}
-
-func updateMapField(m map[string]interface{}, pathToField []string, replacement interface{}) error {
-	path, key, value, isArray := getFirstPathSegment(pathToField[0])
-
-	path, substringPattern := extractSubstringPattern(path)
-
-	v, found := m[path]
-	if !found {
-		m[path] = make(map[string]interface{})
-		v = m[path]
-	}
-
-	if v == nil {
-		return ErrTypeMismatch{Actual: v, Expectation: "is not expected be nil"}
-	}
-
-	if len(pathToField) == 1 {
-		if !isArray {
-			renderedRepl, err := applySubstringPattern(m[path], replacement, substringPattern)
-			if err != nil {
-				return err
-			}
-			m[path] = renderedRepl
-			return nil
-		}
-		switch typedV := v.(type) {
-		case []interface{}:
-			for i, item := range typedV {
-				typedItem, ok := item.(map[string]interface{})
-				if !ok {
-					return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)}
-				}
-				if actualValue, ok := typedItem[key]; ok {
-					if value == actualValue {
-						typedV[i] = replacement
-						return nil
-					}
-				}
-			}
-		default:
-			return ErrTypeMismatch{Actual: typedV, Expectation: "is not expected be a primitive type"}
-		}
-	}
-
-	if isArray {
-		return updateField(v, pathToField, replacement)
-	}
-	return updateField(v, pathToField[1:], replacement)
-}
-
-func updateSliceField(m []interface{}, pathToField []string, replacement interface{}) error {
-	if len(pathToField) == 0 {
-		return nil
-	}
-	_, key, value, isArray := getFirstPathSegment(pathToField[0])
-
-	if isArray {
-		for _, item := range m {
-			typedItem, ok := item.(map[string]interface{})
-			if !ok {
-				return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)}
-			}
-			if actualValue, ok := typedItem[key]; ok {
-				if value == actualValue {
-					return updateField(typedItem, pathToField[1:], replacement)
-				}
-			}
-		}
-		return nil
-	}
-
-	index, err := strconv.Atoi(pathToField[0])
-	if err != nil {
-		return err
-	}
-
-	if len(m) < index || index < 0 {
-		return ErrIndexOutOfBound{Index: index, Length: len(m)}
-	}
-	if len(pathToField) == 1 {
-		m[index] = replacement
-		return nil
-	}
-	return updateField(m[index], pathToField[1:], replacement)
-}

pkg/document/plugin/replacement/transformer_test.go

@@ -1,12 +1,22 @@
-// Copyright 2019 The Kubernetes Authors.
+/*
-// SPDX-License-Identifier: Apache-2.0
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
 
 package replacement_test
 
 import (
 	"bytes"
 	"fmt"
-	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -16,62 +26,8 @@ import (
 	"sigs.k8s.io/yaml"
 
 	"opendev.org/airship/airshipctl/pkg/document/plugin/replacement"
-	plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
 )
 
-func samplePlugin(t *testing.T) plugtypes.Plugin {
-	cfg := make(map[string]interface{})
-	conf := `
-apiVersion: airshipit.org/v1alpha1
-kind: ReplacementTransformer
-metadata:
-  name: notImportantHere
-replacements:
-- source:
-    value: nginx:newtag
-  target:
-    objref:
-      kind: Deployment
-    fieldrefs:
-    - spec.template.spec.containers[name=nginx-latest].image`
-
-	err := yaml.Unmarshal([]byte(conf), &cfg)
-	require.NoError(t, err)
-	plugin, err := replacement.New(cfg)
-	require.NoError(t, err)
-	return plugin
-}
-
-func TestMalformedInput(t *testing.T) {
-	plugin := samplePlugin(t)
-	err := plugin.Run(strings.NewReader("--"), &bytes.Buffer{})
-	assert.Error(t, err)
-}
-
-func TestDuplicatedResources(t *testing.T) {
-	plugin := samplePlugin(t)
-	err := plugin.Run(strings.NewReader(`
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod
-spec:
-  containers:
-  - name: myapp-container
-    image: busybox
----
-apiVersion: v1
-kind: Pod
-metadata:
-  name: pod
-spec:
-  containers:
-  - name: myapp-container
-    image: busybox
- `), &bytes.Buffer{})
-	assert.Error(t, err)
-}
-
 var testCases = []struct {
 	cfg         string
 	in          string
@@ -804,7 +760,8 @@ spec:
   containers:
   - name: myapp-container
     image: busybox`,
-		expectedErr: `"some string value" is not expected be a primitive type`,
+		expectedErr: "wrong Node Kind for labels.somelabel expected: " +
+			"MappingNode was ScalarNode: value: {'some string value'}",
 	},
 	{
 		cfg: `
@@ -843,7 +800,8 @@ spec:
   containers:
   - name: myapp-container
     image: busybox`,
-		expectedErr: `"some string value" is not expected be a primitive type`,
+		expectedErr: "wrong Node Kind for labels.somelabel expected: " +
+			"SequenceNode was ScalarNode: value: {'some string value'}",
 	},
 	{
 		cfg: `
@@ -882,9 +840,8 @@ spec:
   containers:
   - name: myapp-container
     image: busybox`,
-		expectedErr: "map[string]interface {}{\"containers\":[]interface " +
+		expectedErr: "wrong Node Kind for spec expected: " +
-			"{}{map[string]interface {}{\"image\":\"busybox\", \"name\":\"myapp-container\"}}} " +
+			"SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n  image: busybox}",
-			"is not expected be a primitive type",
 	},
 	{
 		cfg: `
@@ -962,7 +919,8 @@ spec:
   containers:
   - name: myapp-container
     image: busybox`,
-		expectedErr: `strconv.Atoi: parsing "notInteger": invalid syntax`,
+		expectedErr: "wrong Node Kind for spec.containers expected: " +
+			"MappingNode was SequenceNode: value: {- name: myapp-container\n  image: busybox}",
 	},
 	{
 		cfg: `
@@ -1001,7 +959,8 @@ spec:
       containers:
       - image: nginx:TAG
         name: nginx-latest`,
-		expectedErr: "pattern-based substitution can only be applied to string or array of strings target fields",
+		expectedErr: "wrong Node Kind for  expected: " +
+			"ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}",
 	},
 	{
 		cfg: `
@@ -1058,39 +1017,7 @@ spec:
 	},
 }
 
-func TestReplacementTransformer(t *testing.T) {
-	for _, tc := range testCases {
-		cfg := make(map[string]interface{})
-		err := yaml.Unmarshal([]byte(tc.cfg), &cfg)
-		require.NoError(t, err)
-		plugin, err := replacement.New(cfg)
-		require.NoError(t, err)
-
-		buf := &bytes.Buffer{}
-		err = plugin.Run(strings.NewReader(tc.in), buf)
-		errString := ""
-		if err != nil {
-			errString = err.Error()
-		}
-		assert.Equal(t, tc.expectedErr, errString)
-		assert.Equal(t, tc.expectedOut, buf.String())
-	}
-}
-
 func TestExec(t *testing.T) {
-	// TODO (dukov) Remove this once we migrate to new kustomize plugin approach
-	// NOTE (dukov) we need this since error format is different for new kustomize plugins
-	testCases[13].expectedErr = "wrong Node Kind for labels.somelabel expected: " +
-		"MappingNode was ScalarNode: value: {'some string value'}"
-	testCases[14].expectedErr = "wrong Node Kind for labels.somelabel expected: " +
-		"SequenceNode was ScalarNode: value: {'some string value'}"
-	testCases[15].expectedErr = "wrong Node Kind for spec expected: " +
-		"SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n  image: busybox}"
-	testCases[17].expectedErr = "wrong Node Kind for spec.containers expected: " +
-		"MappingNode was SequenceNode: value: {- name: myapp-container\n  image: busybox}"
-	testCases[18].expectedErr = "wrong Node Kind for  expected: " +
-		"ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}"
-
 	for i, tc := range testCases {
 		tc := tc
 		t.Run(fmt.Sprintf("Test Case %d", i), func(t *testing.T) {