Merge "Remove legacy code from ReplacementTransformer"

This commit is contained in:
Zuul 2020-10-26 21:22:22 +00:00 committed by Gerrit Code Review
commit 19360953b0
10 changed files with 66 additions and 439 deletions

View File

@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: baremetal-operator-env-vars-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
# Replace the proxy vars
- source:

View File

@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: baremetal-operator-networking-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
# Replace the pod & service networks
- source:

View File

@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: calico-v3-versions-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
- source:
objref:

View File

@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: ephemeral-env-vars-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
# Replace the proxy vars
- source:

View File

@ -3,6 +3,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: ephemeral-networking-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
# Substring-replace the ephemeral control plane's info
- source:

View File

@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: helm-operator-versions-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
- source:
objref:

View File

@ -2,6 +2,10 @@ apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: hwcc-versions-replacements
annotations:
config.kubernetes.io/function: |-
container:
image: quay.io/airshipit/replacement-transformer:dev
replacements:
- source:
objref:

View File

@ -22,23 +22,6 @@ import (
"sigs.k8s.io/kustomize/api/types"
)
// ErrTypeMismatch is returned for type conversion errors. This error
// is raised if JSON path element points to a wrong data structure e.g.
// JSON path 'a.b[x=y]c' considers that there is a list of maps under key 'b'
// therefore ErrTypeMismatch will be returned for following structure
//
// a:
// b:
// - 'some string'
type ErrTypeMismatch struct {
Actual interface{}
Expectation string
}
func (e ErrTypeMismatch) Error() string {
return fmt.Sprintf("%#v %s", e.Actual, e.Expectation)
}
// ErrBadConfiguration returned in case of plugin misconfiguration
type ErrBadConfiguration struct {
Msg string
@ -94,15 +77,6 @@ func (e ErrIndexOutOfBound) Error() string {
return fmt.Sprintf("array index out of bounds: index %d, length %d", e.Index, e.Length)
}
// ErrValueNotFound returned if value specified in fieldRef option was not found
type ErrValueNotFound struct {
ID string
}
func (e ErrValueNotFound) Error() string {
return fmt.Sprintf("unable to find value identified by %s", e.ID)
}
func printFields(objRef interface{}) string {
val := reflect.ValueOf(objRef).Elem()
valType := val.Type()

View File

@ -1,39 +1,39 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package replacement
import (
"fmt"
"io"
"io/ioutil"
"reflect"
"regexp"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/api/resource"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/kustomize/kyaml/yaml"
airshipv1 "opendev.org/airship/airshipctl/pkg/api/v1alpha1"
"opendev.org/airship/airshipctl/pkg/document/plugin/kyamlutils"
plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
"opendev.org/airship/airshipctl/pkg/errors"
)
var (
pattern = regexp.MustCompile(`(\S+)\[(\S+)=(\S+)\]`)
// substring substitutions are appended to paths as: ...%VARNAME%
substringPatternRegex = regexp.MustCompile(`(.+)%(\S+)%$`)
)
const (
dotReplacer = "$$$$"
)
var _ plugtypes.Plugin = &plugin{}
type plugin struct {
@ -63,56 +63,7 @@ func New(obj map[string]interface{}) (plugtypes.Plugin, error) {
}
func (p *plugin) Run(in io.Reader, out io.Writer) error {
data, err := ioutil.ReadAll(in)
if err != nil {
return err
}
kf := kunstruct.NewKunstructuredFactoryImpl()
rf := resource.NewFactory(kf)
resources, err := rf.SliceFromBytes(data)
if err != nil {
return err
}
rm := resmap.New()
for _, r := range resources {
if err = rm.Append(r); err != nil {
return err
}
}
if err = p.Transform(rm); err != nil {
return err
}
result, err := rm.AsYaml()
if err != nil {
return err
}
fmt.Fprint(out, string(result))
return nil
}
// Transform resources using configured replacements
func (p *plugin) Transform(m resmap.ResMap) error {
var err error
for _, r := range p.Replacements {
var replacement interface{}
if r.Source.ObjRef != nil {
replacement, err = getReplacement(m, r.Source.ObjRef, r.Source.FieldRef)
if err != nil {
return err
}
}
if r.Source.Value != "" {
replacement = r.Source.Value
}
if err = substitute(m, r.Target, replacement); err != nil {
return err
}
}
return nil
return errors.ErrNotImplemented{What: "ReplacementTransformer method Run is deprecated and will be removed"}
}
func (p *plugin) Filter(items []*yaml.RNode) ([]*yaml.RNode, error) {
@ -229,256 +180,3 @@ func substituteSubstring(tgt *yaml.RNode, fieldRef, substringPattern string, val
}
return nil
}
func getReplacement(m resmap.ResMap, objRef *types.Target, fieldRef string) (interface{}, error) {
s := types.Selector{
Gvk: objRef.Gvk,
Name: objRef.Name,
Namespace: objRef.Namespace,
}
resources, err := m.Select(s)
if err != nil {
return nil, err
}
if len(resources) > 1 {
resList := make([]string, len(resources))
for i := range resources {
resList[i] = resources[i].String()
}
return nil, ErrMultipleResources{ObjRef: objRef}
}
if len(resources) == 0 {
return nil, ErrSourceNotFound{ObjRef: objRef}
}
if fieldRef == "" {
fieldRef = "metadata.name"
}
return resources[0].GetFieldValue(fieldRef)
}
func substitute(m resmap.ResMap, to *types.ReplTarget, replacement interface{}) error {
resources, err := m.Select(*to.ObjRef)
if err != nil {
return err
}
if len(resources) == 0 {
return ErrTargetNotFound{ObjRef: to.ObjRef}
}
for _, r := range resources {
for _, p := range to.FieldRefs {
// TODO (dukov) rework this using k8s.io/client-go/util/jsonpath
parts := strings.Split(p, "[")
var tmp []string
for _, part := range parts {
if strings.Contains(part, "]") {
filter := strings.Split(part, "]")
filter[0] = strings.ReplaceAll(filter[0], ".", dotReplacer)
part = strings.Join(filter, "]")
}
tmp = append(tmp, part)
}
p = strings.Join(tmp, "[")
// Exclude substring portion from dot replacer
// substring can contain IP or any dot separated string
substringPattern := ""
p, substringPattern = extractSubstringPattern(p)
pathSlice := strings.Split(p, ".")
// append back the extracted substring
if len(substringPattern) > 0 {
pathSlice[len(pathSlice)-1] = pathSlice[len(pathSlice)-1] + "%" +
substringPattern + "%"
}
for i, part := range pathSlice {
pathSlice[i] = strings.ReplaceAll(part, dotReplacer, ".")
}
if err := updateField(r.Map(), pathSlice, replacement); err != nil {
return err
}
}
}
return nil
}
func getFirstPathSegment(path string) (field string, key string, value string, isArray bool) {
groups := pattern.FindStringSubmatch(path)
if len(groups) != 4 {
return path, "", "", false
}
return groups[1], groups[2], groups[3], groups[2] != ""
}
func updateField(m interface{}, pathToField []string, replacement interface{}) error {
if len(pathToField) == 0 {
return nil
}
switch typedM := m.(type) {
case map[string]interface{}:
return updateMapField(typedM, pathToField, replacement)
case []interface{}:
return updateSliceField(typedM, pathToField, replacement)
default:
return ErrTypeMismatch{Actual: typedM, Expectation: "is not expected be a primitive type"}
}
}
// Extract the substring pattern (if present) from the target path spec
func extractSubstringPattern(path string) (extractedPath string, substringPattern string) {
groups := substringPatternRegex.FindStringSubmatch(path)
if len(groups) != 3 {
return path, ""
}
return groups[1], groups[2]
}
// replaces substring in a string if pattern applies
func processString(field string, substringPattern string, replacement string) string {
pattern := regexp.MustCompile(substringPattern)
return pattern.ReplaceAllString(field, replacement)
}
// replaces substring in any string in the array if pattern applies
func processArray(tgt []string, pattern string, replacement string) []string {
result := make([]string, 0, len(tgt))
for _, field := range tgt {
result = append(result, processString(field, pattern, replacement))
}
return result
}
// converts array of interfaces to array of strings
func convertToStrings(iFaces []interface{}) ([]string, bool) {
result := []string{}
for _, val := range iFaces {
if str, ok := val.(string); ok {
result = append(result, str)
}
}
return result, (len(result) != 0)
}
// apply a substring substitution based on a pattern
func applySubstringPattern(target interface{}, replacement interface{},
substringPattern string) (regexedReplacement interface{}, err error) {
// no regex'ing needed if there is no substringPattern
if substringPattern == "" {
return replacement, nil
}
// if replacement is numeric, convert it to a string for the sake of
// substring substitution.
var replacementString string
switch t := replacement.(type) {
case uint, uint8, uint16, uint32, uint64, int, int8, int16, int32, int64:
replacementString = fmt.Sprint(t)
case string:
replacementString = t
default:
return nil, ErrPatternSubstring{Msg: "pattern-based substitution can only be applied " +
"with string or numeric replacement values"}
}
switch reflect.TypeOf(target).Kind() {
case reflect.String:
return processString(target.(string), substringPattern, replacementString), nil
case reflect.Slice:
if ifaceArray, ok := target.([]interface{}); ok {
if strArray, ok := convertToStrings(ifaceArray); ok {
return processArray(strArray, substringPattern, replacementString), nil
}
}
if strArray, ok := target.([]string); ok {
return processArray(strArray, substringPattern, replacementString), nil
}
}
return nil, ErrPatternSubstring{
Msg: "pattern-based substitution can only be applied to string " +
"or array of strings target fields",
}
}
func updateMapField(m map[string]interface{}, pathToField []string, replacement interface{}) error {
path, key, value, isArray := getFirstPathSegment(pathToField[0])
path, substringPattern := extractSubstringPattern(path)
v, found := m[path]
if !found {
m[path] = make(map[string]interface{})
v = m[path]
}
if v == nil {
return ErrTypeMismatch{Actual: v, Expectation: "is not expected be nil"}
}
if len(pathToField) == 1 {
if !isArray {
renderedRepl, err := applySubstringPattern(m[path], replacement, substringPattern)
if err != nil {
return err
}
m[path] = renderedRepl
return nil
}
switch typedV := v.(type) {
case []interface{}:
for i, item := range typedV {
typedItem, ok := item.(map[string]interface{})
if !ok {
return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)}
}
if actualValue, ok := typedItem[key]; ok {
if value == actualValue {
typedV[i] = replacement
return nil
}
}
}
default:
return ErrTypeMismatch{Actual: typedV, Expectation: "is not expected be a primitive type"}
}
}
if isArray {
return updateField(v, pathToField, replacement)
}
return updateField(v, pathToField[1:], replacement)
}
func updateSliceField(m []interface{}, pathToField []string, replacement interface{}) error {
if len(pathToField) == 0 {
return nil
}
_, key, value, isArray := getFirstPathSegment(pathToField[0])
if isArray {
for _, item := range m {
typedItem, ok := item.(map[string]interface{})
if !ok {
return ErrTypeMismatch{Actual: item, Expectation: fmt.Sprintf("is expected to be %T", typedItem)}
}
if actualValue, ok := typedItem[key]; ok {
if value == actualValue {
return updateField(typedItem, pathToField[1:], replacement)
}
}
}
return nil
}
index, err := strconv.Atoi(pathToField[0])
if err != nil {
return err
}
if len(m) < index || index < 0 {
return ErrIndexOutOfBound{Index: index, Length: len(m)}
}
if len(pathToField) == 1 {
m[index] = replacement
return nil
}
return updateField(m[index], pathToField[1:], replacement)
}

View File

@ -1,12 +1,22 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package replacement_test
import (
"bytes"
"fmt"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@ -16,62 +26,8 @@ import (
"sigs.k8s.io/yaml"
"opendev.org/airship/airshipctl/pkg/document/plugin/replacement"
plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
)
func samplePlugin(t *testing.T) plugtypes.Plugin {
cfg := make(map[string]interface{})
conf := `
apiVersion: airshipit.org/v1alpha1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
value: nginx:newtag
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[name=nginx-latest].image`
err := yaml.Unmarshal([]byte(conf), &cfg)
require.NoError(t, err)
plugin, err := replacement.New(cfg)
require.NoError(t, err)
return plugin
}
func TestMalformedInput(t *testing.T) {
plugin := samplePlugin(t)
err := plugin.Run(strings.NewReader("--"), &bytes.Buffer{})
assert.Error(t, err)
}
func TestDuplicatedResources(t *testing.T) {
plugin := samplePlugin(t)
err := plugin.Run(strings.NewReader(`
apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- name: myapp-container
image: busybox
---
apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- name: myapp-container
image: busybox
`), &bytes.Buffer{})
assert.Error(t, err)
}
var testCases = []struct {
cfg string
in string
@ -804,7 +760,8 @@ spec:
containers:
- name: myapp-container
image: busybox`,
expectedErr: `"some string value" is not expected be a primitive type`,
expectedErr: "wrong Node Kind for labels.somelabel expected: " +
"MappingNode was ScalarNode: value: {'some string value'}",
},
{
cfg: `
@ -843,7 +800,8 @@ spec:
containers:
- name: myapp-container
image: busybox`,
expectedErr: `"some string value" is not expected be a primitive type`,
expectedErr: "wrong Node Kind for labels.somelabel expected: " +
"SequenceNode was ScalarNode: value: {'some string value'}",
},
{
cfg: `
@ -882,9 +840,8 @@ spec:
containers:
- name: myapp-container
image: busybox`,
expectedErr: "map[string]interface {}{\"containers\":[]interface " +
"{}{map[string]interface {}{\"image\":\"busybox\", \"name\":\"myapp-container\"}}} " +
"is not expected be a primitive type",
expectedErr: "wrong Node Kind for spec expected: " +
"SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n image: busybox}",
},
{
cfg: `
@ -962,7 +919,8 @@ spec:
containers:
- name: myapp-container
image: busybox`,
expectedErr: `strconv.Atoi: parsing "notInteger": invalid syntax`,
expectedErr: "wrong Node Kind for spec.containers expected: " +
"MappingNode was SequenceNode: value: {- name: myapp-container\n image: busybox}",
},
{
cfg: `
@ -1001,7 +959,8 @@ spec:
containers:
- image: nginx:TAG
name: nginx-latest`,
expectedErr: "pattern-based substitution can only be applied to string or array of strings target fields",
expectedErr: "wrong Node Kind for expected: " +
"ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}",
},
{
cfg: `
@ -1058,39 +1017,7 @@ spec:
},
}
func TestReplacementTransformer(t *testing.T) {
for _, tc := range testCases {
cfg := make(map[string]interface{})
err := yaml.Unmarshal([]byte(tc.cfg), &cfg)
require.NoError(t, err)
plugin, err := replacement.New(cfg)
require.NoError(t, err)
buf := &bytes.Buffer{}
err = plugin.Run(strings.NewReader(tc.in), buf)
errString := ""
if err != nil {
errString = err.Error()
}
assert.Equal(t, tc.expectedErr, errString)
assert.Equal(t, tc.expectedOut, buf.String())
}
}
func TestExec(t *testing.T) {
// TODO (dukov) Remove this once we migrate to new kustomize plugin approach
// NOTE (dukov) we need this since error format is different for new kustomize plugins
testCases[13].expectedErr = "wrong Node Kind for labels.somelabel expected: " +
"MappingNode was ScalarNode: value: {'some string value'}"
testCases[14].expectedErr = "wrong Node Kind for labels.somelabel expected: " +
"SequenceNode was ScalarNode: value: {'some string value'}"
testCases[15].expectedErr = "wrong Node Kind for spec expected: " +
"SequenceNode was MappingNode: value: {containers:\n- name: myapp-container\n image: busybox}"
testCases[17].expectedErr = "wrong Node Kind for spec.containers expected: " +
"MappingNode was SequenceNode: value: {- name: myapp-container\n image: busybox}"
testCases[18].expectedErr = "wrong Node Kind for expected: " +
"ScalarNode was MappingNode: value: {image: nginx:TAG\nname: nginx-latest}"
for i, tc := range testCases {
tc := tc
t.Run(fmt.Sprintf("Test Case %d", i), func(t *testing.T) {