Add replacement transformer

* Replace paramter specified by JSON path with predefined value or
  with the value from another resource
* Replace substring in a paramter specified by JSON path with predefined
  value or with the value from another resource

Transformer copied from
https://github.com/mattmceuen/kustomize/tree/substring-subst

Closes: #174
Change-Id: I3a958a0df724fb2eb81bb199a02cf1db81bb0d2f
Co-Authored-By: Matt McEuen <matt.mceuen@att.com>
This commit is contained in:
Dmitry Ukov 2020-04-17 20:15:24 +04:00
parent c62a369fd0
commit 6c716e1a57
5 changed files with 787 additions and 0 deletions

View File

@ -0,0 +1,27 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package replacement
import (
"k8s.io/apimachinery/pkg/runtime/schema"
replv1alpha1 "opendev.org/airship/airshipctl/pkg/document/plugin/replacement/v1alpha1"
"opendev.org/airship/airshipctl/pkg/document/plugin/types"
)
// RegisterPlugin registers BareMetalHost generator plugin
func RegisterPlugin(registry map[schema.GroupVersionKind]types.Factory) {
registry[replv1alpha1.GetGVK()] = replv1alpha1.New
}

View File

@ -0,0 +1,323 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package v1alpha1
import (
"fmt"
"io"
"io/ioutil"
"regexp"
"strconv"
"strings"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/kustomize/api/k8sdeps/kunstruct"
"sigs.k8s.io/kustomize/api/resmap"
"sigs.k8s.io/kustomize/api/resource"
"sigs.k8s.io/kustomize/api/types"
"sigs.k8s.io/yaml"
plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
"opendev.org/airship/airshipctl/pkg/environment"
)
var (
pattern = regexp.MustCompile(`(\S+)\[(\S+)=(\S+)\]`)
// substring substitutions are appended to paths as: ...%VARNAME%
substringPatternRegex = regexp.MustCompile(`(\S+)%(\S+)%$`)
)
// GetGVK returns group, version, kind object used to register version
// of the plugin
func GetGVK() schema.GroupVersionKind {
return schema.GroupVersionKind{
Group: "airshipit.org",
Version: "v1alpha1",
Kind: "ReplacementTransformer",
}
}
// New creates new instance of the plugin
func New(_ *environment.AirshipCTLSettings, cfg []byte) (plugtypes.Plugin, error) {
p := &plugin{}
if err := p.Config(nil, cfg); err != nil {
return nil, err
}
return p, nil
}
func (p *plugin) Run(in io.Reader, out io.Writer) error {
data, err := ioutil.ReadAll(in)
if err != nil {
return err
}
kf := kunstruct.NewKunstructuredFactoryImpl()
rf := resource.NewFactory(kf)
resources, err := rf.SliceFromBytes(data)
if err != nil {
return err
}
rm := resmap.New()
for _, r := range resources {
if err = rm.Append(r); err != nil {
return err
}
}
if err = p.Transform(rm); err != nil {
return err
}
result, err := rm.AsYaml()
if err != nil {
return err
}
fmt.Fprint(out, string(result))
return nil
}
func (p *plugin) Config(
_ *resmap.PluginHelpers, c []byte) (err error) {
p.Replacements = []types.Replacement{}
err = yaml.Unmarshal(c, p)
if err != nil {
return err
}
for _, r := range p.Replacements {
if r.Source == nil {
return fmt.Errorf("`from` must be specified in one replacement")
}
if r.Target == nil {
return fmt.Errorf("`to` must be specified in one replacement")
}
count := 0
if r.Source.ObjRef != nil {
count += 1
}
if r.Source.Value != "" {
count += 1
}
if count > 1 {
return fmt.Errorf("only one of fieldref and value is allowed in one replacement")
}
}
return nil
}
func (p *plugin) Transform(m resmap.ResMap) (err error) {
for _, r := range p.Replacements {
var replacement interface{}
if r.Source.ObjRef != nil {
replacement, err = getReplacement(m, r.Source.ObjRef, r.Source.FieldRef)
if err != nil {
return err
}
}
if r.Source.Value != "" {
replacement = r.Source.Value
}
err = substitute(m, r.Target, replacement)
if err != nil {
return err
}
}
return nil
}
func getReplacement(m resmap.ResMap, objRef *types.Target, fieldRef string) (interface{}, error) {
s := types.Selector{
Gvk: objRef.Gvk,
Name: objRef.Name,
Namespace: objRef.Namespace,
}
resources, err := m.Select(s)
if err != nil {
return "", err
}
if len(resources) > 1 {
return "", fmt.Errorf("found more than one resources matching from %v", resources)
}
if len(resources) == 0 {
return "", fmt.Errorf("failed to find one resource matching from %v", objRef)
}
if fieldRef == "" {
fieldRef = ".metadata.name"
}
return resources[0].GetFieldValue(fieldRef)
}
func substitute(m resmap.ResMap, to *types.ReplTarget, replacement interface{}) error {
resources, err := m.Select(*to.ObjRef)
if err != nil {
return err
}
for _, r := range resources {
for _, p := range to.FieldRefs {
pathSlice := strings.Split(p, ".")
if err := updateField(r.Map(), pathSlice, replacement); err != nil {
return err
}
}
}
return nil
}
func getFirstPathSegment(path string) (field string, key string, value string, array bool) {
groups := pattern.FindStringSubmatch(path)
if len(groups) != 4 {
return path, "", "", false
}
return groups[1], groups[2], groups[3], groups[2] != ""
}
func updateField(m interface{}, pathToField []string, replacement interface{}) error {
if len(pathToField) == 0 {
return nil
}
switch typedM := m.(type) {
case map[string]interface{}:
return updateMapField(typedM, pathToField, replacement)
case []interface{}:
return updateSliceField(typedM, pathToField, replacement)
default:
return fmt.Errorf("%#v is not expected to be a primitive type", typedM)
}
}
// Extract the substring pattern (if present) from the target path spec
//nolint:unparam // TODO (dukov) refactor this or remove
func extractSubstringPattern(path string) (extractedPath string, substringPattern string, err error) {
substringPattern = ""
groups := substringPatternRegex.FindStringSubmatch(path)
if groups != nil {
path = groups[1]
substringPattern = groups[2]
}
return path, substringPattern, nil
}
// apply a substring substitution based on a pattern
func applySubstringPattern(target interface{}, replacement interface{},
substringPattern string) (regexedReplacement interface{}, err error) {
// no regex'ing needed if there is no substringPattern
if substringPattern == "" {
return replacement, nil
}
switch replacement.(type) {
case string:
default:
return nil, fmt.Errorf("pattern-based substitution can only be applied with string replacement values")
}
switch target.(type) {
case string:
default:
return nil, fmt.Errorf("pattern-based substitution can only be applied to string target fields")
}
p := regexp.MustCompile(substringPattern)
if !p.MatchString(target.(string)) {
return nil, fmt.Errorf("pattern %s not found in target value %s", substringPattern, target.(string))
}
return p.ReplaceAllString(target.(string), replacement.(string)), nil
}
//nolint:gocyclo // TODO (dukov) Refactor this or remove
func updateMapField(m map[string]interface{}, pathToField []string, replacement interface{}) error {
path, key, value, isArray := getFirstPathSegment(pathToField[0])
path, substringPattern, err := extractSubstringPattern(path)
if err != nil {
return err
}
v, found := m[path]
if !found {
m[path] = map[string]interface{}{}
v = m[path]
}
if len(pathToField) == 1 {
if !isArray {
replacement, err = applySubstringPattern(m[path], replacement, substringPattern)
if err != nil {
return err
}
m[path] = replacement
return nil
}
switch typedV := v.(type) {
case nil:
fmt.Printf("nil value at `%s` ignored in mutation attempt", strings.Join(pathToField, "."))
case []interface{}:
for i := range typedV {
item := typedV[i]
typedItem, ok := item.(map[string]interface{})
if !ok {
return fmt.Errorf("%#v is expected to be %T", item, typedItem)
}
if actualValue, ok := typedItem[key]; ok {
if value == actualValue {
typedItem[key] = value
}
}
}
default:
return fmt.Errorf("%#v is not expected to be a primitive type", typedV)
}
}
newPathToField := pathToField[1:]
switch typedV := v.(type) {
case nil:
fmt.Printf(
"nil value at `%s` ignored in mutation attempt",
strings.Join(pathToField, "."))
return nil
case map[string]interface{}:
return updateField(typedV, newPathToField, replacement)
case []interface{}:
if !isArray {
return updateField(typedV, newPathToField, replacement)
}
for i := range typedV {
item := typedV[i]
typedItem, ok := item.(map[string]interface{})
if !ok {
return fmt.Errorf("%#v is expected to be %T", item, typedItem)
}
if actualValue, ok := typedItem[key]; ok {
if value == actualValue {
return updateField(typedItem, newPathToField, replacement)
}
}
}
default:
return fmt.Errorf("%#v is not expected to be a primitive type", typedV)
}
return nil
}
func updateSliceField(m []interface{}, pathToField []string, replacement interface{}) error {
if len(pathToField) == 0 {
return nil
}
index, err := strconv.Atoi(pathToField[0])
if err != nil {
return err
}
if len(m) > index && index >= 0 {
if len(pathToField) == 1 {
m[index] = replacement
return nil
} else {
return updateField(m[index], pathToField[1:], replacement)
}
}
return fmt.Errorf("index %v is out of bound", index)
}

View File

@ -0,0 +1,404 @@
// Copyright 2019 The Kubernetes Authors.
// SPDX-License-Identifier: Apache-2.0
package v1alpha1_test
import (
"bytes"
"strings"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
replv1alpha1 "opendev.org/airship/airshipctl/pkg/document/plugin/replacement/v1alpha1"
plugtypes "opendev.org/airship/airshipctl/pkg/document/plugin/types"
)
func samplePlugin(t *testing.T) plugtypes.Plugin {
plugin, err := replv1alpha1.New(nil, []byte(`
apiVersion: airshipit.org/v1beta1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
value: nginx:newtag
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[name=nginx-latest].image`))
require.NoError(t, err)
return plugin
}
func TestMalformedConfig(t *testing.T) {
_, err := replv1alpha1.New(nil, []byte("--"))
assert.Error(t, err)
}
func TestMalformedInput(t *testing.T) {
plugin := samplePlugin(t)
err := plugin.Run(strings.NewReader("--"), &bytes.Buffer{})
assert.Error(t, err)
}
func TestDuplicatedResources(t *testing.T) {
plugin := samplePlugin(t)
err := plugin.Run(strings.NewReader(`
apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- name: myapp-container
image: busybox
---
apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- name: myapp-container
image: busybox
`), &bytes.Buffer{})
assert.Error(t, err)
}
func TestReplacementTransformer(t *testing.T) {
testCases := []struct {
cfg string
in string
expectedOut string
}{
{
cfg: `
apiVersion: airshipit.org/v1beta1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
value: nginx:newtag
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[name=nginx-latest].image
- source:
value: postgres:latest
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers.3.image
`,
in: `
group: apps
apiVersion: v1
kind: Deployment
metadata:
name: deploy1
spec:
template:
spec:
containers:
- image: nginx:1.7.9
name: nginx-tagged
- image: nginx:latest
name: nginx-latest
- image: foobar:1
name: replaced-with-digest
- image: postgres:1.8.0
name: postgresdb
initContainers:
- image: nginx
name: nginx-notag
- image: nginx@sha256:111111111111111111
name: nginx-sha256
- image: alpine:1.8.0
name: init-alpine
`,
expectedOut: `apiVersion: v1
group: apps
kind: Deployment
metadata:
name: deploy1
spec:
template:
spec:
containers:
- image: nginx:1.7.9
name: nginx-tagged
- image: nginx:newtag
name: nginx-latest
- image: foobar:1
name: replaced-with-digest
- image: postgres:latest
name: postgresdb
initContainers:
- image: nginx
name: nginx-notag
- image: nginx@sha256:111111111111111111
name: nginx-sha256
- image: alpine:1.8.0
name: init-alpine
`,
},
{
cfg: `
apiVersion: airshipit.org/v1beta1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
objref:
kind: Pod
name: pod
fieldref: spec.containers
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers`,
in: `
apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- name: myapp-container
image: busybox
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy2
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy3
`,
expectedOut: `apiVersion: v1
kind: Pod
metadata:
name: pod
spec:
containers:
- image: busybox
name: myapp-container
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy2
spec:
template:
spec:
containers:
- image: busybox
name: myapp-container
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy3
spec:
template:
spec:
containers:
- image: busybox
name: myapp-container
`,
},
{
cfg: `
apiVersion: airshipit.org/v1beta1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
objref:
kind: ConfigMap
name: cm
fieldref: data.HOSTNAME
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[image=debian].args.0
- spec.template.spec.containers[name=busybox].args.1
- source:
objref:
kind: ConfigMap
name: cm
fieldref: data.PORT
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[image=debian].args.1
- spec.template.spec.containers[name=busybox].args.2`,
in: `
apiVersion: apps/v1
kind: Deployment
metadata:
name: deploy
labels:
foo: bar
spec:
template:
metadata:
labels:
foo: bar
spec:
containers:
- name: command-demo-container
image: debian
command: ["printenv"]
args:
- HOSTNAME
- PORT
- name: busybox
image: busybox:latest
args:
- echo
- HOSTNAME
- PORT
---
apiVersion: v1
kind: ConfigMap
metadata:
name: cm
data:
HOSTNAME: example.com
PORT: 8080`,
expectedOut: `apiVersion: apps/v1
kind: Deployment
metadata:
labels:
foo: bar
name: deploy
spec:
template:
metadata:
labels:
foo: bar
spec:
containers:
- args:
- example.com
- 8080
command:
- printenv
image: debian
name: command-demo-container
- args:
- echo
- example.com
- 8080
image: busybox:latest
name: busybox
---
apiVersion: v1
data:
HOSTNAME: example.com
PORT: 8080
kind: ConfigMap
metadata:
name: cm
`,
},
{
cfg: `
apiVersion: airshipit.org/v1beta1
kind: ReplacementTransformer
metadata:
name: notImportantHere
replacements:
- source:
value: regexedtag
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers[name=nginx-latest].image%TAG%
- source:
value: postgres:latest
target:
objref:
kind: Deployment
fieldrefs:
- spec.template.spec.containers.3.image`,
in: `
group: apps
apiVersion: v1
kind: Deployment
metadata:
name: deploy1
spec:
template:
spec:
containers:
- image: nginx:1.7.9
name: nginx-tagged
- image: nginx:TAG
name: nginx-latest
- image: foobar:1
name: replaced-with-digest
- image: postgres:1.8.0
name: postgresdb
initContainers:
- image: nginx
name: nginx-notag
- image: nginx@sha256:111111111111111111
name: nginx-sha256
- image: alpine:1.8.0
name: init-alpine`,
expectedOut: `apiVersion: v1
group: apps
kind: Deployment
metadata:
name: deploy1
spec:
template:
spec:
containers:
- image: nginx:1.7.9
name: nginx-tagged
- image: nginx:regexedtag
name: nginx-latest
- image: foobar:1
name: replaced-with-digest
- image: postgres:latest
name: postgresdb
initContainers:
- image: nginx
name: nginx-notag
- image: nginx@sha256:111111111111111111
name: nginx-sha256
- image: alpine:1.8.0
name: init-alpine
`,
},
}
for _, tc := range testCases {
plugun, err := replv1alpha1.New(nil, []byte(tc.cfg))
require.NoError(t, err)
buf := &bytes.Buffer{}
err = plugun.Run(strings.NewReader(tc.in), buf)
require.NoError(t, err)
assert.Equal(t, tc.expectedOut, buf.String())
}
}

View File

@ -0,0 +1,28 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
"sigs.k8s.io/kustomize/api/types"
)
//noinspection GoUnusedGlobalVariable
var KustomizePlugin plugin
// Find matching image declarations and replace
// the name, tag and/or digest.
type plugin struct {
Replacements []types.Replacement `json:"replacements,omitempty" yaml:"replacements,omitempty"`
}

View File

@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime/schema"
"opendev.org/airship/airshipctl/pkg/document/plugin/replacement"
"opendev.org/airship/airshipctl/pkg/document/plugin/types"
"opendev.org/airship/airshipctl/pkg/environment"
)
@ -29,6 +30,10 @@ import (
// Registry contains factory functions for the available plugins
var Registry = make(map[schema.GroupVersionKind]types.Factory)
func init() {
replacement.RegisterPlugin(Registry)
}
// ConfigureAndRun executes particular plugin based on group, version, kind
// which have been specified in configuration file. Config file should be
// supplied as a first element of args slice