From d5c03772073289ddbf9c02543dc1a19d922e8d8f Mon Sep 17 00:00:00 2001 From: Vladislav Kuzmin Date: Tue, 23 Mar 2021 16:39:40 +0400 Subject: [PATCH] Add toolbox krm function Add krm function to execute bash scripts inside container. Closes: #494 Change-Id: I3074a27a022f65e87f190ab5a39c252f225ca1fa --- Makefile | 2 + krm-functions/toolbox/Dockerfile | 26 +++ krm-functions/toolbox/image/go.mod | 8 + krm-functions/toolbox/main.go | 151 ++++++++++++++++ krm-functions/toolbox/main_test.go | 168 ++++++++++++++++++ manifests/phases/executors.yaml | 15 ++ manifests/phases/phases.yaml | 12 ++ .../ephemeral/stub/kustomization.yaml | 0 .../phases/helpers/kubectl_get_node.sh | 32 ++++ .../phases/helpers/kustomization.yaml | 6 + .../site/test-site/phases/kustomization.yaml | 1 + tools/deployment/25_deploy_ephemeral_node.sh | 19 +- 12 files changed, 422 insertions(+), 18 deletions(-) create mode 100644 krm-functions/toolbox/Dockerfile create mode 100644 krm-functions/toolbox/image/go.mod create mode 100644 krm-functions/toolbox/main.go create mode 100644 krm-functions/toolbox/main_test.go create mode 100644 manifests/site/test-site/ephemeral/stub/kustomization.yaml create mode 100644 manifests/site/test-site/phases/helpers/kubectl_get_node.sh create mode 100644 manifests/site/test-site/phases/helpers/kustomization.yaml diff --git a/Makefile b/Makefile index ec3a86bf0..5f47c8274 100644 --- a/Makefile +++ b/Makefile @@ -155,6 +155,8 @@ ifeq ($(PUBLISH), true) @docker push $(DOCKER_IMAGE) endif +# Use specific Dockerfile instead of general one to make image for toolbox +docker-image-toolbox: DOCKER_CMD_FLAGS+=-f krm-functions/toolbox/Dockerfile .PHONY: $(PLUGINS_IMAGE_TGT) $(PLUGINS_IMAGE_TGT): $(eval plugin_name=$(subst docker-image-,,$@)) diff --git a/krm-functions/toolbox/Dockerfile b/krm-functions/toolbox/Dockerfile new file mode 100644 index 000000000..7f4a9302c --- /dev/null +++ b/krm-functions/toolbox/Dockerfile @@ -0,0 +1,26 @@ +ARG RELEASE_IMAGE=scratch +FROM ${RELEASE_IMAGE} as kctl +RUN apk add curl +RUN curl -L "https://storage.googleapis.com/kubernetes-release/release/$(curl -s https://storage.googleapis.com/kubernetes-release/release/stable.txt)/bin/linux/amd64/kubectl" \ + -o /kubectl +RUN chmod +x /kubectl + +FROM gcr.io/gcp-runtimes/go1-builder:1.13 as builder +ENV CGO_ENABLED=0 +WORKDIR /go/src/ +COPY krm-functions/toolbox/image/go.mod . +RUN /usr/local/go/bin/go mod download +COPY krm-functions/toolbox/main.go . +RUN /usr/local/go/bin/go build -v -o /usr/local/bin/config-function ./ + +FROM ${RELEASE_IMAGE} as calicoctl +RUN apk add curl +RUN curl -L "https://github.com/projectcalico/calicoctl/releases/download/v3.18.1/calicoctl" \ + -o /calicoctl +RUN chmod +x /calicoctl + +FROM ${RELEASE_IMAGE} as release +COPY --from=kctl /kubectl /usr/local/bin/kubectl +COPY --from=calicoctl /calicoctl /usr/local/bin/calicoctl +COPY --from=builder /usr/local/bin/config-function /usr/local/bin/config-function +CMD ["/usr/local/bin/config-function"] diff --git a/krm-functions/toolbox/image/go.mod b/krm-functions/toolbox/image/go.mod new file mode 100644 index 000000000..cf2108748 --- /dev/null +++ b/krm-functions/toolbox/image/go.mod @@ -0,0 +1,8 @@ +module opendev.org/airship/airshipctl/krm-fnunctions/toolbox/image + +go 1.14 + +require ( + k8s.io/api v0.17.9 + sigs.k8s.io/kustomize/kyaml v0.10.6 +) diff --git a/krm-functions/toolbox/main.go b/krm-functions/toolbox/main.go new file mode 100644 index 000000000..281ec186c --- /dev/null +++ b/krm-functions/toolbox/main.go @@ -0,0 +1,151 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + + v1 "k8s.io/api/core/v1" + kerror "k8s.io/apimachinery/pkg/util/errors" + "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/kio" + kyaml "sigs.k8s.io/kustomize/kyaml/yaml" + + "opendev.org/airship/airshipctl/pkg/log" +) + +const ( + // EnvRenderedBundlePath will be passed to the script, it will contain path to the rendered bundle + EnvRenderedBundlePath = "RENDERED_BUNDLE_PATH" + scriptPath = "script.sh" + scriptKey = "script" + bundleFile = "bundle.yaml" + workdir = "/tmp" +) + +func main() { + cfg := &v1.ConfigMap{} + resourceList := &framework.ResourceList{FunctionConfig: &cfg} + runner := ScriptRunner{ + ScriptFile: scriptPath, + WorkDir: workdir, + RenderedBundleFile: bundleFile, + DataKey: scriptKey, + ResourceList: resourceList, + ConfigMap: cfg, + ErrStream: os.Stderr, + OutStream: os.Stdout, + } + cmd := framework.Command(resourceList, runner.Run) + if err := cmd.Execute(); err != nil { + log.Print(err) + os.Exit(1) + } +} + +// ScriptRunner writes to file system and executes the script +type ScriptRunner struct { + ScriptFile, WorkDir, DataKey, RenderedBundleFile string + + ErrStream io.Writer + OutStream io.Writer + + ConfigMap *v1.ConfigMap + ResourceList *framework.ResourceList +} + +// Run writes the script and bundle to the file system and executes it +func (c *ScriptRunner) Run() error { + bundlePath, scriptPath := c.getBundleAndScriptPath() + + script, exist := c.ConfigMap.Data[c.DataKey] + if !exist { + return fmt.Errorf("ConfigMap '%s/%s' doesnt' have specified script key '%s'", + c.ConfigMap.Namespace, c.ConfigMap.Name, c.DataKey) + } + + err := ioutil.WriteFile(scriptPath, []byte(script), 0555) + if err != nil { + return err + } + err = c.writeBundle(bundlePath, c.ResourceList.Items) + if err != nil { + return err + } + + c.ResourceList.Items = nil + + os.Setenv(EnvRenderedBundlePath, bundlePath) + + clicmd := exec.Command(scriptPath) + clicmd.Stdout = c.OutStream + clicmd.Stderr = c.ErrStream + + err = clicmd.Start() + if err != nil { + return err + } + + return clicmd.Wait() +} + +// Cleanup removes script and bundle files from filesystem +func (c *ScriptRunner) Cleanup() error { + bundlePath, scriptPath := c.getBundleAndScriptPath() + + scriptErr := os.Remove(scriptPath) + if os.IsNotExist(scriptErr) { + // If file doesn't exist no error happened + scriptErr = nil + } + + bundleErr := os.Remove(bundlePath) + if os.IsNotExist(bundleErr) { + // If file doesn't exist no error happened + bundleErr = nil + } + + return kerror.NewAggregate([]error{scriptErr, bundleErr}) +} + +func (c *ScriptRunner) getBundleAndScriptPath() (string, string) { + return filepath.Join(c.WorkDir, c.RenderedBundleFile), filepath.Join(c.WorkDir, c.ScriptFile) +} + +func (c *ScriptRunner) writeBundle(path string, items []*kyaml.RNode) error { + f, err := os.Create(path) + if err != nil { + return err + } + defer f.Close() + + pipeline := kio.Pipeline{ + Outputs: []kio.Writer{ + kio.ByteWriter{ + Writer: f, + }, + }, + Inputs: []kio.Reader{ + kio.ResourceNodeSlice(items), + }, + } + + return pipeline.Execute() +} diff --git a/krm-functions/toolbox/main_test.go b/krm-functions/toolbox/main_test.go new file mode 100644 index 000000000..660f66cc8 --- /dev/null +++ b/krm-functions/toolbox/main_test.go @@ -0,0 +1,168 @@ +/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package main + +import ( + "bytes" + "io/ioutil" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "sigs.k8s.io/kustomize/kyaml/fn/framework" + "sigs.k8s.io/kustomize/kyaml/yaml" +) + +const ( + dir = "image" + targetFile = "my-script.sh" + dataKey = "script" + wrongDataKey = "foobar" + bundlePath = "bundle.yaml" + script = `#!/bin/bash +echo -n 'stderr' 1>&2 +echo -n 'stdout'` + wrongScript = `#!/usr/bin/p +print("Hello world!")` + inputString = `kind: testkind +metadata: + name: test-name + namespace: test-namespace +` +) + +func TestCmdRun(t *testing.T) { + tests := []struct { + name string + workdir string + errContains string + configMap *v1.ConfigMap + }{ + { + name: "Successful run", + workdir: dir, + configMap: &v1.ConfigMap{ + Data: map[string]string{ + dataKey: script, + }, + }, + }, + { + name: "Wrong key in ConfigMap", + workdir: dir, + errContains: "ConfigMap '/' doesnt' have specified script key 'script'", + configMap: &v1.ConfigMap{ + Data: map[string]string{ + wrongDataKey: "", + }, + }, + }, + { + name: "WorkDir that doesnt' exist", + workdir: "foobar", + errContains: "open foobar/my-script.sh: no such file or directory", + configMap: &v1.ConfigMap{ + Data: map[string]string{ + dataKey: script, + }, + }, + }, + { + name: "Wrong interpreter", + workdir: dir, + errContains: "fork/exec image/my-script.sh: no such file or directory", + configMap: &v1.ConfigMap{ + Data: map[string]string{ + dataKey: wrongScript, + }, + }, + }, + } + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + input, err := yaml.Parse(inputString) + require.NoError(t, err) + + stderr := bytes.NewBuffer([]byte{}) + stdout := bytes.NewBuffer([]byte{}) + + cmd := &ScriptRunner{ + ScriptFile: targetFile, + WorkDir: tt.workdir, + DataKey: dataKey, + ErrStream: stderr, + OutStream: stdout, + ResourceList: &framework.ResourceList{Items: []*yaml.RNode{input}}, + ConfigMap: tt.configMap, + RenderedBundleFile: bundlePath, + } + err = cmd.Run() + defer func() { + require.NoError(t, cmd.Cleanup()) + }() + + if tt.errContains != "" { + require.Error(t, err) + assert.Contains(t, err.Error(), tt.errContains) + } else { + assert.NoError(t, err) + assert.Equal(t, "stderr", stderr.String()) + assert.Equal(t, "stdout", stdout.String()) + bundleFullPath := filepath.Join(tt.workdir, bundlePath) + assert.FileExists(t, bundleFullPath) + result, err := ioutil.ReadFile(filepath.Join(tt.workdir, bundlePath)) + require.NoError(t, err) + assert.Contains(t, string(result), "testkind") + assert.Contains(t, string(result), "test-name") + assert.Contains(t, string(result), "test-namespace") + } + }) + } +} + +func TestCmdRunCleanup(t *testing.T) { + cMap := &v1.ConfigMap{ + Data: map[string]string{ + dataKey: script, + }, + } + + input, err := yaml.Parse(inputString) + require.NoError(t, err) + + stderr := bytes.NewBuffer([]byte{}) + stdout := bytes.NewBuffer([]byte{}) + + cmd := &ScriptRunner{ + ScriptFile: targetFile, + WorkDir: dir, + DataKey: dataKey, + ErrStream: stderr, + OutStream: stdout, + ResourceList: &framework.ResourceList{Items: []*yaml.RNode{input}}, + ConfigMap: cMap, + RenderedBundleFile: bundlePath, + } + + require.NoError(t, cmd.Cleanup()) + err = cmd.Run() + defer func() { + require.NoError(t, cmd.Cleanup()) + }() + assert.NoError(t, err) +} diff --git a/manifests/phases/executors.yaml b/manifests/phases/executors.yaml index aeb72b3ac..9c9927ae4 100644 --- a/manifests/phases/executors.yaml +++ b/manifests/phases/executors.yaml @@ -273,3 +273,18 @@ config: | kind: DoesNotMatter metadata: name: isogen +--- +apiVersion: airshipit.org/v1alpha1 +kind: GenericContainer +metadata: + name: kubectl-get-node + labels: + airshipit.org/deploy-k8s: "false" +spec: + type: krm + image: quay.io/airshipit/toolbox:latest + hostNetwork: true +configRef: + kind: ConfigMap + name: kubectl-get-node + apiVersion: v1 diff --git a/manifests/phases/phases.yaml b/manifests/phases/phases.yaml index a58d55d2a..51e813738 100644 --- a/manifests/phases/phases.yaml +++ b/manifests/phases/phases.yaml @@ -278,3 +278,15 @@ config: apiVersion: airshipit.org/v1alpha1 kind: GenericContainer name: iso-build-image +--- +apiVersion: airshipit.org/v1alpha1 +kind: Phase +metadata: + name: kubectl-get-node-ephemeral + clusterName: ephemeral-cluster +config: + executorRef: + apiVersion: airshipit.org/v1alpha1 + kind: GenericContainer + name: kubectl-get-node + documentEntryPoint: ephemeral/initinfra-networking diff --git a/manifests/site/test-site/ephemeral/stub/kustomization.yaml b/manifests/site/test-site/ephemeral/stub/kustomization.yaml new file mode 100644 index 000000000..e69de29bb diff --git a/manifests/site/test-site/phases/helpers/kubectl_get_node.sh b/manifests/site/test-site/phases/helpers/kubectl_get_node.sh new file mode 100644 index 000000000..b3805ca41 --- /dev/null +++ b/manifests/site/test-site/phases/helpers/kubectl_get_node.sh @@ -0,0 +1,32 @@ +#!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +N=0 +MAX_RETRY=30 +DELAY=60 +until [ "$N" -ge ${MAX_RETRY} ] +do + if timeout 20 kubectl --kubeconfig $KUBECONFIG --context $KCTL_CONTEXT get node 1>&2; then + break + fi + + N=$((N+1)) + echo "$N: Retrying to reach the apiserver" 1>&2 + sleep ${DELAY} +done + +if [ "$N" -ge ${MAX_RETRY} ]; then + echo "Could not reach the apiserver" 1>&2 + exit 1 +fi diff --git a/manifests/site/test-site/phases/helpers/kustomization.yaml b/manifests/site/test-site/phases/helpers/kustomization.yaml new file mode 100644 index 000000000..c5ce0a797 --- /dev/null +++ b/manifests/site/test-site/phases/helpers/kustomization.yaml @@ -0,0 +1,6 @@ +configMapGenerator: +- name: kubectl-get-node + options: + disableNameSuffixHash: true + files: + - script=kubectl_get_node.sh diff --git a/manifests/site/test-site/phases/kustomization.yaml b/manifests/site/test-site/phases/kustomization.yaml index 55159500e..aa67ae5ad 100644 --- a/manifests/site/test-site/phases/kustomization.yaml +++ b/manifests/site/test-site/phases/kustomization.yaml @@ -2,5 +2,6 @@ resources: - ../kubeconfig - ../../../phases - catalogue.yaml + - helpers transformers: - ../../../function/bootstrap/replacements diff --git a/tools/deployment/25_deploy_ephemeral_node.sh b/tools/deployment/25_deploy_ephemeral_node.sh index da1ba7b87..7969ff91e 100755 --- a/tools/deployment/25_deploy_ephemeral_node.sh +++ b/tools/deployment/25_deploy_ephemeral_node.sh @@ -23,24 +23,7 @@ echo "Deploy ephemeral node using redfish with iso" airshipctl phase run remotedirect-ephemeral --debug echo "Wait for apiserver to become available" -N=0 -MAX_RETRY=30 -DELAY=60 -until [ "$N" -ge ${MAX_RETRY} ] -do - if timeout 20 kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get node; then - break - fi - - N=$((N+1)) - echo "$N: Retrying to reach the apiserver" - sleep ${DELAY} -done - -if [ "$N" -ge ${MAX_RETRY} ]; then - echo "Could not reach the apiserver" - exit 1 -fi +airshipctl phase run kubectl-get-node-ephemeral echo "List all pods" kubectl --kubeconfig $KUBECONFIG --context $KUBECONFIG_EPHEMERAL_CONTEXT get pods --all-namespaces