Build ephemeral iso using generic container

We build iso image in two steps
 1) We prepare cloud-init data using a krm function
    krm-functions/cloud-init which uses arishipctl capabilities
    to gather necessary data from the executor document bundle.
    Cloud-init data files are written into a directory mounted
    to the krm function container.
 2) We build iso image using image-builder. While doing this
    we mount the directory with cloud-init data files and set
    necessary environment variables defined in the executor
    document.

Relates-To: #440
Change-Id: Id0b34822e95f494d2e2f8fb407700b7f873e7c69

krm-functions/cloud-init/Dockerfile.sample

@@ -0,0 +1,11 @@
+FROM gcr.io/gcp-runtimes/go1-builder:1.13 as builder
+ENV CGO_ENABLED=0
+WORKDIR /go/src/
+COPY image/go.mod .
+RUN /usr/local/go/bin/go mod download
+COPY main.go .
+RUN /usr/local/go/bin/go build -v -o /usr/local/bin/config-function ./
+
+FROM alpine:latest
+COPY --from=builder /usr/local/bin/config-function /usr/local/bin/config-function
+CMD ["/usr/local/bin/config-function"]

krm-functions/cloud-init/Makefile

@@ -0,0 +1,78 @@
+.PHONY: generate license fix vet fmt test build tidy image
+
+SHELL := /bin/bash
+GOBIN := $(shell go env GOPATH)/bin
+
+# docker image options
+DOCKER_REGISTRY     ?= quay.io
+DOCKER_IMAGE_NAME   ?= cloud-init
+DOCKER_IMAGE_PREFIX ?= airshipit
+DOCKER_IMAGE_TAG    ?= latest
+DOCKER_IMAGE        ?= $(DOCKER_REGISTRY)/$(DOCKER_IMAGE_PREFIX)/$(DOCKER_IMAGE_NAME):$(DOCKER_IMAGE_TAG)
+PUBLISH             ?= false
+DOCKER_FORCE_CLEAN  ?= true
+
+# proxy options
+PROXY               ?= http://proxy.foo.com:8000
+NO_PROXY            ?= localhost,127.0.0.1,.svc.cluster.local
+USE_PROXY           ?= false
+
+.PHONY: build
+build:
+	(cd image && go build -v -o $(GOBIN)/config-function .)
+
+.PHONY: all
+all: generate license build fix vet fmt test lint tidy
+
+.PHONY: fix
+fix:
+	(cd image && go fix ./...)
+
+.PHONY: fmt
+fmt:
+	(cd image && go fmt ./...)
+
+.PHONY: generate
+generate:
+	(which $(GOBIN)/mdtogo || go get sigs.k8s.io/kustomize/cmd/mdtogo)
+	(cd image && GOBIN=$(GOBIN) go generate ./...)
+
+.PHONY: tidy
+tidy:
+	(cd image && go mod tidy)
+
+.PHONY: fix
+lint:
+	(which $(GOBIN)/golangci-lint || go get github.com/golangci/golangci-lint/cmd/golangci-lint@v1.19.1)
+	(cd image && $(GOBIN)/golangci-lint run ./...)
+
+.PHONY: test
+test:
+	(cd image && go test -cover ./...)
+
+.PHONY: vet
+vet:
+	(cd image && go vet ./...)
+
+.PHONY: image
+image:
+ifeq ($(USE_PROXY), true)
+	cd image && \
+	docker build . --network=host \
+		--build-arg http_proxy=$(PROXY) \
+		--build-arg https_proxy=$(PROXY) \
+		--build-arg HTTP_PROXY=$(PROXY) \
+		--build-arg HTTPS_PROXY=$(PROXY) \
+		--build-arg no_proxy=$(NO_PROXY) \
+		--build-arg NO_PROXY=$(NO_PROXY) \
+	  --tag $(DOCKER_IMAGE) \
+	  --force-rm=$(DOCKER_FORCE_CLEAN)
+else
+	cd image && \
+	docker build . --network=host \
+	    --tag $(DOCKER_IMAGE) \
+	    --force-rm=$(DOCKER_FORCE_CLEAN)
+endif
+ifeq ($(PUBLISH), true)
+	@docker push $(DOCKER_IMAGE)
+endif

krm-functions/cloud-init/README.md

@@ -0,0 +1,6 @@
+# Cloud-init
+
+This function generates yaml files (user-data and network-data) needed for building the bootstrap ISO image.
+It assumes a `ResourceList` is passed to its input where `items` field is a phase executor document bundle
+that contains necessary data. To get the data from the bundle the `functionConfig` field must contain
+`IsoConfiguration` document that defines document selector parameters.

krm-functions/cloud-init/image/go.mod

@@ -0,0 +1,9 @@
+module opendev.org/airship/airshipctl/functions/cloud-init/image
+
+go 1.14
+
+require (
+	opendev.org/airship/airshipctl v0.0.0-20210217205206-b8a4b6ad734c
+	sigs.k8s.io/kustomize/kyaml v0.10.0
+	sigs.k8s.io/kustomize/api v0.7.2
+)

krm-functions/cloud-init/local-resource/example-input-resource-list.yaml

@@ -0,0 +1,32 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+apiVersion: config.kubernetes.io/v1alpha1
+kind: ResourceList
+items: []
+functionConfig:
+  apiVersion: airshipit.org/v1alpha1
+  kind: IsoConfiguration
+  metadata:
+    annotations:
+      config.kubernetes.io/function: |-
+        container:
+          image: krm-function-cloud-init:latest
+    name: cloud-init
+  builder:
+    userDataSelector:
+      kind: Secret
+      labelSelector: airshipit.org/ephemeral-user-data
+    userDataKey: userData
+    networkConfigSelector:
+      kind: BareMetalHost
+      labelSelector: airshipit.org/ephemeral-node
+    networkConfigKey: networkData

krm-functions/cloud-init/main.go

@@ -0,0 +1,128 @@
+/*
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     https://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+package main
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	"sigs.k8s.io/kustomize/api/provider"
+	"sigs.k8s.io/kustomize/api/resmap"
+	"sigs.k8s.io/kustomize/kyaml/fn/framework"
+	"sigs.k8s.io/kustomize/kyaml/fn/runtime/runtimeutil"
+	"sigs.k8s.io/kustomize/kyaml/yaml"
+
+	"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
+	"opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit"
+	"opendev.org/airship/airshipctl/pkg/document"
+	"opendev.org/airship/airshipctl/pkg/util"
+)
+
+const (
+	builderConfigFileName = "builder-conf.yaml"
+	userDataFileName      = "user-data"
+	networkConfigFileName = "network-data"
+)
+
+func bundleFromRNodes(rnodes []*yaml.RNode) (document.Bundle, error) {
+	p := provider.NewDefaultDepProvider()
+	resmapFactory := resmap.NewFactory(p.GetResourceFactory(), p.GetConflictDetectorFactory())
+	resmap, err := resmapFactory.NewResMapFromRNodeSlice(rnodes)
+	if err != nil {
+		return &document.BundleFactory{}, err
+	}
+	return &document.BundleFactory{
+		ResMap: resmap,
+	}, nil
+}
+
+func docFromRNode(rnode *yaml.RNode) (document.Document, error) {
+	rnodes := []*yaml.RNode{rnode}
+	bundle, err := bundleFromRNodes(rnodes)
+	if err != nil {
+		return nil, err
+	}
+	collection, err := bundle.GetAllDocuments()
+	if err != nil {
+		return nil, err
+	}
+	if len(collection) == 0 {
+		return nil, errors.New("Error while converting RNode to Document: empty document bundle")
+	}
+	return collection[0], nil
+}
+
+func main() {
+	resourceList := &framework.ResourceList{}
+	cmd := framework.Command(resourceList, func() error {
+		functionConfig, ok := resourceList.FunctionConfig.(*yaml.RNode)
+		if !ok {
+			return errors.New("Error while type assert of FunctionConfig")
+		}
+
+		functionConfigDocument, err := docFromRNode(functionConfig)
+		if err != nil {
+			return err
+		}
+		functionConfigYaml, err := functionConfigDocument.AsYAML()
+		if err != nil {
+			return err
+		}
+
+		isoConfiguration := &v1alpha1.IsoConfiguration{}
+		err = functionConfigDocument.ToAPIObject(isoConfiguration, v1alpha1.Scheme)
+		if err != nil {
+			return err
+		}
+
+		docBundle, err := bundleFromRNodes(resourceList.Items)
+		if err != nil {
+			return err
+		}
+
+		userData, netConf, err := cloudinit.GetCloudData(
+			docBundle,
+			isoConfiguration.Isogen.UserDataSelector,
+			isoConfiguration.Isogen.UserDataKey,
+			isoConfiguration.Isogen.NetworkConfigSelector,
+			isoConfiguration.Isogen.NetworkConfigKey,
+		)
+		if err != nil {
+			return err
+		}
+
+		functionSpec := runtimeutil.GetFunctionSpec(functionConfig)
+		configPath := functionSpec.Container.StorageMounts[0].DstPath
+
+		fls := make(map[string][]byte)
+		fls[filepath.Join(configPath, userDataFileName)] = userData
+		fls[filepath.Join(configPath, networkConfigFileName)] = netConf
+		fls[filepath.Join(configPath, builderConfigFileName)] = functionConfigYaml
+
+		if err = util.WriteFiles(fls, 0600); err != nil {
+			return err
+		}
+
+		resourceList.Items = []*yaml.RNode{}
+		return nil
+	})
+
+	if err := cmd.Execute(); err != nil {
+		fmt.Fprint(os.Stderr, err)
+		os.Exit(1)
+	}
+}

manifests/phases/executors.yaml

@@ -184,3 +184,75 @@ spec:
     remoteDirect:
       isoURL: REPLACE_ME
 ---
+apiVersion: airshipit.org/v1alpha1
+kind: GenericContainer
+metadata:
+  name: iso-cloud-init-data
+  labels:
+    airshipit.org/deploy-k8s: "false"
+spec:
+  type: krm
+  image: quay.io/airshipit/cloud-init:latest
+  mounts:
+  - type: bind
+    src: /srv/iso
+    dst: /config
+    rw: true
+config: |
+  apiVersion: airshipit.org/v1alpha1
+  kind: IsoConfiguration
+  metadata:
+    name: isogen
+  builder:
+    userDataSelector:
+      kind: Secret
+      labelSelector: airshipit.org/ephemeral-user-data
+    userDataKey: userData
+    networkConfigSelector:
+      kind: BareMetalHost
+      labelSelector: airshipit.org/ephemeral-node
+    networkConfigKey: networkData
+    outputFileName: ephemeral.iso
+  container:
+    volume: /fake/path/iso:/config # for compatibility with image-builder
+
+---
+apiVersion: airshipit.org/v1alpha1
+kind: GenericContainer
+metadata:
+  name: iso-build-image
+  labels:
+    airshipit.org/deploy-k8s: "false"
+spec:
+  type: airship
+  airship:
+    privileged: true
+    containerRuntime: docker
+    cmd:
+    - /bin/bash
+    - -c
+    - /usr/bin/local/entrypoint.sh 1>&2
+  image: quay.io/airshipit/image-builder:latest-ubuntu_focal
+  mounts:
+  - type: bind
+    src: /srv/iso
+    dst: /config
+    rw: true
+  envVars:
+  - IMAGE_TYPE=iso
+  - BUILDER_CONFIG=/config/builder-conf.yaml
+  - USER_DATA_FILE=user-data
+  - NET_CONFIG_FILE=network-data
+  - OUTPUT_FILE_NAME=ephemerial.iso
+  - OUTPUT_METADATA_FILE_NAME=output-metadata.yaml
+  - http_proxy
+  - https_proxy
+  - HTTP_PROXY
+  - HTTPS_PROXY
+  - no_proxy
+  - NO_PROXY
+config: |
+  apiVersion: airshipit.org/v1alpha1
+  kind: DoesNotMatter
+  metadata:
+    name: isogen

manifests/phases/phases.yaml

@@ -246,3 +246,25 @@ config:
     apiVersion: airshipit.org/v1alpha1
     kind: BaremetalManager
     name: RemoteDirectEphemeral
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
+metadata:
+  name: iso-cloud-init-data
+config:
+  executorRef:
+    apiVersion: airshipit.org/v1alpha1
+    kind: GenericContainer
+    name: iso-cloud-init-data
+  documentEntryPoint: ephemeral/bootstrap
+---
+apiVersion: airshipit.org/v1alpha1
+kind: Phase
+metadata:
+  name: iso-build-image
+config:
+  executorRef:
+    apiVersion: airshipit.org/v1alpha1
+    kind: GenericContainer
+    name: iso-build-image
+  documentEntryPoint: empty

manifests/phases/plan.yaml

@@ -14,3 +14,14 @@ phases:
   - name: workers-target
   - name: workers-classification
   - name: workload-target
+
+---
+apiVersion: airshipit.org/v1alpha1
+kind: PhasePlan
+metadata:
+  name: iso
+description: "Runs phases to build iso image"
+phases:
+  - name: iso-cloud-init-data
+  - name: iso-build-image
+

pkg/api/v1alpha1/genericcontainer_types.go

@@ -96,7 +96,7 @@ type AirshipContainerSpec struct {
 	Cmd []string `json:"cmd,omitempty"`
 
 	// Privileged identifies if the container is to be run in a Privileged mode
-	Privileged bool `json:"pivileged,omitempty"`
+	Privileged bool `json:"privileged,omitempty"`
 }
 
 // KRMContainerSpec defines a spec for running a function as a container