From 2daacf5f2abced87811ec631af963a42781bb928 Mon Sep 17 00:00:00 2001 From: "Anderson, Craig (ca846m)" Date: Wed, 28 Oct 2020 05:12:44 -0700 Subject: [PATCH] image-builder integration for ISO builds This introduces airshipctl integration with image-builder [0], which replaces the existing isogen tool for ephemeral ISO generation. The airshipctl isogen executor has been updated for building ephemeral ISOs using the image-builder container. The ability for user-declared filenames for cloud-init user data and network data was removed, since the user's only interest is in supplying the relevant overrides, not in transparent naming coordination with the image-builder container. A new object is added to the document package to identify the document kind, label, and key to retrieve data from since this is pattern we will reuse elsewhere. Progress flag removed as requsted. Progress is reported directly by the image-builder container. Isogen debug flag removed in favor of using log.DebugEnabled() [0] https://review.opendev.org/#/c/730777/ Depends-On: https://review.opendev.org/c/airship/images/+/730777/ Change-Id: I545004feaf2116f8ffb29faf6f7f7f5fcfe24fff --- .../baremetal-ejectmedia-with-help.golden | 2 +- .../baremetal-poweroff-with-help.golden | 2 +- .../baremetal-poweron-with-help.golden | 2 +- .../baremetal-powerstatus-with-help.golden | 2 +- .../baremetal-reboot-with-help.golden | 2 +- cmd/image/build.go | 11 +- docs/source/architecture.md | 20 +- .../cli/airshipctl_baremetal_ejectmedia.md | 2 +- .../cli/airshipctl_baremetal_poweroff.md | 2 +- .../cli/airshipctl_baremetal_poweron.md | 2 +- .../cli/airshipctl_baremetal_powerstatus.md | 2 +- .../source/cli/airshipctl_baremetal_reboot.md | 2 +- docs/source/cli/airshipctl_image_build.md | 1 - go.mod | 1 - go.sum | 11 - .../ephemeral/image_configuration.yaml | 18 +- .../remote_direct_configuration.yaml | 2 +- manifests/function/ephemeral/secret.yaml | 29 +- manifests/phases/executors.yaml | 10 +- manifests/phases/phases.yaml | 4 +- pkg/api/v1alpha1/groupversion_info.go | 2 +- pkg/api/v1alpha1/imageconfiguration_types.go | 59 --- pkg/api/v1alpha1/isoconfiguration.go | 64 ++++ pkg/api/v1alpha1/zz_generated.deepcopy.go | 132 +++---- pkg/bootstrap/cloudinit/cloud-init.go | 112 +++--- pkg/bootstrap/cloudinit/cloud-init_test.go | 117 +++++- pkg/bootstrap/cloudinit/errors.go | 41 -- pkg/bootstrap/isogen/command.go | 237 +++--------- pkg/bootstrap/isogen/command_test.go | 88 ++--- pkg/bootstrap/isogen/errors.go | 8 - pkg/bootstrap/isogen/testdata/config/config | 0 .../isogen/testdata/debian-container-logs | 358 ------------------ pkg/config/constants.go | 2 +- pkg/document/document.go | 29 ++ pkg/document/errors.go | 21 + pkg/document/selectors.go | 34 ++ pkg/document/selectors_test.go | 10 + pkg/phase/executors/errors.go | 6 +- pkg/phase/executors/isogen.go | 26 +- pkg/phase/executors/isogen_test.go | 30 +- .../test-site/ephemeral/bootstrap/phase.yaml | 2 +- .../test-site/ephemeral/bootstrap/phase.yaml | 2 +- .../test-site/ephemeral/bootstrap/phase.yaml | 2 +- playbooks/airshipctl-gate-runner.yaml | 2 +- playbooks/vars/test-config.yaml | 2 +- tools/deployment/22_test_configs.sh | 8 +- ...ld_ephemeral_iso.sh => 24_build_images.sh} | 29 +- tools/deployment/30_deploy_controlplane.sh | 4 +- tools/deployment/azure/31_create_configs.sh | 6 +- tools/deployment/clean.sh | 2 +- tools/gate/00_setup.sh | 2 +- tools/gate/config_template.yaml | 2 +- zuul.d/jobs.yaml | 4 +- 53 files changed, 570 insertions(+), 1000 deletions(-) delete mode 100644 pkg/api/v1alpha1/imageconfiguration_types.go create mode 100644 pkg/api/v1alpha1/isoconfiguration.go delete mode 100644 pkg/bootstrap/cloudinit/errors.go delete mode 100644 pkg/bootstrap/isogen/testdata/config/config delete mode 100755 pkg/bootstrap/isogen/testdata/debian-container-logs rename tools/deployment/{24_build_ephemeral_iso.sh => 24_build_images.sh} (63%) diff --git a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-ejectmedia-with-help.golden b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-ejectmedia-with-help.golden index 39cbcec7b..cb1c657a2 100644 --- a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-ejectmedia-with-help.golden +++ b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-ejectmedia-with-help.golden @@ -7,4 +7,4 @@ Flags: -h, --help help for ejectmedia -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") diff --git a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweroff-with-help.golden b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweroff-with-help.golden index af65bb181..370705f3a 100644 --- a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweroff-with-help.golden +++ b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweroff-with-help.golden @@ -7,4 +7,4 @@ Flags: -h, --help help for poweroff -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") diff --git a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweron-with-help.golden b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweron-with-help.golden index f58abcf1c..bee25b489 100644 --- a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweron-with-help.golden +++ b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-poweron-with-help.golden @@ -7,4 +7,4 @@ Flags: -h, --help help for poweron -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") diff --git a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-powerstatus-with-help.golden b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-powerstatus-with-help.golden index f5242d5a8..009717273 100644 --- a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-powerstatus-with-help.golden +++ b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-powerstatus-with-help.golden @@ -7,4 +7,4 @@ Flags: -h, --help help for powerstatus -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") diff --git a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-reboot-with-help.golden b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-reboot-with-help.golden index 579533816..e3fc5a8da 100644 --- a/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-reboot-with-help.golden +++ b/cmd/baremetal/testdata/TestBaremetalGoldenOutput/baremetal-reboot-with-help.golden @@ -7,4 +7,4 @@ Flags: -h, --help help for reboot -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") diff --git a/cmd/image/build.go b/cmd/image/build.go index dff470d18..a59a6ceff 100644 --- a/cmd/image/build.go +++ b/cmd/image/build.go @@ -21,28 +21,19 @@ import ( "opendev.org/airship/airshipctl/pkg/phase" ) -// NewImageBuildCommand creates a new command with the capability to build an ISO image. +// NewImageBuildCommand creates a new command with the capability to build ISO image. func NewImageBuildCommand(cfgFactory config.Factory) *cobra.Command { - var progress bool cmd := &cobra.Command{ Use: "build", Short: "Build ISO image", RunE: func(cmd *cobra.Command, args []string) error { p := &phase.RunCommand{ Factory: cfgFactory, - Options: phase.RunFlags{Progress: progress}, } p.Options.PhaseID.Name = config.BootstrapPhase return p.RunE() }, } - flags := cmd.Flags() - flags.BoolVar( - &progress, - "progress", - false, - "show progress") - return cmd } diff --git a/docs/source/architecture.md b/docs/source/architecture.md index 53cff1385..02b16b0e5 100644 --- a/docs/source/architecture.md +++ b/docs/source/architecture.md @@ -12,28 +12,28 @@ In a nutshell, users of `airshipctl` should be able to do the following: 1. Create an `airshipctl` Airship Configuration for their site - sort of like a kubeconfig file. Airshipctl can create a pre-configured config file by running `airshipctl config init`. -2. Create a set of declarative documents representing the infrastructure +1. Create a set of declarative documents representing the infrastructure (baremetal, cloud) and software. -3. Run `airshipctl document pull` to clone the document repositories in your +1. Run `airshipctl document pull` to clone the document repositories in your Airship Configuration. -4. When deploying against baremetal infrastructure, run - `airshipctl baremetal isogen` to generate a self-contained ISO that can be - used to boot the first host in the cluster into an ephemeral Kubernetes node. -5. When deploying against baremetal infrastructure, run +1. Run `airshipctl image build` to generate a self-contained ISO + that can be used to boot the first host in the cluster into an ephemeral + Kubernetes node. +1. When deploying against baremetal infrastructure, run `airshipctl baremetal remotedirect` to remotely provision the first machine in the cluster using the generated ISO, providing an ephemeral Kubernetes instance that `airshipctl` can communicate with for subsequent steps. This ephemeral host provides a foothold in the target environment so we can follow the standard cluster-api bootstrap flow. -6. Run `airshipctl phase run initinfra-ephemeral` to bootstrap the new ephemeral cluster +1. Run `airshipctl phase run initinfra-ephemeral` to bootstrap the new ephemeral cluster with enough of the chosen cluster-api provider components to provision the target cluster. -7. Run `airshipctl clusterctl` to use the ephemeral Kubernetes host to provision +1. Run `airshipctl clusterctl` to use the ephemeral Kubernetes host to provision at least one node of the target cluster using the cluster-api bootstrap flow. -8. Run `airshipctl cluster initinfra --clustertype=target` to bootstrap the new +1. Run `airshipctl cluster initinfra --clustertype=target` to bootstrap the new target cluster with any remaining infrastructure necessary to begin running more complex workflows such as Argo. -9. Run `airshipctl workflow submit sitemanage` to run the out of the box sitemanage +1. Run `airshipctl workflow submit sitemanage` to run the out of the box sitemanage workflow, which will leverage Argo to handle bootstrapping the remaining infrastructure as well as deploying and/or updating software. diff --git a/docs/source/cli/airshipctl_baremetal_ejectmedia.md b/docs/source/cli/airshipctl_baremetal_ejectmedia.md index 40698d260..ddff9e08c 100644 --- a/docs/source/cli/airshipctl_baremetal_ejectmedia.md +++ b/docs/source/cli/airshipctl_baremetal_ejectmedia.md @@ -16,7 +16,7 @@ airshipctl baremetal ejectmedia [flags] -h, --help help for ejectmedia -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") ``` ### Options inherited from parent commands diff --git a/docs/source/cli/airshipctl_baremetal_poweroff.md b/docs/source/cli/airshipctl_baremetal_poweroff.md index edee71ec7..172a86402 100644 --- a/docs/source/cli/airshipctl_baremetal_poweroff.md +++ b/docs/source/cli/airshipctl_baremetal_poweroff.md @@ -16,7 +16,7 @@ airshipctl baremetal poweroff [flags] -h, --help help for poweroff -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") ``` ### Options inherited from parent commands diff --git a/docs/source/cli/airshipctl_baremetal_poweron.md b/docs/source/cli/airshipctl_baremetal_poweron.md index bc0cc6e6a..6d7b62163 100644 --- a/docs/source/cli/airshipctl_baremetal_poweron.md +++ b/docs/source/cli/airshipctl_baremetal_poweron.md @@ -16,7 +16,7 @@ airshipctl baremetal poweron [flags] -h, --help help for poweron -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") ``` ### Options inherited from parent commands diff --git a/docs/source/cli/airshipctl_baremetal_powerstatus.md b/docs/source/cli/airshipctl_baremetal_powerstatus.md index 888fb6017..6ca56283c 100644 --- a/docs/source/cli/airshipctl_baremetal_powerstatus.md +++ b/docs/source/cli/airshipctl_baremetal_powerstatus.md @@ -16,7 +16,7 @@ airshipctl baremetal powerstatus [flags] -h, --help help for powerstatus -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") ``` ### Options inherited from parent commands diff --git a/docs/source/cli/airshipctl_baremetal_reboot.md b/docs/source/cli/airshipctl_baremetal_reboot.md index 468bd4ceb..f8bc55c7c 100644 --- a/docs/source/cli/airshipctl_baremetal_reboot.md +++ b/docs/source/cli/airshipctl_baremetal_reboot.md @@ -16,7 +16,7 @@ airshipctl baremetal reboot [flags] -h, --help help for reboot -l, --labels string Label(s) to filter desired baremetal host documents -n, --name string Name to filter desired baremetal host document - --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap") + --phase string airshipctl phase that contains the desired baremetal host document(s) (default "bootstrap-iso") ``` ### Options inherited from parent commands diff --git a/docs/source/cli/airshipctl_image_build.md b/docs/source/cli/airshipctl_image_build.md index 80534bd59..ae76310c5 100644 --- a/docs/source/cli/airshipctl_image_build.md +++ b/docs/source/cli/airshipctl_image_build.md @@ -14,7 +14,6 @@ airshipctl image build [flags] ``` -h, --help help for build - --progress show progress ``` ### Options inherited from parent commands diff --git a/go.mod b/go.mod index 044e6547e..1552b8440 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/Masterminds/sprig v2.22.0+incompatible github.com/Microsoft/go-winio v0.4.14 // indirect github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect - github.com/cheggaaa/pb/v3 v3.0.4 github.com/containerd/containerd v1.4.1 // indirect github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce github.com/docker/go-connections v0.4.0 // indirect diff --git a/go.sum b/go.sum index d1d94fbb1..1c4093aaf 100644 --- a/go.sum +++ b/go.sum @@ -65,8 +65,6 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs= github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs= @@ -108,8 +106,6 @@ github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1 github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 h1:HD4PLRzjuCVW79mQ0/pdsalOLHJ+FaEoqJLxfltpb2U= github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM= -github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -489,18 +485,13 @@ github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= -github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= -github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -842,9 +833,7 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/manifests/function/ephemeral/image_configuration.yaml b/manifests/function/ephemeral/image_configuration.yaml index 6a7d73dd8..451c29de9 100644 --- a/manifests/function/ephemeral/image_configuration.yaml +++ b/manifests/function/ephemeral/image_configuration.yaml @@ -1,14 +1,20 @@ apiVersion: airshipit.org/v1alpha1 -kind: ImageConfiguration +kind: IsoConfiguration metadata: name: isogen labels: airshipit.org/deploy-k8s: "false" builder: - networkConfigFileName: network-config - outputMetadataFileName: output-metadata.yaml - userDataFileName: user-data + userDataSelector: + kind: Secret + labelSelector: airshipit.org/ephemeral-user-data + userDataKey: userData + networkConfigSelector: + kind: BareMetalHost + labelSelector: airshipit.org/ephemeral-node + networkConfigKey: networkData + outputFileName: ephemeral.iso container: containerRuntime: docker - image: quay.io/airshipit/isogen:latest-ubuntu_focal - volume: /srv/iso:/config + image: quay.io/airshipit/image-builder:latest-ubuntu_focal + volume: /srv/images:/config diff --git a/manifests/function/ephemeral/remote_direct_configuration.yaml b/manifests/function/ephemeral/remote_direct_configuration.yaml index 27b1f0b47..913d19c10 100644 --- a/manifests/function/ephemeral/remote_direct_configuration.yaml +++ b/manifests/function/ephemeral/remote_direct_configuration.yaml @@ -4,4 +4,4 @@ metadata: name: default labels: airshipit.org/deploy-k8s: "false" -isoUrl: http://localhost:8099/ubuntu-focal.iso +isoUrl: http://localhost:8099/ephemeral.iso diff --git a/manifests/function/ephemeral/secret.yaml b/manifests/function/ephemeral/secret.yaml index fa35d04d4..b7c8617e8 100644 --- a/manifests/function/ephemeral/secret.yaml +++ b/manifests/function/ephemeral/secret.yaml @@ -12,6 +12,8 @@ stringData: # TODO: add download sources to the versions catalogue userData: | #cloud-config + # Expect that packages are already installed in base image + package_update: false ssh_pwauth: True chpasswd: list: | @@ -24,32 +26,7 @@ stringData: gecos: deployer ssh_pwauth: True runcmd: - - | - cat <______] 20%' where Prefix is trimmed log line from docker container - progressBarTemplate = `{{string . "prefix"}} {{bar . }} {{percent . }} ` - // defaultTerminalWidth is a default width of terminal if it's impossible to determine the actual one - defaultTerminalWidth = 80 - // multiplier is a number of log lines produces while installing 1 package - multiplier = 3 - // reInstallActions is a regular expression to check whether the log line contains of this substrings - reInstallActions = `Extracting|Unpacking|Configuring|Preparing|Setting` - reInstallBegin = `Retrieving Packages|newly installed` - reInstallFinish = `Base system installed successfully|mksquashfs` + builderConfigFileName = "builder-conf.yaml" + outputFileNameDefault = "ephemerial.iso" + userDataFileName = "user-data" + networkConfigFileName = "network-data" + outputMetadataFileName = "output-metadata.yaml" ) // BootstrapIsoOptions are used to generate bootstrap ISO @@ -56,60 +43,63 @@ type BootstrapIsoOptions struct { DocBundle document.Bundle Builder container.Container Doc document.Document - Cfg *v1alpha1.ImageConfiguration + Cfg *v1alpha1.IsoConfiguration // optional fields for verbose output - Debug bool - Progress bool - Writer io.Writer + Writer io.Writer } -// VerifyInputs verifies image configuration -func VerifyInputs(cfg *v1alpha1.ImageConfiguration) error { - if cfg.Container.Volume == "" { +func VerifyInputs(cfg *v1alpha1.IsoConfiguration) error { + if cfg.IsoContainer.Volume == "" { return config.ErrMissingConfig{ What: "Must specify volume bind for ISO builder container", } } - if (cfg.Builder.UserDataFileName == "") || (cfg.Builder.NetworkConfigFileName == "") { - return config.ErrMissingConfig{ - What: "UserDataFileName or NetworkConfigFileName are not specified in ISO builder config", - } - } - - vols := strings.Split(cfg.Container.Volume, ":") + vols := strings.Split(cfg.IsoContainer.Volume, ":") switch { case len(vols) == 1: - cfg.Container.Volume = fmt.Sprintf("%s:%s", vols[0], vols[0]) + cfg.IsoContainer.Volume = fmt.Sprintf("%s:%s", vols[0], vols[0]) case len(vols) > 2: return config.ErrInvalidConfig{ What: "Bad container volume format. Use hostPath:contPath", } } + + if cfg.Isogen.OutputFileName == "" { + log.Debugf("No outputFileName provided to Isogen. Using default: %s", outputFileNameDefault) + cfg.Isogen.OutputFileName = outputFileNameDefault + } + return nil } -func getContainerCfg( - cfg *v1alpha1.ImageConfiguration, +func getIsoContainerCfg( + cfg *v1alpha1.IsoConfiguration, builderCfgYaml []byte, userData []byte, netConf []byte, ) map[string][]byte { - hostVol := strings.Split(cfg.Container.Volume, ":")[0] + hostVol := strings.Split(cfg.IsoContainer.Volume, ":")[0] fls := make(map[string][]byte) - fls[filepath.Join(hostVol, cfg.Builder.UserDataFileName)] = userData - fls[filepath.Join(hostVol, cfg.Builder.NetworkConfigFileName)] = netConf + fls[filepath.Join(hostVol, userDataFileName)] = userData + fls[filepath.Join(hostVol, networkConfigFileName)] = netConf fls[filepath.Join(hostVol, builderConfigFileName)] = builderCfgYaml return fls } // CreateBootstrapIso prepares and runs appropriate container to create a bootstrap ISO func (opts BootstrapIsoOptions) CreateBootstrapIso() error { - cntVol := strings.Split(opts.Cfg.Container.Volume, ":")[1] + cntVol := strings.Split(opts.Cfg.IsoContainer.Volume, ":")[1] log.Print("Creating cloud-init for ephemeral K8s") - userData, netConf, err := cloudinit.GetCloudData(opts.DocBundle) + userData, netConf, err := cloudinit.GetCloudData( + opts.DocBundle, + opts.Cfg.Isogen.UserDataSelector, + opts.Cfg.Isogen.UserDataKey, + opts.Cfg.Isogen.NetworkConfigSelector, + opts.Cfg.Isogen.NetworkConfigKey, + ) if err != nil { return err } @@ -119,21 +109,27 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error { return err } - fls := getContainerCfg(opts.Cfg, builderCfgYaml, userData, netConf) + fls := getIsoContainerCfg(opts.Cfg, builderCfgYaml, userData, netConf) if err = util.WriteFiles(fls, 0600); err != nil { return err } - vols := []string{opts.Cfg.Container.Volume} + vols := []string{opts.Cfg.IsoContainer.Volume} builderCfgLocation := filepath.Join(cntVol, builderConfigFileName) log.Printf("Running default container command. Mounted dir: %s", vols) envVars := []string{ + fmt.Sprintf("IMAGE_TYPE=iso"), fmt.Sprintf("BUILDER_CONFIG=%s", builderCfgLocation), + fmt.Sprintf("USER_DATA_FILE=%s", userDataFileName), + fmt.Sprintf("NET_CONFIG_FILE=%s", networkConfigFileName), + fmt.Sprintf("OUTPUT_FILE_NAME=%s", opts.Cfg.Isogen.OutputFileName), + fmt.Sprintf("OUTPUT_METADATA_FILE_NAME=%s", outputMetadataFileName), fmt.Sprintf("http_proxy=%s", os.Getenv("http_proxy")), fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy")), fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")), fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")), + fmt.Sprintf("no_proxy=%s", os.Getenv("no_proxy")), fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY")), } @@ -144,25 +140,17 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error { log.Print("ISO generation is in progress. The whole process could take up to several minutes, please wait...") - if opts.Debug || opts.Progress { + if log.DebugEnabled() { var cLogs io.ReadCloser cLogs, err = opts.Builder.GetContainerLogs() if err != nil { log.Printf("failed to read container logs %s", err) } else { - switch { - case opts.Progress: - if err = ShowProgress(cLogs, opts.Writer); err != nil { - log.Debugf("the following error occurred while showing progress bar: %s", err.Error()) - } - case opts.Debug: - log.Print("start reading container logs") - // either container log output or progress bar will be shown - if _, err = io.Copy(opts.Writer, cLogs); err != nil { - log.Debugf("failed to write container logs to log output %s", err) - } - log.Print("got EOF from container logs") + log.Print("start reading container logs") + if _, err = io.Copy(opts.Writer, cLogs); err != nil { + log.Debugf("failed to write container logs to log output %s", err) } + log.Print("got EOF from container logs") } } @@ -171,7 +159,7 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error { } log.Print("ISO successfully built.") - if !opts.Debug { + if !log.DebugEnabled() { log.Print("Removing container.") return opts.Builder.RmContainer() } @@ -179,140 +167,3 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error { log.Debugf("Debug flag is set. Container %s stopped but not deleted.", opts.Builder.GetID()) return nil } - -// ShowProgress prints progress bar during bootstrap ISO preparation -func ShowProgress(reader io.ReadCloser, writer io.Writer) error { - reFindActions := regexp.MustCompile(reInstallActions) - reBeginInstall := regexp.MustCompile(reInstallBegin) - reFinishInstall := regexp.MustCompile(reInstallFinish) - - var bar *pb.ProgressBar - - scanner := bufio.NewScanner(reader) - scanner.Split(bufio.ScanLines) - // Reading container log line by line - for scanner.Scan() { - curLine := scanner.Text() - // Trying to find entry points of package installation - switch { - case reBeginInstall.MatchString(curLine): - if err := finalizePb(bar, nil); err != nil { - return err - } - - pkgCount, err := calculatePkgCount(scanner, writer, curLine) - if err != nil { - return finalizePb(bar, err) - } - - bar, err = initPb(pkgCount, writer) - if err != nil { - return err - } - case reFinishInstall.MatchString(curLine): - if err := finalizePb(bar, nil); err != nil { - return err - } - case reFindActions.MatchString(curLine): - if err := incrementPb(bar, curLine); err != nil { - return finalizePb(bar, err) - } - case strings.Contains(curLine, "filesystem.squashfs"): - fmt.Fprintln(writer, curLine) - } - } - - if bar != nil && bar.IsStarted() { - return finalizePb(bar, ErrUnexpectedPb{}) - } - - return nil -} - -func finalizePb(bar *pb.ProgressBar, e error) error { - if bar != nil && bar.IsStarted() { - bar.SetCurrent(bar.Total()) - if e != nil { - setPbPrefix(bar, "An error occurred while log parsing") - bar.Finish() - return e - } - - setPbPrefix(bar, "Completed") - bar.Finish() - if err := bar.Err(); err != nil { - return err - } - } - return e -} - -func initPb(pkgCount int, w io.Writer) (*pb.ProgressBar, error) { - bar := pb.ProgressBarTemplate(progressBarTemplate).New(pkgCount * multiplier) - bar.SetWriter(w).Start() - setPbPrefix(bar, "Installing required packages") - if err := bar.Err(); err != nil { - return nil, finalizePb(bar, err) - } - return bar, nil -} - -func incrementPb(bar *pb.ProgressBar, curLine string) error { - if bar != nil && bar.IsStarted() && bar.Current() < bar.Total() { - setPbPrefix(bar, curLine) - bar.Increment() - if err := bar.Err(); err != nil { - return finalizePb(bar, err) - } - } - return nil -} - -func setPbPrefix(bar *pb.ProgressBar, msg string) { - terminalWidth := defaultTerminalWidth - halfWidth := terminalWidth / 2 - bar.SetWidth(terminalWidth) - if len(msg) > halfWidth { - msg = fmt.Sprintf("%v...", msg[0:halfWidth-3]) - } else { - msg = fmt.Sprintf("%-*v", halfWidth, msg) - } - bar.Set("prefix", msg) -} - -func calculatePkgCount(scanner *bufio.Scanner, writer io.Writer, curLine string) (int, error) { - reFindNumbers := regexp.MustCompile("[0-9]+") - - // Trying to count how many packages is going to be installed - pkgCount := 0 - matches := reFindNumbers.FindAllString(curLine, -1) - if matches == nil { - // There is no numbers in line about base packages, counting them manually to get estimates - fmt.Fprint(writer, "Retrieving base packages ") - for scanner.Scan() { - curLine = scanner.Text() - if strings.Contains(curLine, "Retrieving") { - pkgCount++ - fmt.Fprint(writer, ".") - } - if strings.Contains(curLine, "Chosen extractor") { - fmt.Fprintln(writer, " Done") - return pkgCount, nil - } - } - } - if len(matches) >= 2 { - for _, v := range matches[0:2] { - j, err := strconv.Atoi(v) - if err != nil { - continue - } - pkgCount += j - } - if pkgCount > 0 { - return pkgCount, nil - } - } - - return pkgCount, ErrNoParsedNumPkgs{} -} diff --git a/pkg/bootstrap/isogen/command_test.go b/pkg/bootstrap/isogen/command_test.go index 2759a0351..8c44c9714 100644 --- a/pkg/bootstrap/isogen/command_test.go +++ b/pkg/bootstrap/isogen/command_test.go @@ -46,20 +46,19 @@ func TestBootstrapIso(t *testing.T) { volBind := tempVol + ":/dst" testErr := fmt.Errorf("TestErr") - testCfg := &api.ImageConfiguration{ - Container: &api.Container{ + testCfg := &api.IsoConfiguration{ + IsoContainer: &api.IsoContainer{ Volume: volBind, ContainerRuntime: "docker", }, - Builder: &api.Builder{ - UserDataFileName: "user-data", - NetworkConfigFileName: "net-conf", + Isogen: &api.Isogen{ + OutputFileName: "ephemeral.iso", }, } testDoc := &testdoc.MockDocument{ MockAsYAML: func() ([]byte, error) { return []byte("TESTDOC"), nil }, } - testBuilder := &testcontainer.MockContainer{ + testIsogen := &testcontainer.MockContainer{ MockRunCommand: func() error { return nil }, MockGetID: func() string { return testID }, MockRmContainer: func() error { return nil }, @@ -75,7 +74,7 @@ func TestBootstrapIso(t *testing.T) { tests := []struct { builder *testcontainer.MockContainer - cfg *api.ImageConfiguration + cfg *api.IsoConfiguration doc *testdoc.MockDocument debug bool expectedOut []string @@ -121,7 +120,7 @@ func TestBootstrapIso(t *testing.T) { expectedErr: testErr, }, { - builder: testBuilder, + builder: testIsogen, cfg: testCfg, doc: &testdoc.MockDocument{ MockAsYAML: func() ([]byte, error) { return nil, testErr }, @@ -140,7 +139,6 @@ func TestBootstrapIso(t *testing.T) { Builder: tt.builder, Doc: tt.doc, Cfg: tt.cfg, - Debug: tt.debug, } actualErr := bootstrapOpts.CreateBootstrapIso() actualOut := outBuf.String() @@ -159,40 +157,27 @@ func TestVerifyInputs(t *testing.T) { tests := []struct { name string - cfg *api.ImageConfiguration + cfg *api.IsoConfiguration args []string expectedErr error }{ { name: "missing-container-field", - cfg: &api.ImageConfiguration{ - Container: &api.Container{}, + cfg: &api.IsoConfiguration{ + IsoContainer: &api.IsoContainer{}, }, expectedErr: config.ErrMissingConfig{ What: "Must specify volume bind for ISO builder container", }, }, - { - name: "missing-filenames", - cfg: &api.ImageConfiguration{ - Container: &api.Container{ - Volume: tempVol + ":/dst", - }, - Builder: &api.Builder{}, - }, - expectedErr: config.ErrMissingConfig{ - What: "UserDataFileName or NetworkConfigFileName are not specified in ISO builder config", - }, - }, { name: "invalid-host-path", - cfg: &api.ImageConfiguration{ - Container: &api.Container{ + cfg: &api.IsoConfiguration{ + IsoContainer: &api.IsoContainer{ Volume: tempVol + ":/dst:/dst1", }, - Builder: &api.Builder{ - UserDataFileName: "user-data", - NetworkConfigFileName: "net-conf", + Isogen: &api.Isogen{ + OutputFileName: "ephemeral.iso", }, }, expectedErr: config.ErrInvalidConfig{ @@ -201,17 +186,26 @@ func TestVerifyInputs(t *testing.T) { }, { name: "success", - cfg: &api.ImageConfiguration{ - Container: &api.Container{ + cfg: &api.IsoConfiguration{ + IsoContainer: &api.IsoContainer{ Volume: tempVol, }, - Builder: &api.Builder{ - UserDataFileName: "user-data", - NetworkConfigFileName: "net-conf", + Isogen: &api.Isogen{ + OutputFileName: "ephemeral.iso", }, }, expectedErr: nil, }, + { + name: "success-using-output-file-default-", + cfg: &api.IsoConfiguration{ + IsoContainer: &api.IsoContainer{ + Volume: tempVol, + }, + Isogen: &api.Isogen{}, + }, + expectedErr: nil, + }, } for _, tt := range tests { @@ -222,29 +216,3 @@ func TestVerifyInputs(t *testing.T) { }) } } - -func TestShowProgress(t *testing.T) { - tests := []struct { - name string - input string - output string - }{ - { - name: "Process-debian-based-logs", - input: "testdata/debian-container-logs", - output: "testdata/pb-output-debian", - }, - } - - for _, tt := range tests { - tt := tt - - testInput, err := ioutil.ReadFile(tt.input) - require.NoError(t, err) - reader := ioutil.NopCloser(bytes.NewReader(testInput)) - writer := bytes.NewBuffer(nil) - err = isogen.ShowProgress(reader, writer) - require.NoError(t, err) - assert.Contains(t, writer.String(), "Completed") - } -} diff --git a/pkg/bootstrap/isogen/errors.go b/pkg/bootstrap/isogen/errors.go index 9c2adc29c..4d594c466 100644 --- a/pkg/bootstrap/isogen/errors.go +++ b/pkg/bootstrap/isogen/errors.go @@ -21,11 +21,3 @@ type ErrNoParsedNumPkgs struct { func (e ErrNoParsedNumPkgs) Error() string { return "No number of packages to install found in parsed container logs" } - -// ErrUnexpectedPb is returned when progress bar was not finished for some reason -type ErrUnexpectedPb struct { -} - -func (e ErrUnexpectedPb) Error() string { - return "An unexpected error occurred while parsing container logs" -} diff --git a/pkg/bootstrap/isogen/testdata/config/config b/pkg/bootstrap/isogen/testdata/config/config deleted file mode 100644 index e69de29bb..000000000 diff --git a/pkg/bootstrap/isogen/testdata/debian-container-logs b/pkg/bootstrap/isogen/testdata/debian-container-logs deleted file mode 100755 index 6f91f734d..000000000 --- a/pkg/bootstrap/isogen/testdata/debian-container-logs +++ /dev/null @@ -1,358 +0,0 @@ -+ _debootstrap -+ debootstrap --arch=amd64 --variant=minbase --foreign focal /root/LIVE_BOOT/chroot http://archive.ubuntu.com/ubuntu/ -I: Retrieving InRelease -I: Checking Release signature -I: Valid Release signature (key id F6ECB3762474EDA9D21B7022871920D1991BC93C) -I: Retrieving Packages -I: Validating Packages -I: Resolving dependencies of required packages... -I: Resolving dependencies of base packages... -I: Checking component main on http://archive.ubuntu.com/ubuntu... -I: Retrieving adduser 3.118ubuntu2 -I: Validating adduser 3.118ubuntu2 -I: Retrieving apt 2.0.2 -I: Validating apt 2.0.2 -I: Retrieving base-files 11ubuntu5 -I: Validating base-files 11ubuntu5 -I: Chosen extractor for .deb packages: dpkg-deb -I: Extracting base-files... -I: Installing core packages... -I: Unpacking required packages... -I: Unpacking base-files... -I: Configuring required packages... -I: Unpacking the base system... -I: Unpacking adduser... -I: Unpacking apt... -I: Configuring the base system... -I: Configuring adduser... -I: Configuring apt... -I: Base system installed successfully. -Reading package lists... Done -Building dependency tree -Reading state information... Done -The following additional packages will be installed: - ca-certificates cloud-guest-utils isc-dhcp-client libdns-export1109 - libglib2.0-0 libisc-export1105 libnetplan0 libyaml-0-2 netplan.io openssl - python3-attr python3-blinker python3-certifi python3-cffi-backend - python3-chardet python3-configobj python3-cryptography python3-distutils - python3-idna python3-importlib-metadata python3-jinja2 python3-json-pointer - python3-jsonpatch python3-jsonschema python3-jwt python3-lib2to3 - python3-markupsafe python3-more-itertools python3-netifaces python3-oauthlib - python3-pkg-resources python3-pyrsistent python3-requests python3-serial - python3-setuptools python3-six python3-urllib3 python3-yaml python3-zipp -Suggested packages: - resolvconf avahi-autoipd isc-dhcp-client-ddns network-manager - | wpasupplicant python-attr-doc python-blinker-doc python-configobj-doc - python-cryptography-doc python3-cryptography-vectors python-jinja2-doc - python-jsonschema-doc python3-crypto python3-openssl python3-socks - python3-wxgtk3.0 | python3-wxgtk python-setuptools-doc -Recommended packages: - eatmydata gdisk software-properties-common isc-dhcp-common libglib2.0-data - shared-mime-info xdg-user-dirs -The following NEW packages will be installed: - ca-certificates cloud-guest-utils cloud-init isc-dhcp-client - libdns-export1109 libglib2.0-0 libisc-export1105 libnetplan0 libyaml-0-2 - netplan.io openssl python3-attr python3-blinker python3-certifi - python3-cffi-backend python3-chardet python3-configobj python3-cryptography - python3-distutils python3-idna python3-importlib-metadata python3-jinja2 - python3-json-pointer python3-jsonpatch python3-jsonschema python3-jwt - python3-lib2to3 python3-markupsafe python3-more-itertools python3-netifaces - python3-oauthlib python3-pkg-resources python3-pyrsistent python3-requests - python3-serial python3-setuptools python3-six python3-urllib3 python3-yaml - python3-zipp -0 upgraded, 40 newly installed, 0 to remove and 0 not upgraded. -Need to get 5855 kB of archives. -After this operation, 22.3 MB of additional disk space will be used. -Get:1 http://archive.ubuntu.com/ubuntu focal/main amd64 openssl amd64 1.1.1f-1ubuntu2 [621 kB] -Get:2 http://archive.ubuntu.com/ubuntu focal/main amd64 ca-certificates all 20190110ubuntu1 [146 kB] -Get:3 http://archive.ubuntu.com/ubuntu focal/main amd64 libisc-export1105 amd64 1:9.11.16+dfsg-3~build1 [174 kB] -Get:4 http://archive.ubuntu.com/ubuntu focal/main amd64 libdns-export1109 amd64 1:9.11.16+dfsg-3~build1 [767 kB] -Get:5 http://archive.ubuntu.com/ubuntu focal/main amd64 isc-dhcp-client amd64 4.4.1-2.1ubuntu5 [246 kB] -Get:6 http://archive.ubuntu.com/ubuntu focal/main amd64 libglib2.0-0 amd64 2.64.2-1~fakesync1 [1284 kB] -Get:7 http://archive.ubuntu.com/ubuntu focal/main amd64 libyaml-0-2 amd64 0.2.2-1 [48.9 kB] -Get:8 http://archive.ubuntu.com/ubuntu focal/main amd64 libnetplan0 amd64 0.99-0ubuntu1 [22.9 kB] -Get:9 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-yaml amd64 5.3.1-1 [135 kB] -Get:10 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-netifaces amd64 0.10.4-1ubuntu4 [16.1 kB] -Get:11 http://archive.ubuntu.com/ubuntu focal/main amd64 netplan.io amd64 0.99-0ubuntu1 [70.8 kB] -Get:12 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cffi-backend amd64 1.14.0-1build1 [68.7 kB] -Get:13 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pkg-resources all 45.2.0-1 [130 kB] -Get:14 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-six all 1.14.0-2 [12.1 kB] -Get:15 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-attr all 19.3.0-2 [33.9 kB] -Get:16 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-blinker all 1.4+dfsg1-0.3ubuntu1 [13.2 kB] -Get:17 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-certifi all 2019.11.28-1 [149 kB] -Get:18 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-chardet all 3.0.4-4build1 [80.4 kB] -Get:19 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-configobj all 5.0.6-4 [34.1 kB] -Get:20 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-cryptography amd64 2.8-3 [211 kB] -Get:21 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-lib2to3 all 3.8.2-1ubuntu1 [74.1 kB] -Get:22 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-distutils all 3.8.2-1ubuntu1 [140 kB] -Get:23 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-idna all 2.8-1 [34.6 kB] -Get:24 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-more-itertools all 4.2.0-1build1 [39.4 kB] -Get:25 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-zipp all 1.0.0-1 [5312 B] -Get:26 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-importlib-metadata all 1.5.0-1 [9992 B] -Get:27 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-markupsafe amd64 1.1.0-1build2 [13.9 kB] -Get:28 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jinja2 all 2.10.1-2 [95.5 kB] -Get:29 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-json-pointer all 2.0-0ubuntu1 [8320 B] -Get:30 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jsonpatch all 1.23-3 [12.0 kB] -Get:31 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-setuptools all 45.2.0-1 [330 kB] -Get:32 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-pyrsistent amd64 0.15.5-1build1 [52.1 kB] -Get:33 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jsonschema all 3.2.0-0ubuntu2 [43.1 kB] -Get:34 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-jwt all 1.7.1-2ubuntu2 [17.4 kB] -Get:35 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-oauthlib all 3.1.0-1ubuntu2 [84.8 kB] -Get:36 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-urllib3 all 1.25.8-2 [88.1 kB] -Get:37 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-requests all 2.22.0-2ubuntu1 [47.1 kB] -Get:38 http://archive.ubuntu.com/ubuntu focal/main amd64 python3-serial all 3.4-5.1 [72.4 kB] -Get:39 http://archive.ubuntu.com/ubuntu focal/main amd64 cloud-guest-utils all 0.31-7-gd99b2d76-0ubuntu1 [16.2 kB] -Get:40 http://archive.ubuntu.com/ubuntu focal/main amd64 cloud-init all 20.1-10-g71af48df-0ubuntu5 [406 kB] -Fetched 5855 kB in 2s (2541 kB/s) -Selecting previously unselected package openssl. -(Reading database ... 46818 files and directories currently installed.) -Preparing to unpack .../00-openssl_1.1.1f-1ubuntu2_amd64.deb ... -Unpacking openssl (1.1.1f-1ubuntu2) ... -Selecting previously unselected package ca-certificates. -Preparing to unpack .../01-ca-certificates_20190110ubuntu1_all.deb ... -Unpacking ca-certificates (20190110ubuntu1) ... -Selecting previously unselected package libisc-export1105:amd64. -Preparing to unpack .../02-libisc-export1105_1%3a9.11.16+dfsg-3~build1_amd64.deb ... -Unpacking libisc-export1105:amd64 (1:9.11.16+dfsg-3~build1) ... -Selecting previously unselected package libdns-export1109. -Preparing to unpack .../03-libdns-export1109_1%3a9.11.16+dfsg-3~build1_amd64.deb ... -Unpacking libdns-export1109 (1:9.11.16+dfsg-3~build1) ... -Selecting previously unselected package isc-dhcp-client. -Preparing to unpack .../04-isc-dhcp-client_4.4.1-2.1ubuntu5_amd64.deb ... -Unpacking isc-dhcp-client (4.4.1-2.1ubuntu5) ... -Selecting previously unselected package libglib2.0-0:amd64. -Preparing to unpack .../05-libglib2.0-0_2.64.2-1~fakesync1_amd64.deb ... -Unpacking libglib2.0-0:amd64 (2.64.2-1~fakesync1) ... -Selecting previously unselected package libyaml-0-2:amd64. -Preparing to unpack .../06-libyaml-0-2_0.2.2-1_amd64.deb ... -Unpacking libyaml-0-2:amd64 (0.2.2-1) ... -Selecting previously unselected package libnetplan0:amd64. -Preparing to unpack .../07-libnetplan0_0.99-0ubuntu1_amd64.deb ... -Unpacking libnetplan0:amd64 (0.99-0ubuntu1) ... -Selecting previously unselected package python3-yaml. -Preparing to unpack .../08-python3-yaml_5.3.1-1_amd64.deb ... -Unpacking python3-yaml (5.3.1-1) ... -Selecting previously unselected package python3-netifaces. -Preparing to unpack .../09-python3-netifaces_0.10.4-1ubuntu4_amd64.deb ... -Unpacking python3-netifaces (0.10.4-1ubuntu4) ... -Selecting previously unselected package netplan.io. -Preparing to unpack .../10-netplan.io_0.99-0ubuntu1_amd64.deb ... -Unpacking netplan.io (0.99-0ubuntu1) ... -Selecting previously unselected package python3-cffi-backend. -Preparing to unpack .../11-python3-cffi-backend_1.14.0-1build1_amd64.deb ... -Unpacking python3-cffi-backend (1.14.0-1build1) ... -Selecting previously unselected package python3-pkg-resources. -Preparing to unpack .../12-python3-pkg-resources_45.2.0-1_all.deb ... -Unpacking python3-pkg-resources (45.2.0-1) ... -Selecting previously unselected package python3-six. -Preparing to unpack .../13-python3-six_1.14.0-2_all.deb ... -Unpacking python3-six (1.14.0-2) ... -Selecting previously unselected package python3-attr. -Preparing to unpack .../14-python3-attr_19.3.0-2_all.deb ... -Unpacking python3-attr (19.3.0-2) ... -Selecting previously unselected package python3-blinker. -Preparing to unpack .../15-python3-blinker_1.4+dfsg1-0.3ubuntu1_all.deb ... -Unpacking python3-blinker (1.4+dfsg1-0.3ubuntu1) ... -Selecting previously unselected package python3-certifi. -Preparing to unpack .../16-python3-certifi_2019.11.28-1_all.deb ... -Unpacking python3-certifi (2019.11.28-1) ... -Selecting previously unselected package python3-chardet. -Preparing to unpack .../17-python3-chardet_3.0.4-4build1_all.deb ... -Unpacking python3-chardet (3.0.4-4build1) ... -Selecting previously unselected package python3-configobj. -Preparing to unpack .../18-python3-configobj_5.0.6-4_all.deb ... -Unpacking python3-configobj (5.0.6-4) ... -Selecting previously unselected package python3-cryptography. -Preparing to unpack .../19-python3-cryptography_2.8-3_amd64.deb ... -Unpacking python3-cryptography (2.8-3) ... -Selecting previously unselected package python3-lib2to3. -Preparing to unpack .../20-python3-lib2to3_3.8.2-1ubuntu1_all.deb ... -Unpacking python3-lib2to3 (3.8.2-1ubuntu1) ... -Selecting previously unselected package python3-distutils. -Preparing to unpack .../21-python3-distutils_3.8.2-1ubuntu1_all.deb ... -Unpacking python3-distutils (3.8.2-1ubuntu1) ... -Selecting previously unselected package python3-idna. -Preparing to unpack .../22-python3-idna_2.8-1_all.deb ... -Unpacking python3-idna (2.8-1) ... -Selecting previously unselected package python3-more-itertools. -Preparing to unpack .../23-python3-more-itertools_4.2.0-1build1_all.deb ... -Unpacking python3-more-itertools (4.2.0-1build1) ... -Selecting previously unselected package python3-zipp. -Preparing to unpack .../24-python3-zipp_1.0.0-1_all.deb ... -Unpacking python3-zipp (1.0.0-1) ... -Selecting previously unselected package python3-importlib-metadata. -Preparing to unpack .../25-python3-importlib-metadata_1.5.0-1_all.deb ... -Unpacking python3-importlib-metadata (1.5.0-1) ... -Selecting previously unselected package python3-markupsafe. -Preparing to unpack .../26-python3-markupsafe_1.1.0-1build2_amd64.deb ... -Unpacking python3-markupsafe (1.1.0-1build2) ... -Selecting previously unselected package python3-jinja2. -Preparing to unpack .../27-python3-jinja2_2.10.1-2_all.deb ... -Unpacking python3-jinja2 (2.10.1-2) ... -Selecting previously unselected package python3-json-pointer. -Preparing to unpack .../28-python3-json-pointer_2.0-0ubuntu1_all.deb ... -Unpacking python3-json-pointer (2.0-0ubuntu1) ... -Selecting previously unselected package python3-jsonpatch. -Preparing to unpack .../29-python3-jsonpatch_1.23-3_all.deb ... -Unpacking python3-jsonpatch (1.23-3) ... -Selecting previously unselected package python3-setuptools. -Preparing to unpack .../30-python3-setuptools_45.2.0-1_all.deb ... -Unpacking python3-setuptools (45.2.0-1) ... -Selecting previously unselected package python3-pyrsistent:amd64. -Preparing to unpack .../31-python3-pyrsistent_0.15.5-1build1_amd64.deb ... -Unpacking python3-pyrsistent:amd64 (0.15.5-1build1) ... -Selecting previously unselected package python3-jsonschema. -Preparing to unpack .../32-python3-jsonschema_3.2.0-0ubuntu2_all.deb ... -Unpacking python3-jsonschema (3.2.0-0ubuntu2) ... -Selecting previously unselected package python3-jwt. -Preparing to unpack .../33-python3-jwt_1.7.1-2ubuntu2_all.deb ... -Unpacking python3-jwt (1.7.1-2ubuntu2) ... -Selecting previously unselected package python3-oauthlib. -Preparing to unpack .../34-python3-oauthlib_3.1.0-1ubuntu2_all.deb ... -Unpacking python3-oauthlib (3.1.0-1ubuntu2) ... -Selecting previously unselected package python3-urllib3. -Preparing to unpack .../35-python3-urllib3_1.25.8-2_all.deb ... -Unpacking python3-urllib3 (1.25.8-2) ... -Selecting previously unselected package python3-requests. -Preparing to unpack .../36-python3-requests_2.22.0-2ubuntu1_all.deb ... -Unpacking python3-requests (2.22.0-2ubuntu1) ... -Selecting previously unselected package python3-serial. -Preparing to unpack .../37-python3-serial_3.4-5.1_all.deb ... -Unpacking python3-serial (3.4-5.1) ... -Selecting previously unselected package cloud-guest-utils. -Preparing to unpack .../38-cloud-guest-utils_0.31-7-gd99b2d76-0ubuntu1_all.deb ... -Unpacking cloud-guest-utils (0.31-7-gd99b2d76-0ubuntu1) ... -Selecting previously unselected package cloud-init. -Preparing to unpack .../39-cloud-init_20.1-10-g71af48df-0ubuntu5_all.deb ... -Unpacking cloud-init (20.1-10-g71af48df-0ubuntu5) ... -Setting up python3-pkg-resources (45.2.0-1) ... -Setting up python3-attr (19.3.0-2) ... -Setting up python3-jwt (1.7.1-2ubuntu2) ... -Setting up libyaml-0-2:amd64 (0.2.2-1) ... -Setting up libglib2.0-0:amd64 (2.64.2-1~fakesync1) ... -No schema files found: doing nothing. -Setting up libnetplan0:amd64 (0.99-0ubuntu1) ... -Setting up python3-yaml (5.3.1-1) ... -Setting up python3-markupsafe (1.1.0-1build2) ... -Setting up python3-serial (3.4-5.1) ... -Setting up python3-six (1.14.0-2) ... -Setting up python3-jinja2 (2.10.1-2) ... -Setting up libisc-export1105:amd64 (1:9.11.16+dfsg-3~build1) ... -Setting up python3-chardet (3.0.4-4build1) ... -Setting up python3-configobj (5.0.6-4) ... -Setting up python3-idna (2.8-1) ... -Setting up python3-urllib3 (1.25.8-2) ... -Setting up python3-netifaces (0.10.4-1ubuntu4) ... -Setting up python3-pyrsistent:amd64 (0.15.5-1build1) ... -Setting up python3-json-pointer (2.0-0ubuntu1) ... -Setting up openssl (1.1.1f-1ubuntu2) ... -Setting up python3-lib2to3 (3.8.2-1ubuntu1) ... -Setting up cloud-guest-utils (0.31-7-gd99b2d76-0ubuntu1) ... -Setting up python3-cffi-backend (1.14.0-1build1) ... -Setting up python3-blinker (1.4+dfsg1-0.3ubuntu1) ... -Setting up python3-distutils (3.8.2-1ubuntu1) ... -Setting up python3-more-itertools (4.2.0-1build1) ... -Setting up python3-setuptools (45.2.0-1) ... -Setting up libdns-export1109 (1:9.11.16+dfsg-3~build1) ... -Setting up python3-jsonpatch (1.23-3) ... -update-alternatives: using /usr/bin/jsonpatch-jsondiff to provide /usr/bin/jsondiff (jsondiff) in auto mode -Setting up isc-dhcp-client (4.4.1-2.1ubuntu5) ... -Setting up python3-zipp (1.0.0-1) ... -Setting up netplan.io (0.99-0ubuntu1) ... -Setting up ca-certificates (20190110ubuntu1) ... -debconf: unable to initialize frontend: Dialog -debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.) -debconf: falling back to frontend: Readline -debconf: unable to initialize frontend: Readline -debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC contains: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.30.0 /usr/local/share/perl/5.30.0 /usr/lib/x86_64-linux-gnu/perl5/5.30 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl/5.30 /usr/share/perl/5.30 /usr/local/lib/site_perl /usr/lib/x86_64-linux-gnu/perl-base) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 7.) -debconf: falling back to frontend: Teletype -Updating certificates in /etc/ssl/certs... -128 added, 0 removed; done. -Setting up python3-certifi (2019.11.28-1) ... -Setting up python3-cryptography (2.8-3) ... -Setting up python3-requests (2.22.0-2ubuntu1) ... -Setting up python3-importlib-metadata (1.5.0-1) ... -Setting up python3-oauthlib (3.1.0-1ubuntu2) ... -Setting up python3-jsonschema (3.2.0-0ubuntu2) ... -Setting up cloud-init (20.1-10-g71af48df-0ubuntu5) ... -debconf: unable to initialize frontend: Dialog -debconf: (No usable dialog-like program is installed, so the dialog based frontend cannot be used. at /usr/share/perl5/Debconf/FrontEnd/Dialog.pm line 76.) -debconf: falling back to frontend: Readline -debconf: unable to initialize frontend: Readline -debconf: (Can't locate Term/ReadLine.pm in @INC (you may need to install the Term::ReadLine module) (@INC contains: /etc/perl /usr/local/lib/x86_64-linux-gnu/perl/5.30.0 /usr/local/share/perl/5.30.0 /usr/lib/x86_64-linux-gnu/perl5/5.30 /usr/share/perl5 /usr/lib/x86_64-linux-gnu/perl/5.30 /usr/share/perl/5.30 /usr/local/lib/site_perl /usr/lib/x86_64-linux-gnu/perl-base) at /usr/share/perl5/Debconf/FrontEnd/Readline.pm line 7.) -debconf: falling back to frontend: Teletype -No diversion 'diversion of /etc/init/ureadahead.conf to /etc/init/ureadahead.conf.disabled by cloud-init', none removed. -Created symlink /etc/systemd/system/cloud-init.target.wants/cloud-config.service → /lib/systemd/system/cloud-config.service. -Created symlink /etc/systemd/system/cloud-init.target.wants/cloud-final.service → /lib/systemd/system/cloud-final.service. -Created symlink /etc/systemd/system/cloud-init.target.wants/cloud-init-local.service → /lib/systemd/system/cloud-init-local.service. -Created symlink /etc/systemd/system/cloud-init.target.wants/cloud-init.service → /lib/systemd/system/cloud-init.service. -Processing triggers for libc-bin (2.31-0ubuntu9) ... -Processing triggers for ca-certificates (20190110ubuntu1) ... -Updating certificates in /etc/ssl/certs... -0 added, 0 removed; done. -Running hooks in /etc/ca-certificates/update.d... -done. -# # + rm -rf /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_focal_InRelease /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_focal_main_binary-amd64_Packages /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_focal_main_i18n_Translation-en /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_focal_universe_binary-amd64_Packages /var/lib/apt/lists/archive.ubuntu.com_ubuntu_dists_focal_universe_i18n_Translation-en /var/lib/apt/lists/auxfiles /var/lib/apt/lists/lock /var/lib/apt/lists/partial -# -+ mkdir -p /root/LIVE_BOOT/image/openstack/latest -+ cp /builder/meta_data.json /root/LIVE_BOOT/image/openstack/latest -+ cp /config/user-data /root/LIVE_BOOT/image/openstack/latest/user_data -+ yq r -j /config/network-config -+ echo 'datasource_list: [ ConfigDrive, None ]' -+ _make_kernel -+ mkdir -p /root/LIVE_BOOT/scratch /root/LIVE_BOOT/image/live -+ mksquashfs /root/LIVE_BOOT/chroot /root/LIVE_BOOT/image/live/filesystem.squashfs -e boot -Parallel mksquashfs: Using 24 processors -Creating 4.0 filesystem on /root/LIVE_BOOT/image/live/filesystem.squashfs, block size 131072. - - -Exportable Squashfs 4.0 filesystem, gzip compressed, data block size 131072 - compressed data, compressed metadata, compressed fragments, - compressed xattrs, compressed ids - duplicates are removed -Filesystem size 624492.88 Kbytes (609.86 Mbytes) - 45.67% of uncompressed filesystem size (1367509.59 Kbytes) -Inode table size 459804 bytes (449.03 Kbytes) - 26.38% of uncompressed inode table size (1743271 bytes) -Directory table size 486559 bytes (475.16 Kbytes) - 44.71% of uncompressed directory table size (1088368 bytes) -Xattr table size 36 bytes (0.04 Kbytes) - 90.00% of uncompressed xattr table size (40 bytes) -Number of duplicate files found 8756 -Number of inodes 52089 -Number of files 41051 -Number of fragments 2859 -Number of symbolic links 1709 -Number of device nodes 8 -Number of fifo nodes 0 -Number of socket nodes 0 -Number of directories 9321 -Number of ids (unique uids + gids) 11 -Number of uids 3 - root (0) - unknown (102) - _apt (100) -Number of gids 10 - root (0) - shadow (42) - unknown (102) - unknown (103) - utmp (43) - unknown (110) - tty (5) - staff (50) - adm (4) - mail (8) -+ cp /root/LIVE_BOOT/chroot/boot/vmlinuz-5.4.0-26-generic /root/LIVE_BOOT/image/vmlinuz -+ cp /root/LIVE_BOOT/chroot/boot/initrd.img-5.4.0-26-generic /root/LIVE_BOOT/image/initrd -+ _grub_install -+ cp /builder/grub.conf /root/LIVE_BOOT/scratch/grub.cfg -+ touch /root/LIVE_BOOT/image/UBUNTU_FOCAL_CUSTOM -+ grub-mkstandalone --format=x86_64-efi --output=/root/LIVE_BOOT/scratch/bootx64.efi --locales= --fonts= boot/grub/grub.cfg=/root/LIVE_BOOT/scratch/grub.cfg -+ cd /root/LIVE_BOOT/scratch -+ dd if=/dev/zero of=efiboot.img bs=1M count=10 -10+0 records in -10+0 records out -10485760 bytes (10 MB, 10 MiB) copied, 0.00700445 s, 1.5 GB/s -+ mkfs.vfat efiboot.img -mkfs.fat 4.1 (2017-01-24) diff --git a/pkg/config/constants.go b/pkg/config/constants.go index 855adbcf2..2621dee01 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -21,7 +21,7 @@ const ( Ephemeral = "ephemeral" InitinfraPhase = "initinfra" ClusterctlPhase = InitinfraPhase - BootstrapPhase = "bootstrap" + BootstrapPhase = "bootstrap-iso" ) // Constants defining default values diff --git a/pkg/document/document.go b/pkg/document/document.go index 6edb57140..3fca2b2c0 100644 --- a/pkg/document/document.go +++ b/pkg/document/document.go @@ -15,6 +15,7 @@ package document import ( + b64 "encoding/base64" "fmt" "k8s.io/apimachinery/pkg/runtime" @@ -290,3 +291,31 @@ func NewDocumentFromBytes(b []byte) (Document, error) { err = doc.SetKustomizeResource(res) return doc, err } + +// DecodeSecretData returns base64-decoded secret data +func DecodeSecretData(cfg Document, key string) ([]byte, error) { + var needsBase64Decode = false + + // TODO(alanmeadows): distinguish between missing key + // and missing data/stringData keys in the Secret + data, err := cfg.GetStringMap("data") + if err == nil { + needsBase64Decode = true + } else { + // we'll catch any error below + data, err = cfg.GetStringMap("stringData") + if err != nil { + return nil, ErrDataNotSupplied{DocName: cfg.GetName(), Key: "data or stringData"} + } + } + + res, ok := data[key] + if !ok { + return nil, ErrDataNotSupplied{DocName: cfg.GetName(), Key: key} + } + + if needsBase64Decode { + return b64.StdEncoding.DecodeString(res) + } + return []byte(res), nil +} diff --git a/pkg/document/errors.go b/pkg/document/errors.go index f66a07a41..c0dc36720 100644 --- a/pkg/document/errors.go +++ b/pkg/document/errors.go @@ -49,6 +49,19 @@ type ErrRuntimeObjectKind struct { Obj runtime.Object } +// ErrDataNotSupplied error returned of no data in the Secret +type ErrDataNotSupplied struct { + DocName string + Key string +} + +// ErrDuplicateNetworkDataDocuments error returned if multiple matching documents +// were found with the same name in the same namespace +type ErrDuplicateNetworkDataDocuments struct { + DocName string + Namespace string +} + // ErrBadValueFormat returned if wrong field type requested type ErrBadValueFormat struct { Value string @@ -76,6 +89,14 @@ func (e ErrRuntimeObjectKind) Error() string { return fmt.Sprintf("object %#v has either none or multiple kinds in scheme (expected one)", e.Obj) } +func (e ErrDataNotSupplied) Error() string { + return fmt.Sprintf("Document %s has no key %s", e.DocName, e.Key) +} + +func (e ErrDuplicateNetworkDataDocuments) Error() string { + return fmt.Sprintf("Found more than one document with the name %s in namespace %s", e.DocName, e.Namespace) +} + func (e ErrBadValueFormat) Error() string { return fmt.Sprintf("value of %s expected to have %s type, got %s", e.Value, e.Expected, e.Actual) } diff --git a/pkg/document/selectors.go b/pkg/document/selectors.go index 8a4fe9f23..e72f105a7 100644 --- a/pkg/document/selectors.go +++ b/pkg/document/selectors.go @@ -189,3 +189,37 @@ func NewClusterctlMetadataSelector() Selector { ClusterctlMetadataVersion, ClusterctlMetadataKind) } + +//GetSecretData returns data located with a given key of a given document +func GetSecretData(docBundle Bundle, apiSelector types.Selector, key string) ([]byte, error) { + s := NewSelector() + if apiSelector.Group != "" && apiSelector.Version != "" && apiSelector.Kind != "" { + s = s.ByGvk(apiSelector.Group, apiSelector.Version, apiSelector.Kind) + } else if apiSelector.Kind != "" { + s = s.ByKind(apiSelector.Kind) + } + if apiSelector.Namespace != "" { + s = s.ByNamespace(apiSelector.Namespace) + } + if apiSelector.Name != "" { + s = s.ByName(apiSelector.Name) + } + if apiSelector.AnnotationSelector != "" { + s = s.ByAnnotation(apiSelector.AnnotationSelector) + } + if apiSelector.LabelSelector != "" { + s = s.ByLabel(apiSelector.LabelSelector) + } + + doc, docErr := docBundle.SelectOne(s) + if docErr != nil { + return nil, docErr + } + + // finally, try and retrieve the data we want from the document + data, keyErr := DecodeSecretData(doc, key) + if keyErr != nil { + return nil, keyErr + } + return data, nil +} diff --git a/pkg/document/selectors_test.go b/pkg/document/selectors_test.go index f87a91ed1..0ddd97f38 100644 --- a/pkg/document/selectors_test.go +++ b/pkg/document/selectors_test.go @@ -60,6 +60,16 @@ func TestSelectorsPositive(t *testing.T) { require.NoError(t, err) assert.Len(t, doc, 1) }) + + t.Run("TestGetSecretData", func(t *testing.T) { + data, err := document.GetSecretData(bundle, types.Selector{ + Gvk: resid.Gvk{Kind: "Secret"}, + LabelSelector: "airshipit.org/ephemeral-user-data", + }, + "userData") + require.NoError(t, err) + assert.Equal(t, []byte("cloud-init"), data) + }) } func TestSelectorsNegative(t *testing.T) { diff --git a/pkg/phase/executors/errors.go b/pkg/phase/executors/errors.go index a17da11ba..5c886d6f0 100755 --- a/pkg/phase/executors/errors.go +++ b/pkg/phase/executors/errors.go @@ -28,10 +28,10 @@ func (e ErrUnknownExecutorAction) Error() string { return fmt.Sprintf("unknown action type '%s'", e.Action) } -// ErrIsoGenNilBundle is returned when isogen executor is not provided with bundle -type ErrIsoGenNilBundle struct { +// ErrIsogenNilBundle is returned when isogen executor is not provided with bundle +type ErrIsogenNilBundle struct { } -func (e ErrIsoGenNilBundle) Error() string { +func (e ErrIsogenNilBundle) Error() string { return "Cannot build iso with empty bundle, no data source is available" } diff --git a/pkg/phase/executors/isogen.go b/pkg/phase/executors/isogen.go index 8f15d407a..925ae7a92 100644 --- a/pkg/phase/executors/isogen.go +++ b/pkg/phase/executors/isogen.go @@ -40,13 +40,13 @@ type IsogenExecutor struct { ExecutorBundle document.Bundle ExecutorDocument document.Document - ImgConf *v1alpha1.ImageConfiguration + ImgConf *v1alpha1.IsoConfiguration Builder container.Container } // RegisterIsogenExecutor adds executor to phase executor registry func RegisterIsogenExecutor(registry map[schema.GroupVersionKind]ifc.ExecutorFactory) error { - obj := v1alpha1.DefaultImageConfiguration() + obj := v1alpha1.DefaultIsoConfiguration() gvks, _, err := v1alpha1.Scheme.ObjectKinds(obj) if err != nil { return err @@ -57,9 +57,9 @@ func RegisterIsogenExecutor(registry map[schema.GroupVersionKind]ifc.ExecutorFac // NewIsogenExecutor creates instance of phase executor func NewIsogenExecutor(cfg ifc.ExecutorConfig) (ifc.Executor, error) { - apiObj := &v1alpha1.ImageConfiguration{ - Container: &v1alpha1.Container{}, - Builder: &v1alpha1.Builder{}, + apiObj := &v1alpha1.IsoConfiguration{ + IsoContainer: &v1alpha1.IsoContainer{}, + Isogen: &v1alpha1.Isogen{}, } err := cfg.ExecutorDocument.ToAPIObject(apiObj, v1alpha1.Scheme) if err != nil { @@ -83,7 +83,7 @@ func (c *IsogenExecutor) Run(evtCh chan events.Event, opts ifc.RunOptions) { defer close(evtCh) if c.ExecutorBundle == nil { - handleError(evtCh, ErrIsoGenNilBundle{}) + handleError(evtCh, ErrIsogenNilBundle{}) return } @@ -104,8 +104,8 @@ func (c *IsogenExecutor) Run(evtCh chan events.Event, opts ifc.RunOptions) { ctx := context.Background() builder, err := container.NewContainer( ctx, - c.ImgConf.Container.ContainerRuntime, - c.ImgConf.Container.Image) + c.ImgConf.IsoContainer.ContainerRuntime, + c.ImgConf.IsoContainer.Image) c.Builder = builder if err != nil { handleError(evtCh, err) @@ -118,8 +118,6 @@ func (c *IsogenExecutor) Run(evtCh chan events.Event, opts ifc.RunOptions) { Builder: c.Builder, Doc: c.ExecutorDocument, Cfg: c.ImgConf, - Debug: log.DebugEnabled(), - Progress: opts.Progress, Writer: log.Writer(), } @@ -145,10 +143,10 @@ func (c *IsogenExecutor) Run(evtCh chan events.Event, opts ifc.RunOptions) { }) } -func verifyArtifacts(cfg *v1alpha1.ImageConfiguration) error { - hostVol := strings.Split(cfg.Container.Volume, ":")[0] - metadataPath := filepath.Join(hostVol, cfg.Builder.OutputMetadataFileName) - _, err := os.Stat(metadataPath) +func verifyArtifacts(cfg *v1alpha1.IsoConfiguration) error { + hostVol := strings.Split(cfg.IsoContainer.Volume, ":")[0] + outputFilePath := filepath.Join(hostVol, cfg.Isogen.OutputFileName) + _, err := os.Stat(outputFilePath) return err } diff --git a/pkg/phase/executors/isogen_test.go b/pkg/phase/executors/isogen_test.go index 3f242ce7f..9837aba40 100644 --- a/pkg/phase/executors/isogen_test.go +++ b/pkg/phase/executors/isogen_test.go @@ -15,6 +15,9 @@ package executors_test import ( + "log" + "os" + "path/filepath" "testing" "time" @@ -36,19 +39,17 @@ import ( var ( isogenExecutorDoc = ` apiVersion: airshipit.org/v1alpha1 -kind: ImageConfiguration +kind: IsoConfiguration metadata: name: isogen labels: airshipit.org/deploy-k8s: "false" builder: - networkConfigFileName: network-config - outputMetadataFileName: output-metadata.yaml - userDataFileName: user-data + outputFileName: ephemeral.iso container: containerRuntime: docker - image: quay.io/airshipit/isogen:latest-ubuntu_focal - volume: /srv/iso:/config` + image: quay.io/airshipit/image-builder:latest-ubuntu_focal + volume: /srv/images:/config` executorBundlePath = "../../bootstrap/isogen/testdata/primary/site/test-site/ephemeral/bootstrap" ) @@ -57,7 +58,7 @@ func TestRegisterIsogenExecutor(t *testing.T) { expectedGVK := schema.GroupVersionKind{ Group: "airshipit.org", Version: "v1alpha1", - Kind: "ImageConfiguration", + Kind: "IsoConfiguration", } err := executors.RegisterIsogenExecutor(registry) require.NoError(t, err) @@ -83,15 +84,20 @@ func TestIsogenExecutorRun(t *testing.T) { tempVol, cleanup := testutil.TempDir(t, "bootstrap-test") defer cleanup(t) + emptyFileName := filepath.Join(tempVol, "ephemeral.iso") + emptyFile, createErr := os.Create(emptyFileName) + require.NoError(t, createErr) + log.Println(emptyFile) + emptyFile.Close() + volBind := tempVol + ":/dst" - testCfg := &v1alpha1.ImageConfiguration{ - Container: &v1alpha1.Container{ + testCfg := &v1alpha1.IsoConfiguration{ + IsoContainer: &v1alpha1.IsoContainer{ Volume: volBind, ContainerRuntime: "docker", }, - Builder: &v1alpha1.Builder{ - UserDataFileName: "user-data", - NetworkConfigFileName: "net-conf", + Isogen: &v1alpha1.Isogen{ + OutputFileName: "ephemeral.iso", }, } testDoc := &testdoc.MockDocument{ diff --git a/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/bootstrap/phase.yaml b/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/bootstrap/phase.yaml index 26fcb21db..c3910bf48 100644 --- a/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/bootstrap/phase.yaml +++ b/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/bootstrap/phase.yaml @@ -2,7 +2,7 @@ apiVersion: airshipit.org/v1alpha1 kind: Phase metadata: - name: bootstrap + name: bootstrap-iso clusterName: ephemeral-cluster config: executorRef: diff --git a/pkg/remote/testdata/noisourl/manifests/site/test-site/ephemeral/bootstrap/phase.yaml b/pkg/remote/testdata/noisourl/manifests/site/test-site/ephemeral/bootstrap/phase.yaml index 26fcb21db..c3910bf48 100644 --- a/pkg/remote/testdata/noisourl/manifests/site/test-site/ephemeral/bootstrap/phase.yaml +++ b/pkg/remote/testdata/noisourl/manifests/site/test-site/ephemeral/bootstrap/phase.yaml @@ -2,7 +2,7 @@ apiVersion: airshipit.org/v1alpha1 kind: Phase metadata: - name: bootstrap + name: bootstrap-iso clusterName: ephemeral-cluster config: executorRef: diff --git a/pkg/remote/testdata/noremote/manifests/site/test-site/ephemeral/bootstrap/phase.yaml b/pkg/remote/testdata/noremote/manifests/site/test-site/ephemeral/bootstrap/phase.yaml index 26fcb21db..c3910bf48 100644 --- a/pkg/remote/testdata/noremote/manifests/site/test-site/ephemeral/bootstrap/phase.yaml +++ b/pkg/remote/testdata/noremote/manifests/site/test-site/ephemeral/bootstrap/phase.yaml @@ -2,7 +2,7 @@ apiVersion: airshipit.org/v1alpha1 kind: Phase metadata: - name: bootstrap + name: bootstrap-iso clusterName: ephemeral-cluster config: executorRef: diff --git a/playbooks/airshipctl-gate-runner.yaml b/playbooks/airshipctl-gate-runner.yaml index ba6ff0878..52f49a32d 100644 --- a/playbooks/airshipctl-gate-runner.yaml +++ b/playbooks/airshipctl-gate-runner.yaml @@ -19,7 +19,7 @@ - ./tools/deployment/21_systemwide_executable.sh - ./tools/deployment/22_test_configs.sh - ./tools/deployment/23_pull_documents.sh - - ./tools/deployment/24_build_ephemeral_iso.sh + - ./tools/deployment/24_build_images.sh - ./tools/deployment/25_deploy_ephemeral_node.sh - ./tools/deployment/26_deploy_metal3_capi_ephemeral_node.sh - ./tools/deployment/30_deploy_controlplane.sh diff --git a/playbooks/vars/test-config.yaml b/playbooks/vars/test-config.yaml index 48bff310c..50b66798d 100644 --- a/playbooks/vars/test-config.yaml +++ b/playbooks/vars/test-config.yaml @@ -15,7 +15,7 @@ airship_config_iso_gen_target_path: "{{ serve_dir }}" airship_config_phase_repo_url: "https://review.opendev.org/airship/airshipctl" airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) | default(local_src_dir) }}" airship_config_ephemeral_ip: "{{ airship_gate_ipam.nat_network.ephemeral_ip }}" -airship_config_iso_builder_docker_image: "quay.io/airshipit/isogen:latest-ubuntu_focal" +airship_config_iso_builder_docker_image: "quay.io/airshipit/image-builder:latest-ubuntu_focal" airship_config_site_path: manifests/site/test-site airship_config_ca_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= airship_config_client_cert_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUQwRENDQXJnQ0ZFdFBveEZYSjVrVFNWTXQ0OVlqcHBQL3hCYnlNQTBHQ1NxR1NJYjNEUUVCQ3dVQU1CVXgKRXpBUkJnTlZCQU1UQ210MVltVnlibVYwWlhNd0hoY05NakF3TVRJME1Ua3hOVEV3V2hjTk1qa3hNakF5TVRreApOVEV3V2pBME1Sa3dGd1lEVlFRRERCQnJkV0psY201bGRHVnpMV0ZrYldsdU1SY3dGUVlEVlFRS0RBNXplWE4wClpXMDZiV0Z6ZEdWeWN6Q0NBaUl3RFFZSktvWklodmNOQVFFQkJRQURnZ0lQQURDQ0Fnb0NnZ0lCQU1iaFhUUmsKVjZiZXdsUjBhZlpBdTBGYWVsOXRtRThaSFEvaGtaSHhuTjc2bDZUUFltcGJvaDRvRjNGMFFqbzROS1o5NVRuWgo0OWNoV240eFJiZVlPU25EcDBpV0Qzd0pXUlZ5aVFvVUFyYTlNcHVPNkVFU1FpbFVGNXNxc0VXUVdVMjBETStBCkdxK1k0Z2c3eDJ1Q0hTdk1GUmkrNEw5RWlXR2xnRDIvb1hXUm5NWEswNExQajZPb3Vkb2Zid2RmT3J6dTBPVkUKUzR0eGtuS1BCY1BUU3YxMWVaWVhja0JEVjNPbExENEZ3dTB3NTcwcnczNzAraEpYdlZxd3Zjb2RjZjZEL1BXWQowamlnd2ppeUJuZ2dXYW04UVFjd1Nud3o0d05sV3hKOVMyWUJFb1ptdWxVUlFaWVk5ZXRBcEpBdFMzTjlUNlQ2ClovSlJRdEdhZDJmTldTYkxEck5qdU1OTGhBYWRMQnhJUHpBNXZWWk5aalJkdEMwU25pMlFUMTVpSFp4d1RxcjQKakRQQ0pYRXU3KytxcWpQVldUaUZLK3JqcVNhS1pqVWZVaUpHQkJWcm5RZkJENHNtRnNkTjB5cm9tYTZOYzRMNQpKS21RV1NHdmd1aG0zbW5sYjFRaVRZanVyZFJQRFNmdmwrQ0NHbnA1QkkvZ1pwMkF1SHMvNUpKVTJlc1ZvL0xsCkVPdHdSOXdXd3dXcTAvZjhXS3R4bVRrMTUyOUp2dFBGQXQweW1CVjhQbHZlYnVwYmJqeW5pL2xWbTJOYmV6dWUKeCtlMEpNbGtWWnFmYkRSS243SjZZSnJHWW1CUFV0QldoSVkzb1pJVTFEUXI4SUlIbkdmYlZoWlR5ME1IMkFCQQp1dlVQcUtSVk80UGkxRTF4OEE2eWVPeVRDcnB4L0pBazVyR2RBZ01CQUFFd0RRWUpLb1pJaHZjTkFRRUxCUUFECmdnRUJBSWNFM1BxZHZDTVBIMnJzMXJESk9ESHY3QWk4S01PVXZPRi90RjlqR2EvSFBJbkh3RlVFNEltbldQeDYKVUdBMlE1bjFsRDFGQlU0T0M4eElZc3VvS1VQVHk1T0t6SVNMNEZnL0lEcG54STlrTXlmNStMR043aG8rblJmawpCZkpJblVYb0tERW1neHZzSWFGd1h6bGtSTDJzL1lKYUZRRzE1Uis1YzFyckJmd2dJOFA5Tkd6aEM1cXhnSmovCm04K3hPMGhXUmJIYklrQ21NekRib2pCSWhaL00rb3VYR1doei9TakpodXhZTVBnek5MZkFGcy9PMTVaSjd3YXcKZ3ZoSGc3L2E5UzRvUCtEYytPa3VrMkV1MUZjL0E5WHpWMzc5aWhNWW5ub3RQMldWeFZ3b0ZZQUg0NUdQcDZsUApCQmwyNnkxc2JMbjl6aGZYUUJIMVpFN0EwZVE9Ci0tLS0tRU5EIENFUlRJRklDQVRFLS0tLS0K diff --git a/tools/deployment/22_test_configs.sh b/tools/deployment/22_test_configs.sh index 2ddf02919..30d2fdf8c 100755 --- a/tools/deployment/22_test_configs.sh +++ b/tools/deployment/22_test_configs.sh @@ -14,7 +14,7 @@ set -xe -export ISO_DIR=${ISO_DIR:-"/srv/iso"} +export IMAGE_DIR=${IMAGE_DIR:-"/srv/images"} export SERVE_PORT=${SERVE_PORT:-"8099"} export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD} export USER_NAME=${USER:-"ubuntu"} @@ -22,14 +22,14 @@ export USE_PROXY=${USE_PROXY:-"false"} export HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} export HTTP_PROXY=${HTTP_PROXY:-${http_proxy}} export NO_PROXY=${NO_PROXY:-${no_proxy}} -export AIRSHIP_CONFIG_ISO_GEN_TARGET_PATH=${ISO_DIR} -export AIRSHIP_CONFIG_ISO_BUILDER_DOCKER_IMAGE=${BUILDER_IMAGE:-"quay.io/airshipit/isogen:latest-ubuntu_focal"} +export AIRSHIP_CONFIG_ISO_GEN_TARGET_PATH=${IMAGE_DIR} +export AIRSHIP_CONFIG_ISO_BUILDER_DOCKER_IMAGE=${BUILDER_IMAGE:-"quay.io/airshipit/image-builder:latest-ubuntu_focal"} export REMOTE_TYPE=redfish export REMOTE_INSECURE=true export REMOTE_PROXY=false export AIRSHIP_CONFIG_ISO_SERVE_HOST=${HOST:-"localhost"} export AIRSHIP_CONFIG_ISO_PORT=${SERVE_PORT} -export AIRSHIP_CONFIG_ISO_NAME=${ISO_NAME:-"ubuntu-focal.iso"} +export AIRSHIP_CONFIG_ISO_NAME=${ISO_NAME:-"ephemeral.iso"} export AIRSHIP_CONFIG_METADATA_PATH=${AIRSHIP_CONFIG_METADATA_PATH:-"manifests/site/test-site/metadata.yaml"} export SYSTEM_ACTION_RETRIES=30 export SYSTEM_REBOOT_DELAY=30 diff --git a/tools/deployment/24_build_ephemeral_iso.sh b/tools/deployment/24_build_images.sh similarity index 63% rename from tools/deployment/24_build_ephemeral_iso.sh rename to tools/deployment/24_build_images.sh index ee8f91d97..acb1f8725 100755 --- a/tools/deployment/24_build_ephemeral_iso.sh +++ b/tools/deployment/24_build_images.sh @@ -16,27 +16,32 @@ set -xe export USER_NAME=${USER:-"ubuntu"} -ISO_DIR=${ISO_DIR:-"/srv/iso"} +IMAGE_DIR=${IMAGE_DIR:-"/srv/images"} CLEANUP_SERVE_DIR=${CLEANUP_SERVE_DIR:-"false"} SITE_NAME=${SITE_NAME:-test-site} +# List of phases to run to build images. +IMAGE_PHASES=${IMAGE_PHASES:-"bootstrap-iso"} #Create serving directories and assign permission and ownership -sudo rm -rf ${ISO_DIR} -sudo mkdir -p ${ISO_DIR} -sudo chmod -R 755 ${ISO_DIR} -sudo chown -R ${USER_NAME} ${ISO_DIR} +sudo rm -rf ${IMAGE_DIR} +sudo mkdir -p ${IMAGE_DIR} +sudo chmod -R 755 ${IMAGE_DIR} +sudo chown -R ${USER_NAME} ${IMAGE_DIR} -echo "Build ephemeral iso" -airshipctl phase run bootstrap --debug +unset IFS +for phase in $IMAGE_PHASES; do + echo "Build phase: $phase" + airshipctl phase run $phase --debug +done -echo "List generated iso" -ls -lth ${ISO_DIR} +echo "List generated images" +ls -lth ${IMAGE_DIR} -echo "Remove the container used for iso generation" +echo "Remove the container used for image generation" sudo docker rm $(docker ps -a -f status=exited -q) #cleanup the directories if [ "${CLEANUP_SERVE_DIR}" == "true" ] || [ "${CLEANUP_SERVE_DIR}" == "True" ]; then - echo "Clean directories used by ephemeral iso build" - sudo rm -rf ${ISO_DIR} + echo "Clean directories used by image-builder" + sudo rm -rf ${IMAGE_DIR} fi diff --git a/tools/deployment/30_deploy_controlplane.sh b/tools/deployment/30_deploy_controlplane.sh index 2c4068736..b9b374b3b 100755 --- a/tools/deployment/30_deploy_controlplane.sh +++ b/tools/deployment/30_deploy_controlplane.sh @@ -14,7 +14,7 @@ set -ex -TARGET_IMAGE_DIR="/srv/iso" +TARGET_IMAGE_DIR="/srv/images" EPHEMERAL_DOMAIN_NAME="air-ephemeral" TARGET_IMAGE_URL="https://cloud-images.ubuntu.com/focal/current/focal-server-cloudimg-amd64.img" export KUBECONFIG=${KUBECONFIG:-"$HOME/.airship/kubeconfig"} @@ -45,7 +45,7 @@ if [ "${DOWNLOAD}" != "304" ] then curl -sSLo ${TARGET_IMAGE_DIR}/target-image.qcow2 ${TARGET_IMAGE_URL} fi -md5sum /srv/iso/target-image.qcow2 | cut -d ' ' -f 1 > ${TARGET_IMAGE_DIR}/target-image.qcow2.md5sum +md5sum /srv/images/target-image.qcow2 | cut -d ' ' -f 1 > ${TARGET_IMAGE_DIR}/target-image.qcow2.md5sum echo "Create target k8s cluster resources" airshipctl phase run controlplane-ephemeral --debug diff --git a/tools/deployment/azure/31_create_configs.sh b/tools/deployment/azure/31_create_configs.sh index 67f80b25f..46ab093a6 100755 --- a/tools/deployment/azure/31_create_configs.sh +++ b/tools/deployment/azure/31_create_configs.sh @@ -14,7 +14,7 @@ set -xe -export ISO_DIR=${ISO_DIR:-"/srv/iso"} +export IMAGE_DIR=${IMAGE_DIR:-"/srv/images"} export SERVE_PORT=${SERVE_PORT:-"8099"} export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD} export USER_NAME=${USER:-"ubuntu"} @@ -23,8 +23,8 @@ export HTTPS_PROXY=${HTTPS_PROXY:-${https_proxy}} export HTTPS_PROXY=${HTTP_PROXY:-${http_proxy}} export NO_PROXY=${NO_PROXY:-${no_proxy}} export REMOTE_WORK_DIR=${remote_work_dir:-"/tmp/airship"} -export AIRSHIP_CONFIG_ISO_GEN_TARGET_PATH=${ISO_DIR} -export AIRSHIP_CONFIG_ISO_BUILDER_DOCKER_IMAGE=${BUILDER_IMAGE:-"quay.io/airshipit/isogen:latest-debian_stable"} +export AIRSHIP_CONFIG_ISO_GEN_TARGET_PATH=${IMAGE_DIR} +export AIRSHIP_CONFIG_ISO_BUILDER_DOCKER_IMAGE=${BUILDER_IMAGE:-"quay.io/airshipit/image-builder:latest-ubuntu_focal"} export REMOTE_TYPE=redfish export REMOTE_INSECURE=true export REMOTE_PROXY=false diff --git a/tools/deployment/clean.sh b/tools/deployment/clean.sh index 773f9d9e5..e780753ea 100755 --- a/tools/deployment/clean.sh +++ b/tools/deployment/clean.sh @@ -14,7 +14,7 @@ set -x -sudo rm -rf ~/.airship/ ~/.ansible.cfg /srv/iso/* /tmp/airship/ +sudo rm -rf ~/.airship/ ~/.ansible.cfg /srv/images/* /tmp/airship/ sudo service sushy-tools stop sudo service apache2 stop sudo kill -9 $(lsof -t -i:8000 -i:8099) diff --git a/tools/gate/00_setup.sh b/tools/gate/00_setup.sh index 88ce0a1a5..ce8c036f0 100755 --- a/tools/gate/00_setup.sh +++ b/tools/gate/00_setup.sh @@ -19,7 +19,7 @@ set -xe -export ISO_DIR=${ISO_DIR:-"/srv/iso"} +export IMAGE_DIR=${IMAGE_DIR:-"/srv/images"} export SERVE_PORT=${SERVE_PORT:-"8099"} export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD} export TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"} diff --git a/tools/gate/config_template.yaml b/tools/gate/config_template.yaml index e73134b3f..a0c83246d 100644 --- a/tools/gate/config_template.yaml +++ b/tools/gate/config_template.yaml @@ -11,7 +11,7 @@ # limitations under the License. --- -serve_dir: /srv/iso +serve_dir: /srv/images serve_port: 8099 local_src_dir: "$AIRSHIPCTL_WS" ansible_user: root diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index ddbc64dfb..a6c959222 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -132,7 +132,7 @@ # 21_systemwide_executable.sh is run in the build-gate pre-run above - ./tools/deployment/22_test_configs.sh - ./tools/deployment/23_pull_documents.sh - - ./tools/deployment/24_build_ephemeral_iso.sh + - ./tools/deployment/24_build_images.sh - ./tools/deployment/25_deploy_ephemeral_node.sh - ./tools/deployment/26_deploy_metal3_capi_ephemeral_node.sh - ./tools/deployment/30_deploy_controlplane.sh @@ -142,7 +142,7 @@ - ./tools/deployment/34_deploy_worker_node.sh - ./tools/deployment/35_deploy_workload.sh - ./tools/deployment/36_verify_hwcc_profiles.sh - serve_dir: /srv/iso + serve_dir: /srv/images serve_port: 8099 log_roles: - gather-system-logs