Merge "Add progress bar and improve cmd output for image build command"

This commit is contained in:
Zuul 2020-09-30 14:34:03 +00:00 committed by Gerrit Code Review
commit 2a47c00f00
12 changed files with 2042 additions and 142 deletions

View File

@ -23,13 +23,23 @@ import (
// NewImageBuildCommand creates a new command with the capability to build an ISO image.
func NewImageBuildCommand(cfgFactory config.Factory) *cobra.Command {
options := &isogen.Options{
CfgFactory: cfgFactory,
}
cmd := &cobra.Command{
Use: "build",
Short: "Build ISO image",
RunE: func(cmd *cobra.Command, args []string) error {
return isogen.GenerateBootstrapIso(cfgFactory)
return options.GenerateBootstrapIso()
},
}
flags := cmd.Flags()
flags.BoolVar(
&options.Progress,
"progress",
false,
"show progress")
return cmd
}

1
go.mod
View File

@ -5,6 +5,7 @@ go 1.13
require (
github.com/Masterminds/sprig v2.22.0+incompatible
github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 // indirect
github.com/cheggaaa/pb/v3 v3.0.4
github.com/docker/docker v1.4.2-0.20200203170920-46ec8731fbce
github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect

12
go.sum
View File

@ -87,6 +87,8 @@ github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:H
github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg=
github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7 h1:uSoVVbwJiQipAclBbw+8quDsfcvFjOpI5iCf4p/cqCs=
github.com/alcortesm/tgz v0.0.0-20161220082320-9c5fe88206d7/go.mod h1:6zEj6s6u/ghQa61ZWa/C2Aw3RkjiTBOix7dkqa1VLIs=
@ -157,6 +159,8 @@ github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1
github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1 h1:HD4PLRzjuCVW79mQ0/pdsalOLHJ+FaEoqJLxfltpb2U=
github.com/chai2010/gettext-go v0.0.0-20170215093142-bf70f2a70fb1/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw=
github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
github.com/cheggaaa/pb/v3 v3.0.4 h1:QZEPYOj2ix6d5oEg63fbHmpolrnNiwjUsk+h74Yt4bM=
github.com/cheggaaa/pb/v3 v3.0.4/go.mod h1:7rgWxLrAUcFMkvJuv09+DYi7mMUYi8nO9iOWcvGJPfw=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@ -285,6 +289,7 @@ github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d h1:105gxyaGwC
github.com/exponent-io/jsonpath v0.0.0-20151013193312-d6023ce2651d/go.mod h1:ZZMPRZwes7CROmyNKgQzC3XPs6L/G2EJLHddWejkmf4=
github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb/go.mod h1:bH6Xx7IW64qjjJq8M2u4dxNaBiDfKK+z/3eGDpXEQhc=
github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys=
github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
github.com/fatih/structtag v1.1.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
@ -684,6 +689,7 @@ github.com/martinlindhe/base36 v1.0.0/go.mod h1:+AtEs8xrBpCeYgSLoY/aJ6Wf37jtBuR0
github.com/matoous/godox v0.0.0-20190911065817-5d6d842e92eb/go.mod h1:1BELzlh859Sh1c6+90blK8lbYy0kwQf1bYlBhBysy1s=
github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc=
@ -692,10 +698,14 @@ github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx
github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
github.com/mattn/go-isatty v0.0.6/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+twI54=
github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-shellwords v1.0.9/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
@ -1221,11 +1231,13 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191025021431-6c3a3bfe00ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191113165036-4c7a9d0fe056/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

View File

@ -15,12 +15,18 @@
package isogen
import (
"bufio"
"context"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/cheggaaa/pb/v3"
"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
"opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit"
"opendev.org/airship/airshipctl/pkg/config"
@ -32,15 +38,31 @@ import (
const (
builderConfigFileName = "builder-conf.yaml"
// progressBarTemplate is a template string for progress bar
// looks like 'Prefix [-->______] 20%' where Prefix is trimmed log line from docker container
progressBarTemplate = `{{string . "prefix"}} {{bar . }} {{percent . }} `
// defaultTerminalWidth is a default width of terminal if it's impossible to determine the actual one
defaultTerminalWidth = 80
// multiplier is a number of log lines produces while installing 1 package
multiplier = 3
// reInstallActions is a regular expression to check whether the log line contains of this substrings
reInstallActions = `Extracting|Unpacking|Configuring|Preparing|Setting`
)
// Options is used for generate bootstrap ISO
type Options struct {
CfgFactory config.Factory
Progress bool
}
// GenerateBootstrapIso will generate data for cloud init and start ISO builder container
// TODO (vkuzmin): Remove this public function and move another functions
// to the executor module when the phases will be ready
func GenerateBootstrapIso(cfgFactory config.Factory) error {
func (s *Options) GenerateBootstrapIso() error {
ctx := context.Background()
globalConf, err := cfgFactory()
globalConf, err := s.CfgFactory()
if err != nil {
return err
}
@ -80,7 +102,7 @@ func GenerateBootstrapIso(cfgFactory config.Factory) error {
return err
}
err = createBootstrapIso(docBundle, builder, doc, imageConfiguration, log.DebugEnabled())
err = createBootstrapIso(docBundle, builder, doc, imageConfiguration, log.DebugEnabled(), s.Progress)
if err != nil {
return err
}
@ -141,6 +163,7 @@ func createBootstrapIso(
doc document.Document,
cfg *v1alpha1.ImageConfiguration,
debug bool,
progress bool,
) error {
cntVol := strings.Split(cfg.Container.Volume, ":")[1]
log.Print("Creating cloud-init for ephemeral K8s")
@ -162,20 +185,44 @@ func createBootstrapIso(
vols := []string{cfg.Container.Volume}
builderCfgLocation := filepath.Join(cntVol, builderConfigFileName)
log.Printf("Running default container command. Mounted dir: %s", vols)
if err := builder.RunCommand(
[]string{},
nil,
vols,
[]string{
fmt.Sprintf("BUILDER_CONFIG=%s", builderCfgLocation),
fmt.Sprintf("http_proxy=%s", os.Getenv("http_proxy")),
fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy")),
fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")),
fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")),
fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY")),
},
debug,
); err != nil {
envVars := []string{
fmt.Sprintf("BUILDER_CONFIG=%s", builderCfgLocation),
fmt.Sprintf("http_proxy=%s", os.Getenv("http_proxy")),
fmt.Sprintf("https_proxy=%s", os.Getenv("https_proxy")),
fmt.Sprintf("HTTP_PROXY=%s", os.Getenv("HTTP_PROXY")),
fmt.Sprintf("HTTPS_PROXY=%s", os.Getenv("HTTPS_PROXY")),
fmt.Sprintf("NO_PROXY=%s", os.Getenv("NO_PROXY")),
}
err = builder.RunCommand([]string{}, nil, vols, envVars)
if err != nil {
return err
}
log.Print("ISO generation is in progress. The whole process could take up to several minutes, please wait...")
if debug || progress {
var cLogs io.ReadCloser
cLogs, err = builder.GetContainerLogs()
if err != nil {
log.Printf("failed to read container logs %s", err)
} else {
switch {
case progress:
showProgress(cLogs, log.Writer())
case debug:
log.Print("start reading container logs")
// either container log output or progress bar will be shown
if _, err = io.Copy(log.Writer(), cLogs); err != nil {
log.Debugf("failed to write container logs to log output %s", err)
}
log.Print("got EOF from container logs")
}
}
}
if err = builder.WaitUntilFinished(); err != nil {
return err
}
@ -188,3 +235,97 @@ func createBootstrapIso(
log.Debugf("Debug flag is set. Container %s stopped but not deleted.", builder.GetID())
return nil
}
func showProgress(reader io.ReadCloser, writer io.Writer) {
reFindActions := regexp.MustCompile(reInstallActions)
var bar *pb.ProgressBar
scanner := bufio.NewScanner(reader)
scanner.Split(bufio.ScanLines)
// Reading container log line by line
for scanner.Scan() {
curLine := scanner.Text()
// Trying to find entry points of package installation
switch {
case strings.Contains(curLine, "Retrieving Packages") ||
strings.Contains(curLine, "newly installed"):
finalizePb(bar, "Completed")
pkgCount := calculatePkgCount(scanner, writer, curLine)
if pkgCount > 0 {
bar = pb.ProgressBarTemplate(progressBarTemplate).Start(pkgCount * multiplier)
bar.SetWriter(writer)
setPbPrefix(bar, "Installing required packages")
}
case strings.Contains(curLine, "Base system installed successfully") ||
strings.Contains(curLine, "mksquashfs"):
finalizePb(bar, "Completed")
case bar != nil && bar.IsStarted():
if reFindActions.MatchString(curLine) {
if bar.Current() < bar.Total() {
setPbPrefix(bar, curLine)
bar.Increment()
}
}
case strings.Contains(curLine, "filesystem.squashfs"):
fmt.Fprintln(writer, curLine)
}
}
finalizePb(bar, "An unexpected error occurred while log parsing")
}
func finalizePb(bar *pb.ProgressBar, msg string) {
if bar != nil && bar.IsStarted() {
bar.SetCurrent(bar.Total())
setPbPrefix(bar, msg)
bar.Finish()
}
}
func setPbPrefix(bar *pb.ProgressBar, msg string) {
terminalWidth := defaultTerminalWidth
halfWidth := terminalWidth / 2
bar.SetWidth(terminalWidth)
if len(msg) > halfWidth {
msg = fmt.Sprintf("%v...", msg[0:halfWidth-3])
} else {
msg = fmt.Sprintf("%-*v", halfWidth, msg)
}
bar.Set("prefix", msg)
}
func calculatePkgCount(scanner *bufio.Scanner, writer io.Writer, curLine string) int {
reFindNumbers := regexp.MustCompile("[0-9]+")
// Trying to count how many packages is going to be installed
pkgCount := 0
matches := reFindNumbers.FindAllString(curLine, -1)
if matches == nil {
// There is no numbers is line about base packages, counting them manually to get estimates
fmt.Fprint(writer, "Retrieving base packages ")
for scanner.Scan() {
curLine = scanner.Text()
if strings.Contains(curLine, "Retrieving") {
pkgCount++
fmt.Fprint(writer, ".")
}
if strings.Contains(curLine, "Chosen extractor") {
fmt.Fprintln(writer, " Done")
return pkgCount
}
}
}
if len(matches) >= 2 {
for _, v := range matches[0:2] {
j, err := strconv.Atoi(v)
if err != nil {
continue
}
pkgCount += j
}
}
return pkgCount
}

View File

@ -15,9 +15,12 @@
package isogen
import (
"bufio"
"bytes"
"fmt"
"io"
"io/ioutil"
"os"
"strings"
"testing"
@ -32,23 +35,24 @@ import (
)
type mockContainer struct {
imagePull func() error
runCommand func() error
runCommandOutput func() (io.ReadCloser, error)
rmContainer func() error
getID func() string
imagePull func() error
runCommand func() error
getContainerLogs func() (io.ReadCloser, error)
rmContainer func() error
getID func() string
waitUntilFinished func() error
}
func (mc *mockContainer) ImagePull() error {
return mc.imagePull()
}
func (mc *mockContainer) RunCommand([]string, io.Reader, []string, []string, bool) error {
func (mc *mockContainer) RunCommand([]string, io.Reader, []string, []string) error {
return mc.runCommand()
}
func (mc *mockContainer) RunCommandOutput([]string, io.Reader, []string, []string) (io.ReadCloser, error) {
return mc.runCommandOutput()
func (mc *mockContainer) GetContainerLogs() (io.ReadCloser, error) {
return mc.getContainerLogs()
}
func (mc *mockContainer) RmContainer() error {
@ -59,6 +63,12 @@ func (mc *mockContainer) GetID() string {
return mc.getID()
}
func (mc *mockContainer) WaitUntilFinished() error {
return mc.waitUntilFinished()
}
const testID = "TESTID"
func TestBootstrapIso(t *testing.T) {
bundle, err := document.NewBundleByPath("testdata/primary/site/test-site/ephemeral/bootstrap")
require.NoError(t, err, "Building Bundle Failed")
@ -83,7 +93,7 @@ func TestBootstrapIso(t *testing.T) {
}
testBuilder := &mockContainer{
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
getID: func() string { return testID },
rmContainer: func() error { return nil },
}
@ -105,7 +115,9 @@ func TestBootstrapIso(t *testing.T) {
}{
{
builder: &mockContainer{
runCommand: func() error { return testErr },
runCommand: func() error { return testErr },
waitUntilFinished: func() error { return nil },
rmContainer: func() error { return nil },
},
cfg: testCfg,
doc: testDoc,
@ -114,7 +126,13 @@ func TestBootstrapIso(t *testing.T) {
expectedErr: testErr,
},
{
builder: testBuilder,
builder: &mockContainer{
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
waitUntilFinished: func() error { return nil },
rmContainer: func() error { return nil },
getContainerLogs: func() (io.ReadCloser, error) { return ioutil.NopCloser(strings.NewReader("")), nil },
},
cfg: testCfg,
doc: testDoc,
debug: true,
@ -123,9 +141,10 @@ func TestBootstrapIso(t *testing.T) {
},
{
builder: &mockContainer{
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
rmContainer: func() error { return testErr },
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
rmContainer: func() error { return testErr },
waitUntilFinished: func() error { return nil },
},
cfg: testCfg,
doc: testDoc,
@ -148,7 +167,7 @@ func TestBootstrapIso(t *testing.T) {
for _, tt := range tests {
outBuf := &bytes.Buffer{}
log.Init(tt.debug, outBuf)
actualErr := createBootstrapIso(bundle, tt.builder, tt.doc, tt.cfg, tt.debug)
actualErr := createBootstrapIso(bundle, tt.builder, tt.doc, tt.cfg, tt.debug, false)
actualOut := outBuf.String()
for _, line := range tt.expectedOut {
@ -233,70 +252,92 @@ func TestGenerateBootstrapIso(t *testing.T) {
airshipConfigPath := "testdata/config/config"
kubeConfigPath := "testdata/config/kubeconfig"
t.Run("EnsureCompleteError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
expectedErr := config.ErrMissingConfig{What: "Context with name ''"}
settings.CurrentContext = ""
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
assert.Equal(t, expectedErr, actualErr)
})
t.Run("ContextEntryPointError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
cfg, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
cfg.Manifests["default"].Repositories = make(map[string]*config.Repository)
settings := &Options{CfgFactory: func() (*config.Config, error) {
return cfg, nil
}}
expectedErr := config.ErrMissingPrimaryRepo{}
settings.Manifests["default"].Repositories = make(map[string]*config.Repository)
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
actualErr := settings.GenerateBootstrapIso()
assert.Equal(t, expectedErr, actualErr)
})
t.Run("NewBundleByPathError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
cfg, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
cfg.Manifests["default"].TargetPath = "/nonexistent"
settings := &Options{CfgFactory: func() (*config.Config, error) {
return cfg, nil
}}
expectedErr := config.ErrMissingPhaseDocument{PhaseName: "bootstrap"}
settings.Manifests["default"].TargetPath = "/nonexistent"
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
actualErr := settings.GenerateBootstrapIso()
assert.Equal(t, expectedErr, actualErr)
})
t.Run("SelectOneError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
cfg, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
cfg.Manifests["default"].SubPath = "missingkinddoc/site/test-site"
settings := &Options{CfgFactory: func() (*config.Config, error) {
return cfg, nil
}}
expectedErr := document.ErrDocNotFound{
Selector: document.NewSelector().ByGvk("airshipit.org", "v1alpha1", "ImageConfiguration")}
settings.Manifests["default"].SubPath = "missingkinddoc/site/test-site"
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
actualErr := settings.GenerateBootstrapIso()
assert.Equal(t, expectedErr, actualErr)
})
t.Run("ToObjectError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
cfg, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
cfg.Manifests["default"].SubPath = "missingmetadoc/site/test-site"
settings := &Options{CfgFactory: func() (*config.Config, error) {
return cfg, nil
}}
expectedErrMessage := "missing metadata.name in object"
settings.Manifests["default"].SubPath = "missingmetadoc/site/test-site"
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
actualErr := settings.GenerateBootstrapIso()
assert.Contains(t, actualErr.Error(), expectedErrMessage)
})
t.Run("verifyInputsError", func(t *testing.T) {
settings, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
cfg, err := config.CreateFactory(&airshipConfigPath, &kubeConfigPath)()
require.NoError(t, err)
cfg.Manifests["default"].SubPath = "missingvoldoc/site/test-site"
settings := &Options{CfgFactory: func() (*config.Config, error) {
return cfg, nil
}}
expectedErr := config.ErrMissingConfig{What: "Must specify volume bind for ISO builder container"}
settings.Manifests["default"].SubPath = "missingvoldoc/site/test-site"
actualErr := GenerateBootstrapIso(func() (*config.Config, error) {
return settings, nil
})
actualErr := settings.GenerateBootstrapIso()
assert.Equal(t, expectedErr, actualErr)
})
}
func TestShowProgress(t *testing.T) {
tests := []struct {
name string
input string
output string
}{
{
name: "Process-debian-based-logs",
input: "testdata/debian-container-logs",
output: "testdata/pb-output-debian",
},
}
for _, tt := range tests {
tt := tt
file, err := os.Open(tt.input)
require.NoError(t, err)
reader := ioutil.NopCloser(bufio.NewReader(file))
writer := &bytes.Buffer{}
showProgress(reader, writer)
err = file.Close()
require.NoError(t, err)
expected, err := ioutil.ReadFile(tt.output)
require.NoError(t, err)
assert.Equal(t, expected, writer.Bytes())
}
}

View File

@ -115,7 +115,7 @@ func (c *Executor) Run(evtCh chan events.Event, opts ifc.RunOptions) {
}
}
err := createBootstrapIso(c.ExecutorBundle, c.builder, c.ExecutorDocument, c.imgConf, log.DebugEnabled())
err := createBootstrapIso(c.ExecutorBundle, c.builder, c.ExecutorDocument, c.imgConf, log.DebugEnabled(), false)
if err != nil {
handleError(evtCh, err)
return

View File

@ -103,9 +103,10 @@ func TestExecutorRun(t *testing.T) {
{
name: "Run isogen successfully",
builder: &mockContainer{
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
rmContainer: func() error { return nil },
runCommand: func() error { return nil },
getID: func() string { return "TESTID" },
rmContainer: func() error { return nil },
waitUntilFinished: func() error { return nil },
},
expectedEvt: []events.Event{
{

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,2 @@
Retrieving base packages ........................................................................................... Done
Completed [----------------------------] 100.00% Completed [----------------------------] 100.00% Completed [----------------------------] 100.00% Creating 4.0 filesystem on /root/LIVE_BOOT/image/live/filesystem.squashfs, block size 131072.

View File

@ -24,10 +24,11 @@ import (
// defines methods that must be implemented for CRE (e.g. docker, containerd or CRI-O)
type Container interface {
ImagePull() error
RunCommand([]string, io.Reader, []string, []string, bool) error
RunCommandOutput([]string, io.Reader, []string, []string) (io.ReadCloser, error)
RunCommand([]string, io.Reader, []string, []string) error
GetContainerLogs() (io.ReadCloser, error)
RmContainer() error
GetID() string
WaitUntilFinished() error
}
// NewContainer returns instance of Container interface implemented by particular driver

View File

@ -241,7 +241,6 @@ func (c *DockerContainer) RunCommand(
containerInput io.Reader,
volumeMounts []string,
envVars []string,
debug bool,
) error {
realCmd, err := c.getCmd(cmd)
if err != nil {
@ -279,53 +278,13 @@ func (c *DockerContainer) RunCommand(
return err
}
if debug {
log.Debug("start reading container logs")
var reader io.ReadCloser
reader, err = c.dockerClient.ContainerLogs(*c.ctx, c.id, types.ContainerLogsOptions{ShowStdout: true, Follow: true})
if err != nil {
log.Debugf("failed to read container logs %s", err)
reader = ioutil.NopCloser(strings.NewReader(""))
}
_, err = io.Copy(log.Writer(), reader)
if err != nil {
log.Debugf("failed to write container logs to log output %s", err)
}
log.Debug("got EOF from container logs")
}
statusCh, errCh := c.dockerClient.ContainerWait(*c.ctx, c.id, container.WaitConditionNotRunning)
log.Debugf("waiting until command '%s' is finished...", realCmd)
select {
case err = <-errCh:
if err != nil {
return err
}
case retCode := <-statusCh:
if retCode.StatusCode != 0 {
logsCmd := fmt.Sprintf("docker logs %s", c.id)
return ErrRunContainerCommand{Cmd: logsCmd}
}
}
log.Debug("docker container is started")
return nil
}
// RunCommandOutput executes specified command in Docker container and
// returns command output as ReadCloser object. RunCommand debug option is
// set to false explicitly
func (c *DockerContainer) RunCommandOutput(
cmd []string,
containerInput io.Reader,
volumeMounts []string,
envVars []string,
) (io.ReadCloser, error) {
if err := c.RunCommand(cmd, containerInput, volumeMounts, envVars, false); err != nil {
return nil, err
}
return c.dockerClient.ContainerLogs(*c.ctx, c.id, types.ContainerLogsOptions{ShowStdout: true})
// GetContainerLogs returns logs from the container as io.ReadCloser
func (c *DockerContainer) GetContainerLogs() (io.ReadCloser, error) {
return c.dockerClient.ContainerLogs(*c.ctx, c.id, types.ContainerLogsOptions{ShowStdout: true, Follow: true})
}
// RmContainer kills and removes a container from the docker host.
@ -338,3 +297,21 @@ func (c *DockerContainer) RmContainer() error {
},
)
}
// WaitUntilFinished waits unit container command is finished, return an error if failed
func (c *DockerContainer) WaitUntilFinished() error {
statusCh, errCh := c.dockerClient.ContainerWait(*c.ctx, c.id, container.WaitConditionNotRunning)
log.Debugf("waiting until command is finished...")
select {
case err := <-errCh:
if err != nil {
return err
}
case retCode := <-statusCh:
if retCode.StatusCode != 0 {
logsCmd := fmt.Sprintf("docker logs %s", c.id)
return ErrRunContainerCommand{Cmd: logsCmd}
}
}
return nil
}

View File

@ -260,7 +260,7 @@ func TestImagePull(t *testing.T) {
func TestGetId(t *testing.T) {
cnt := getDockerContainerMock(mockDockerClient{})
err := cnt.RunCommand([]string{"testCmd"}, nil, nil, []string{}, false)
err := cnt.RunCommand([]string{"testCmd"}, nil, nil, []string{})
require.NoError(t, err)
actualID := cnt.GetID()
@ -278,7 +278,8 @@ func TestRunCommand(t *testing.T) {
volumeMounts []string
debug bool
mockDockerClient mockDockerClient
expectedErr error
expectedRunErr error
expectedWaitErr error
assertF func(*testing.T)
}{
{
@ -287,7 +288,8 @@ func TestRunCommand(t *testing.T) {
volumeMounts: nil,
debug: false,
mockDockerClient: mockDockerClient{},
expectedErr: nil,
expectedRunErr: nil,
expectedWaitErr: nil,
assertF: func(t *testing.T) {},
},
{
@ -300,8 +302,9 @@ func TestRunCommand(t *testing.T) {
return nil, imageListError
},
},
expectedErr: imageListError,
assertF: func(t *testing.T) {},
expectedRunErr: imageListError,
expectedWaitErr: nil,
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -316,8 +319,9 @@ func TestRunCommand(t *testing.T) {
return conn, nil
},
},
expectedErr: nil,
assertF: func(t *testing.T) {},
expectedRunErr: nil,
expectedWaitErr: nil,
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -329,8 +333,9 @@ func TestRunCommand(t *testing.T) {
return types.HijackedResponse{}, attachError
},
},
expectedErr: attachError,
assertF: func(t *testing.T) {},
expectedRunErr: attachError,
expectedWaitErr: nil,
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -342,8 +347,9 @@ func TestRunCommand(t *testing.T) {
return containerStartError
},
},
expectedErr: containerStartError,
assertF: func(t *testing.T) {},
expectedRunErr: containerStartError,
expectedWaitErr: nil,
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -359,8 +365,9 @@ func TestRunCommand(t *testing.T) {
return nil, errC
},
},
expectedErr: containerWaitError,
assertF: func(t *testing.T) {},
expectedRunErr: nil,
expectedWaitErr: containerWaitError,
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -376,8 +383,9 @@ func TestRunCommand(t *testing.T) {
return resC, nil
},
},
expectedErr: ErrRunContainerCommand{Cmd: "docker logs testID"},
assertF: func(t *testing.T) {},
expectedRunErr: nil,
expectedWaitErr: ErrRunContainerCommand{Cmd: "docker logs testID"},
assertF: func(t *testing.T) {},
},
{
cmd: []string{"testCmd"},
@ -396,15 +404,17 @@ func TestRunCommand(t *testing.T) {
return nil, fmt.Errorf("logs error")
},
},
expectedErr: ErrRunContainerCommand{Cmd: "docker logs testID"},
assertF: func(t *testing.T) {},
expectedRunErr: nil,
expectedWaitErr: ErrRunContainerCommand{Cmd: "docker logs testID"},
assertF: func(t *testing.T) {},
},
}
for _, tt := range tests {
cnt := getDockerContainerMock(tt.mockDockerClient)
actualErr := cnt.RunCommand(tt.cmd, tt.containerInput, tt.volumeMounts, []string{}, tt.debug)
assert.Equal(t, tt.expectedErr, actualErr)
actualErr := cnt.RunCommand(tt.cmd, tt.containerInput, tt.volumeMounts, []string{})
assert.Equal(t, tt.expectedRunErr, actualErr)
actualErr = cnt.WaitUntilFinished()
assert.Equal(t, tt.expectedWaitErr, actualErr)
tt.assertF(t)
}
@ -443,9 +453,10 @@ func TestRunCommandOutput(t *testing.T) {
}
for _, tt := range tests {
cnt := getDockerContainerMock(tt.mockDockerClient)
actualRes, actualErr := cnt.RunCommandOutput(tt.cmd, tt.containerInput, tt.volumeMounts, []string{})
actualErr := cnt.RunCommand(tt.cmd, tt.containerInput, tt.volumeMounts, []string{})
assert.Equal(t, tt.expectedErr, actualErr)
actualRes, actualErr := cnt.GetContainerLogs()
require.NoError(t, actualErr)
var actualResBytes []byte
if actualRes != nil {