@ -15,17 +15,12 @@
package isogen
import (
"bufio"
"fmt"
"io"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"github.com/cheggaaa/pb/v3"
"opendev.org/airship/airshipctl/pkg/api/v1alpha1"
"opendev.org/airship/airshipctl/pkg/bootstrap/cloudinit"
"opendev.org/airship/airshipctl/pkg/config"
@ -36,19 +31,11 @@ import (
)
const (
builderConfigFileName = "builder-conf.yaml"
// progressBarTemplate is a template string for progress bar
// looks like 'Prefix [-->______] 20%' where Prefix is trimmed log line from docker container
progressBarTemplate = ` {{ string . "prefix" }} {{ bar . }} {{ percent . }} `
// defaultTerminalWidth is a default width of terminal if it's impossible to determine the actual one
defaultTerminalWidth = 80
// multiplier is a number of log lines produces while installing 1 package
multiplier = 3
// reInstallActions is a regular expression to check whether the log line contains of this substrings
reInstallActions = ` Extracting|Unpacking|Configuring|Preparing|Setting `
reInstallBegin = ` Retrieving Packages|newly installed `
reInstallFinish = ` Base system installed successfully|mksquashfs `
builderConfigFileName = "builder-conf.yaml"
outputFileNameDefault = "ephemerial.iso"
userDataFileName = "user-data"
networkConfigFileName = "network-data"
outputMetadataFileName = "output-metadata.yaml"
)
// BootstrapIsoOptions are used to generate bootstrap ISO
@ -56,60 +43,63 @@ type BootstrapIsoOptions struct {
DocBundle document . Bundle
Builder container . Container
Doc document . Document
Cfg * v1alpha1 . I mage Configuration
Cfg * v1alpha1 . I so Configuration
// optional fields for verbose output
Debug bool
Progress bool
Writer io . Writer
Writer io . Writer
}
// VerifyInputs verifies image configuration
func VerifyInputs ( cfg * v1alpha1 . ImageConfiguration ) error {
if cfg . Container . Volume == "" {
func VerifyInputs ( cfg * v1alpha1 . IsoConfiguration ) error {
if cfg . IsoContainer . Volume == "" {
return config . ErrMissingConfig {
What : "Must specify volume bind for ISO builder container" ,
}
}
if ( cfg . Builder . UserDataFileName == "" ) || ( cfg . Builder . NetworkConfigFileName == "" ) {
return config . ErrMissingConfig {
What : "UserDataFileName or NetworkConfigFileName are not specified in ISO builder config" ,
}
}
vols := strings . Split ( cfg . Container . Volume , ":" )
vols := strings . Split ( cfg . IsoContainer . Volume , ":" )
switch {
case len ( vols ) == 1 :
cfg . . Volume = fmt . Sprintf ( "%s:%s" , vols [ 0 ] , vols [ 0 ] )
cfg . IsoContainer . Volume = fmt . Sprintf ( "%s:%s" , vols [ 0 ] , vols [ 0 ] )
case len ( vols ) > 2 :
return config . ErrInvalidConfig {
What : "Bad container volume format. Use hostPath:contPath" ,
}
}
if cfg . Isogen . OutputFileName == "" {
log . Debugf ( "No outputFileName provided to Isogen. Using default: %s" , outputFileNameDefault )
cfg . Isogen . OutputFileName = outputFileNameDefault
}
return nil
}
func getContainerCfg (
cfg * v1alpha1 . ImageConfiguration ,
func get Iso ContainerCfg(
cfg * v1alpha1 . I so Configuration,
builderCfgYaml [ ] byte ,
userData [ ] byte ,
netConf [ ] byte ,
) map [ string ] [ ] byte {
hostVol := strings . Split ( cfg . . Volume , ":" ) [ 0 ]
hostVol := strings . Split ( cfg . Iso Container. Volume , ":" ) [ 0 ]
fls := make ( map [ string ] [ ] byte )
fls [ filepath . Join ( hostVol , cfg. B uilder. U serDataFileName) ] = userData
fls [ filepath . Join ( hostVol , cfg. Builder . N etworkConfigFileName) ] = netConf
fls [ filepath . Join ( hostVol , ) ] = userData
fls [ filepath . Join ( hostVol , n etworkConfigFileName) ] = netConf
fls [ filepath . Join ( hostVol , builderConfigFileName ) ] = builderCfgYaml
return fls
}
// CreateBootstrapIso prepares and runs appropriate container to create a bootstrap ISO
func ( opts BootstrapIsoOptions ) CreateBootstrapIso ( ) error {
cntVol := strings . Split ( opts . Cfg . . Volume , ":" ) [ 1 ]
cntVol := strings . Split ( opts . Cfg . Iso Container. Volume , ":" ) [ 1 ]
log . Print ( "Creating cloud-init for ephemeral K8s" )
userData , netConf , err := cloudinit . GetCloudData ( opts . DocBundle )
userData , netConf , err := cloudinit . GetCloudData (
opts . DocBundle ,
opts . Cfg . Isogen . UserDataSelector ,
opts . Cfg . Isogen . UserDataKey ,
opts . Cfg . Isogen . NetworkConfigSelector ,
opts . Cfg . Isogen . NetworkConfigKey ,
)
if err != nil {
return err
}
@ -119,21 +109,27 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error {
return err
}
fls := get ContainerCfg( opts . Cfg , builderCfgYaml , userData , netConf )
fls := get Iso ContainerCfg( opts . Cfg , builderCfgYaml , userData , netConf )
if err = util . WriteFiles ( fls , 0600 ) ; err != nil {
return err
}
vols := [ ] string { opts . Cfg . . Volume }
vols := [ ] string { opts . Cfg . Iso Container. Volume }
builderCfgLocation := filepath . Join ( cntVol , builderConfigFileName )
log . Printf ( "Running default container command. Mounted dir: %s" , vols )
envVars := [ ] string {
fmt . Sprintf ( "IMAGE_TYPE=iso" ) ,
fmt . Sprintf ( "BUILDER_CONFIG=%s" , builderCfgLocation ) ,
fmt . Sprintf ( "USER_DATA_FILE=%s" , userDataFileName ) ,
fmt . Sprintf ( "NET_CONFIG_FILE=%s" , networkConfigFileName ) ,
fmt . Sprintf ( "OUTPUT_FILE_NAME=%s" , opts . Cfg . Isogen . OutputFileName ) ,
fmt . Sprintf ( "OUTPUT_METADATA_FILE_NAME=%s" , outputMetadataFileName ) ,
fmt . Sprintf ( "http_proxy=%s" , os . Getenv ( "http_proxy" ) ) ,
fmt . Sprintf ( "https_proxy=%s" , os . Getenv ( "https_proxy" ) ) ,
fmt . Sprintf ( "HTTP_PROXY=%s" , os . Getenv ( "HTTP_PROXY" ) ) ,
fmt . Sprintf ( "HTTPS_PROXY=%s" , os . Getenv ( "HTTPS_PROXY" ) ) ,
fmt . Sprintf ( "no_proxy=%s" , os . Getenv ( "no_proxy" ) ) ,
fmt . Sprintf ( "NO_PROXY=%s" , os . Getenv ( "NO_PROXY" ) ) ,
}
@ -144,25 +140,17 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error {
log . Print ( "ISO generation is in progress. The whole process could take up to several minutes, please wait..." )
if opts. Debug || opts . Progress {
if log. DebugEnabled ( ) {
var cLogs io . ReadCloser
cLogs , err = opts . Builder . GetContainerLogs ( )
if err != nil {
log . Printf ( "failed to read container logs %s" , err )
} else {
switch {
case opts . Progress :
if err = ShowProgress ( cLogs , opts . Writer ) ; err != nil {
log . Debugf ( "the following error occurred while showing progress bar: %s" , err . Error ( ) )
}
case opts . Debug :
log . Print ( "start reading container logs" )
// either container log output or progress bar will be shown
if _ , err = io . Copy ( opts . Writer , cLogs ) ; err != nil {
log . Debugf ( "failed to write container logs to log output %s" , err )
}
log . Print ( "got EOF from container logs" )
log . Print ( "start reading container logs" )
if _ , err = io . Copy ( opts . Writer , cLogs ) ; err != nil {
log . Debugf ( "failed to write container logs to log output %s" , err )
}
log . Print ( "got EOF from container logs" )
}
}
@ -171,7 +159,7 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error {
}
log . Print ( "ISO successfully built." )
if ! opts. Debug {
if ! log. DebugEnabled ( ) {
log . Print ( "Removing container." )
return opts . Builder . RmContainer ( )
}
@ -179,140 +167,3 @@ func (opts BootstrapIsoOptions) CreateBootstrapIso() error {
log . Debugf ( "Debug flag is set. Container %s stopped but not deleted." , opts . Builder . GetID ( ) )
return nil
}
// ShowProgress prints progress bar during bootstrap ISO preparation
func ShowProgress ( reader io . ReadCloser , writer io . Writer ) error {
reFindActions := regexp . MustCompile ( reInstallActions )
reBeginInstall := regexp . MustCompile ( reInstallBegin )
reFinishInstall := regexp . MustCompile ( reInstallFinish )
var bar * pb . ProgressBar
scanner := bufio . NewScanner ( reader )
scanner . Split ( bufio . ScanLines )
// Reading container log line by line
for scanner . Scan ( ) {
curLine := scanner . Text ( )
// Trying to find entry points of package installation
switch {
case reBeginInstall . MatchString ( curLine ) :
if err := finalizePb ( bar , nil ) ; err != nil {
return err
}
pkgCount , err := calculatePkgCount ( scanner , writer , curLine )
if err != nil {
return finalizePb ( bar , err )
}
bar , err = initPb ( pkgCount , writer )
if err != nil {
return err
}
case reFinishInstall . MatchString ( curLine ) :
if err := finalizePb ( bar , nil ) ; err != nil {
return err
}
case reFindActions . MatchString ( curLine ) :
if err := incrementPb ( bar , curLine ) ; err != nil {
return finalizePb ( bar , err )
}
case strings . Contains ( curLine , "filesystem.squashfs" ) :
fmt . Fprintln ( writer , curLine )
}
}
if bar != nil && bar . IsStarted ( ) {
return finalizePb ( bar , ErrUnexpectedPb { } )
}
return nil
}
func finalizePb ( bar * pb . ProgressBar , e error ) error {
if bar != nil && bar . IsStarted ( ) {
bar . SetCurrent ( bar . Total ( ) )
if e != nil {
setPbPrefix ( bar , "An error occurred while log parsing" )
bar . Finish ( )
return e
}
setPbPrefix ( bar , "Completed" )
bar . Finish ( )
if err := bar . Err ( ) ; err != nil {
return err
}
}
return e
}
func initPb ( pkgCount int , w io . Writer ) ( * pb . ProgressBar , error ) {
bar := pb . ProgressBarTemplate ( progressBarTemplate ) . New ( pkgCount * multiplier )
bar . SetWriter ( w ) . Start ( )
setPbPrefix ( bar , "Installing required packages" )
if err := bar . Err ( ) ; err != nil {
return nil , finalizePb ( bar , err )
}
return bar , nil
}
func incrementPb ( bar * pb . ProgressBar , curLine string ) error {
if bar != nil && bar . IsStarted ( ) && bar . Current ( ) < bar . Total ( ) {
setPbPrefix ( bar , curLine )
bar . Increment ( )
if err := bar . Err ( ) ; err != nil {
return finalizePb ( bar , err )
}
}
return nil
}
func setPbPrefix ( bar * pb . ProgressBar , msg string ) {
terminalWidth := defaultTerminalWidth
halfWidth := terminalWidth / 2
bar . SetWidth ( terminalWidth )
if len ( msg ) > halfWidth {
msg = fmt . Sprintf ( "%v..." , msg [ 0 : halfWidth - 3 ] )
} else {
msg = fmt . Sprintf ( "%-*v" , halfWidth , msg )
}
bar . Set ( "prefix" , msg )
}
func calculatePkgCount ( scanner * bufio . Scanner , writer io . Writer , curLine string ) ( int , error ) {
reFindNumbers := regexp . MustCompile ( "[0-9]+" )
// Trying to count how many packages is going to be installed
pkgCount := 0
matches := reFindNumbers . FindAllString ( curLine , - 1 )
if matches == nil {
// There is no numbers in line about base packages, counting them manually to get estimates
fmt . Fprint ( writer , "Retrieving base packages " )
for scanner . Scan ( ) {
curLine = scanner . Text ( )
if strings . Contains ( curLine , "Retrieving" ) {
pkgCount ++
fmt . Fprint ( writer , "." )
}
if strings . Contains ( curLine , "Chosen extractor" ) {
fmt . Fprintln ( writer , " Done" )
return pkgCount , nil
}
}
}
if len ( matches ) >= 2 {
for _ , v := range matches [ 0 : 2 ] {
j , err := strconv . Atoi ( v )
if err != nil {
continue
}
pkgCount += j
}
if pkgCount > 0 {
return pkgCount , nil
}
}
return pkgCount , ErrNoParsedNumPkgs { }
}
Anderson, Craig (ca846m)
3 years ago