Merge "Split document model, add entrypoints for repos"

cmd/document/pull_test.go

@@ -22,7 +22,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings {
 
 	fx := fixtures.Basic().One()
 
-	mfst.Repository = &config.Repository{
+	mfst.Repositories = map[string]*config.Repository{"primary": {
 		URLString: fx.DotGit().Root(),
 		CheckoutOptions: &config.RepoCheckout{
 			Branch:        "master",
@@ -31,6 +31,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings {
 		Auth: &config.RepoAuth{
 			Type: "http-basic",
 		},
+	},
 	}
 	settings.SetConfig(conf)
 	return settings

manifests/function/ephemeral/baremetal.yaml

@@ -2,7 +2,7 @@ apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   labels:
-    airshipit.org/ephemeral: "true"
+    airshipit.org/node-role: "control-plane"
   name: master-0
 spec:
   online: true

manifests/function/ephemeral/secret.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    airshipit.org/ephemeral: "true"
+    airshipit.org/node-role: "control-plane"
   name: node1-bmc-secret
 type: Opaque
 stringData:

manifests/site/test-bootstrap/ephemeral/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../type/test-bootstrap

manifests/site/test-bootstrap/kustomization.yaml

@@ -1,2 +0,0 @@
-resources:
-  - ../../type/test-bootstrap

pkg/bootstrap/cloudinit/cloud-init_test.go

@@ -8,12 +8,10 @@ import (
 	"sigs.k8s.io/kustomize/v3/pkg/types"
 
 	"opendev.org/airship/airshipctl/pkg/document"
-	"opendev.org/airship/airshipctl/testutil"
 )
 
 func TestGetCloudData(t *testing.T) {
-	fSys := testutil.SetupTestFs(t, "testdata")
-	bundle, err := document.NewBundle(fSys, "/", "/")
+	bundle, err := document.NewBundleByPath("testdata")
 	require.NoError(t, err, "Building Bundle Failed")
 
 	tests := []struct {

pkg/bootstrap/cloudinit/testdata/secret.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: node1-bmc-secret
+type: Opaque
+data:
+  netconfig: bmV0Y29uZmlnCg==
+stringData:
+  userdata: cloud-init
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "worker"
+  name: node1-bmc-secret1
+type: Opaque
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    test: nodataforcfg
+  name: node1-bmc-secret2
+type: Opaque
+data:
+  foo: bmV0Y29uZmlnCg==
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    some-data: "True"
+  name: node1-bmc-in-secret2
+type: Opaque
+data:
+  netconfig: bmV0Y29uZmlnCg==
+stringData:
+  userdata: cloud-init

pkg/bootstrap/isogen/command.go

@@ -34,19 +34,17 @@ func GenerateBootstrapIso(settings *environment.AirshipCTLSettings) error {
 		return err
 	}
 
-	var manifest *config.Manifest
-	manifest, err = globalConf.CurrentContextManifest()
-	if err != nil {
-		return err
-	}
-
 	if err = verifyInputs(cfg); err != nil {
 		return err
 	}
 
 	// TODO (dukov) replace with the appropriate function once it's available
 	// in document module
-	docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "")
+	root, err := globalConf.CurrentContextEntryPoint(config.Ephemeral, "")
+	if err != nil {
+		return err
+	}
+	docBundle, err := document.NewBundleByPath(root)
 	if err != nil {
 		return err
 	}

pkg/bootstrap/isogen/command_test.go

@@ -45,8 +45,7 @@ func (mc *mockContainer) GetID() string {
 }
 
 func TestBootstrapIso(t *testing.T) {
-	fSys := testutil.SetupTestFs(t, "testdata")
-	bundle, err := document.NewBundle(fSys, "/", "/")
+	bundle, err := document.NewBundleByPath("testdata/primary/site/test-site/ephemeral")
 	require.NoError(t, err, "Building Bundle Failed")
 
 	tempVol, cleanup := testutil.TempDir(t, "bootstrap-test")

pkg/cluster/initinfra/infra.go

@@ -39,7 +39,6 @@ func (infra *Infra) Run() error {
 // Deploy method deploys documents
 func (infra *Infra) Deploy() error {
 	kctl := infra.Client.Kubectl()
-	var err error
 	ao, err := kctl.ApplyOptions()
 	if err != nil {
 		return err
@@ -56,29 +55,23 @@ func (infra *Infra) Deploy() error {
 		return err
 	}
 
-	var manifest *config.Manifest
-	manifest, err = globalConf.CurrentContextManifest()
+	kustomizePath, err := globalConf.CurrentContextEntryPoint(infra.ClusterType, config.Initinfra)
 	if err != nil {
 		return err
 	}
 
-	b, err := document.NewBundle(infra.FileSystem, manifest.TargetPath, "")
+	b, err := document.NewBundleByPath(kustomizePath)
 	if err != nil {
 		return err
 	}
 
-	ls := document.EphemeralClusterSelector
-	selector := document.NewSelector().ByLabel(ls)
-
-	// Get documents that are annotated to belong to initinfra
-	docs, err := b.Select(selector)
+	// TODO (kkalynovskyi) Add Selector that would filter by label indicating wether to deploy it to k8s
+	docs, err := b.GetAllDocuments()
 	if err != nil {
 		return err
 	}
 	if len(docs) == 0 {
-		return document.ErrDocNotFound{
-			Selector: selector,
-		}
+		return document.ErrDocNotFound{}
 	}
 
 	// Label every document indicating that it was deployed by initinfra module for further reference

pkg/cluster/initinfra/infra_test.go

@@ -28,7 +28,7 @@ func (tc TestClient) Kubectl() kubectl.Interface      { return tc.MockKubectl()
 
 const (
 	kubeconfigPath    = "testdata/kubeconfig.yaml"
-	filenameRC        = "testdata/replicationcontroller.yaml"
+	filenameRC        = "testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml"
 	airshipConfigFile = "testdata/config.yaml"
 )
 

pkg/cluster/initinfra/testdata/config.yaml

@@ -13,20 +13,19 @@ current-context: dummy_cluster
 kind: Config
 manifests:
   dummy_manifest:
+    primary-repository-name: primary
     repositories:
-      dummy:
-        target-path: dummy_targetpath
-        url:
-          ForceQuery: false
-          Fragment: ""
-          Host: dummy.url.com
-          Opaque: ""
-          Path: ""
-          RawPath: ""
-          RawQuery: ""
-          Scheme: http
-          User: null
-        username: dummy_user
+      primary:
+        auth:
+          ssh-key: testdata/test-key.pem
+          type: ssh-key
+        checkout:
+          branch: ""
+          force: false
+          remote-ref: ""
+          tag: v1.0.1
+        url: http://dummy.url.com/primary.git
+    sub-path: primary/site/test-site
     target-path: testdata
 modules-config:
   bootstrapInfo:

pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml

@@ -4,9 +4,7 @@ metadata:
   name: test-rc
   namespace: test
   labels:
-    airshipit.org/ephemeral: "true"
-    name: test-rc
-    airship-component: "initinfra"
+    airshipit.org/initinfra: "true"
 spec:
   replicas: 1
   template:

pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - initinfra

pkg/config/config.go

@@ -21,15 +21,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 
-	"sigs.k8s.io/yaml"
-
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/yaml"
 
 	"opendev.org/airship/airshipctl/pkg/util"
 )
@@ -634,6 +634,28 @@ func (c *Config) CurrentContextManifest() (*Manifest, error) {
 	return c.Manifests[currentContext.Manifest], nil
 }
 
+// CurrentContextEntryPoint returns path to build bundle based on clusterType and phase
+// example CurrentContextEntryPoint("ephemeral", "initinfra")
+func (c *Config) CurrentContextEntryPoint(clusterType string, phase string) (string, error) {
+	err := ValidClusterType(clusterType)
+	if err != nil {
+		return "", err
+	}
+	ccm, err := c.CurrentContextManifest()
+	if err != nil {
+		return "", err
+	}
+	_, exists := ccm.Repositories[ccm.PrimaryRepositoryName]
+	if !exists {
+		return "", ErrMissingPrimaryRepo{}
+	}
+	return path.Join(
+		ccm.TargetPath,
+		ccm.SubPath,
+		clusterType,
+		phase), nil
+}
+
 // Credential or AuthInfo related methods
 func (c *Config) GetAuthInfo(aiName string) (*AuthInfo, error) {
 	authinfo, exists := c.AuthInfos[aiName]
@@ -845,9 +867,8 @@ func (m *Manifest) Equal(n *Manifest) bool {
 	if n == nil {
 		return n == m
 	}
-	repositoryEq := reflect.DeepEqual(m.Repository, n.Repository)
-	extraReposEq := reflect.DeepEqual(m.ExtraRepositories, n.ExtraRepositories)
-	return repositoryEq && extraReposEq && m.TargetPath == n.TargetPath
+	reposEq := reflect.DeepEqual(m.Repositories, n.Repositories)
+	return reposEq && m.TargetPath == n.TargetPath && m.SubPath == n.SubPath
 }
 
 func (m *Manifest) String() string {

pkg/config/constants.go

@@ -11,6 +11,11 @@ const (
 	AirshipClusterDefaultType = Target
 )
 
+// Constants related to Phases
+const (
+	Initinfra = "initinfra"
+)
+
 // Sorted
 var AllClusterTypes = [2]string{Ephemeral, Target}
 
@@ -58,3 +63,8 @@ const (
 	FlagUsername = "username"
 	FlagCurrent  = "current"
 )
+
+// Constants related to filesystem
+const (
+	SiteDirectory = "site"
+)

pkg/config/errors.go

@@ -93,3 +93,11 @@ type ErrMissingCurrentContext struct {
 func (e ErrMissingCurrentContext) Error() string {
 	return "Current context must be set before using --current flag"
 }
+
+// ErrMissingPrimaryRepo returned when Primary Repository is not set in context manifest
+type ErrMissingPrimaryRepo struct {
+}
+
+func (e ErrMissingPrimaryRepo) Error() string {
+	return "Current context manifest must have primary repository set"
+}

pkg/config/testdata/config-string.yaml

@@ -16,7 +16,9 @@ current-context: dummy_context
 kind: Config
 manifests:
   dummy_manifest:
-    repository:
+    primary-repository-name: primary
+    repositories:
+      primary:
         auth:
           ssh-key: testdata/test-key.pem
           type: ssh-key
@@ -25,7 +27,8 @@ manifests:
           force: false
           remote-ref: ""
           tag: v1.0.1
-      url: http://dummy.url.com
+        url: http://dummy.url.com/manifests.git
+    sub-path: manifests/site/test-site
     target-path: /var/tmp/
 modules-config:
   bootstrapInfo:

pkg/config/testdata/manifest-string.yaml

@@ -1,4 +1,6 @@
-repository:
+primary-repository-name: primary
+repositories:
+  primary:
     auth:
       ssh-key: testdata/test-key.pem
       type: ssh-key
@@ -7,5 +9,6 @@ repository:
       force: false
       remote-ref: ""
       tag: v1.0.1
-  url: http://dummy.url.com
+    url: http://dummy.url.com/manifests.git
+sub-path: manifests/site/test-site
 target-path: /var/tmp/

pkg/config/testdata/repository-string.yaml

@@ -6,4 +6,4 @@ checkout:
   force: false
   remote-ref: ""
   tag: v1.0.1
-url: http://dummy.url.com
+url: http://dummy.url.com/manifests.git

pkg/config/types.go

@@ -113,15 +113,23 @@ type AuthInfo struct {
 	authInfo *kubeconfig.AuthInfo
 }
 
-// Manifests is a tuple of references to a Manifest (how do Identify, collect ,
+// Manifest is a tuple of references to a Manifest (how do Identify, collect ,
 // find the yaml manifests that airship uses to perform its operations)
 type Manifest struct {
-	// Repositories is the map of repository adddressable by a name
-	Repository *Repository `json:"repository"`
+	// PrimaryRepositoryName is a name of the repo, that contains site/<site-name> directory
+	// and is a starting point for building document bundle
+	PrimaryRepositoryName string `json:"primary-repository-name"`
 	// ExtraRepositories is the map of extra repositories addressable by a name
-	ExtraRepositories map[string]*Repository `json:"extra-repositories,omitempty"`
+	Repositories map[string]*Repository `json:"repositories,omitempty"`
 	// TargetPath Local Target path for working or home dirctory for all Manifest Cloned/Returned/Generated
 	TargetPath string `json:"target-path"`
+	// SubPath is a path relative to TargetPath + Path where PrimaryRepository is cloned and contains
+	// directories with ClusterType and Phase bundles, example:
+	// Repositories[PrimaryRepositoryName].Url = 'https://github.com/airshipit/treasuremap'
+	// SubPath = "manifests"
+	// you would expect that at treasuremap/manifests you would have ephemeral/initinfra and
+	// ephemera/target directories, containing kustomize.yaml.
+	SubPath string `json:"sub-path"`
 }
 
 // Repository is a tuple that holds the information for the remote sources of manifest yaml documents.

pkg/config/utils.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package config
 
+const (
+	DefaultTestPrimaryRepo = "primary"
+)
+
 // NewConfig returns a newly initialized Config object
 func NewConfig() *Config {
 	return &Config{
@@ -30,7 +34,8 @@ func NewConfig() *Config {
 		},
 		Manifests: map[string]*Manifest{
 			AirshipDefaultManifest: {
-				Repository: &Repository{
+				Repositories: map[string]*Repository{
+					DefaultTestPrimaryRepo: {
 						URLString: AirshipDefaultManifestRepoLocation,
 						CheckoutOptions: &RepoCheckout{
 							CommitHash: "master",
@@ -38,7 +43,10 @@ func NewConfig() *Config {
 							RemoteRef:  "master",
 						},
 					},
+				},
 				TargetPath:            "/tmp/" + AirshipDefaultManifest,
+				PrimaryRepositoryName: DefaultTestPrimaryRepo,
+				SubPath:               AirshipDefaultManifestRepo + "/manifests/site",
 			},
 		},
 		ModulesConfig: &Modules{
@@ -78,8 +86,8 @@ func NewCluster() *Cluster {
 // object with non-nil maps
 func NewManifest() *Manifest {
 	return &Manifest{
-		Repository:        NewRepository(),
-		ExtraRepositories: make(map[string]*Repository),
+		PrimaryRepositoryName: DefaultTestPrimaryRepo,
+		Repositories:          map[string]*Repository{DefaultTestPrimaryRepo: NewRepository()},
 	}
 }
 

pkg/document/bundle.go

@@ -57,6 +57,12 @@ type Bundle interface {
 	GetAllDocuments() ([]Document, error)
 }
 
+// NewBundleByPath helper function that returns new document.Bundle interface based on clusterType and
+// phase, example: helpers.NewBunde(airConfig, "ephemeral", "initinfra")
+func NewBundleByPath(rootPath string) (Bundle, error) {
+	return NewBundle(NewDocumentFs(), rootPath, "")
+}
+
 // NewBundle is a convenience function to create a new bundle
 // Over time, it will evolve to support allowing more control
 // for kustomize plugins

pkg/document/constants.go

@@ -3,7 +3,7 @@ package document
 const (
 	// Selectors
 	BaseAirshipSelector = "airshipit.org"
-	EphemeralClusterSelector = BaseAirshipSelector + "/ephemeral in (True, true)"
+	ControlNodeSelector = BaseAirshipSelector + "/node-role=control-plane"
 
 	// Labels
 	DeployedByLabel = BaseAirshipSelector + "/deployed"

pkg/document/pull/pull.go

@@ -25,19 +25,8 @@ func (s *Settings) cloneRepositories() error {
 		return err
 	}
 
-	mainRepoConfig := currentManifest.Repository
-	repository, err := repo.NewRepository(currentManifest.TargetPath, mainRepoConfig)
-	if err != nil {
-		return err
-	}
-	err = repository.Download(mainRepoConfig.ToCheckoutOptions(true).Force)
-	if err != nil {
-		return err
-	}
-	repository.Driver.Close()
-
-	// Clone extra repositories
-	for _, extraRepoConfig := range currentManifest.ExtraRepositories {
+	// Clone repositories
+	for _, extraRepoConfig := range currentManifest.Repositories {
 		repository, err := repo.NewRepository(currentManifest.TargetPath, extraRepoConfig)
 		if err != nil {
 			return err

pkg/document/pull/pull_test.go

@@ -3,7 +3,6 @@ package pull
 import (
 	"io/ioutil"
 	"path"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -16,6 +15,7 @@ import (
 
 	"opendev.org/airship/airshipctl/pkg/config"
 	"opendev.org/airship/airshipctl/pkg/environment"
+	"opendev.org/airship/airshipctl/pkg/util"
 	"opendev.org/airship/airshipctl/testutil"
 )
 
@@ -42,7 +42,7 @@ func TestPull(t *testing.T) {
 		fx := fixtures.Basic().One()
 
 		dummyGitDir := fx.DotGit().Root()
-		currentManifest.Repository = &config.Repository{
+		currentManifest.Repositories = map[string]*config.Repository{currentManifest.PrimaryRepositoryName: {
 			URLString: dummyGitDir,
 			CheckoutOptions: &config.RepoCheckout{
 				Branch:        "master",
@@ -51,6 +51,7 @@ func TestPull(t *testing.T) {
 			Auth: &config.RepoAuth{
 				Type: "http-basic",
 			},
+		},
 		}
 
 		tmpDir, cleanup := testutil.TempDir(t, "airshipctlPullTest-")
@@ -58,13 +59,13 @@ func TestPull(t *testing.T) {
 
 		currentManifest.TargetPath = tmpDir
 
-		_, err = repo2.NewRepository(".", currentManifest.Repository)
+		_, err = repo2.NewRepository(".", currentManifest.Repositories[currentManifest.PrimaryRepositoryName])
 		require.NoError(err)
 
 		err = dummyPullSettings.cloneRepositories()
 
 		require.NoError(err)
-		dummyRepoDirName := filepath.Base(dummyGitDir)
+		dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir)
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go"))
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
 		contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
@@ -82,7 +83,8 @@ func TestPull(t *testing.T) {
 
 		mfst := conf.Manifests["dummy_manifest"]
 		dummyGitDir := fx.DotGit().Root()
-		mfst.Repository = &config.Repository{
+		mfst.Repositories = map[string]*config.Repository{
+			mfst.PrimaryRepositoryName: {
 				URLString: dummyGitDir,
 				CheckoutOptions: &config.RepoCheckout{
 					Branch:        "master",
@@ -91,6 +93,7 @@ func TestPull(t *testing.T) {
 				Auth: &config.RepoAuth{
 					Type: "http-basic",
 				},
+			},
 		}
 		dummyPullSettings.SetConfig(conf)
 
@@ -103,7 +106,7 @@ func TestPull(t *testing.T) {
 		err = dummyPullSettings.Pull()
 		require.NoError(err)
 
-		dummyRepoDirName := filepath.Base(dummyGitDir)
+		dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir)
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go"))
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
 		contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))

pkg/document/repo/adapter.go

@@ -28,7 +28,7 @@ type GitDriver struct {
 	Storer     storage.Storer
 }
 
-func NewGitDriver(fs billy.Filesystem, s storage.Storer) *GitDriver {
+func NewGitDriver(fs billy.Filesystem, s storage.Storer) Adapter {
 	return &GitDriver{Storer: s, Filesystem: fs}
 }
 

pkg/document/repo/repo.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
-	"strings"
 
 	"gopkg.in/src-d/go-billy.v4"
 	"gopkg.in/src-d/go-billy.v4/osfs"
@@ -15,6 +14,7 @@ import (
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 
 	"opendev.org/airship/airshipctl/pkg/log"
+	"opendev.org/airship/airshipctl/pkg/util"
 )
 
 var (
@@ -41,7 +41,7 @@ type Repository struct {
 // NewRepository create repository object, with real filesystem on disk
 // basePath is used to calculate final path where to clone/open the repository
 func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error) {
-	dirName := nameFromURL(builder.URL())
+	dirName := util.GitDirNameFromURL(builder.URL())
 	if dirName == "" {
 		return nil, fmt.Errorf("URL: %s, original error: %w", builder.URL(), ErrCantParseURL)
 	}
@@ -60,11 +60,6 @@ func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error)
 	}, nil
 }
 
-func nameFromURL(urlString string) string {
-	_, fileName := filepath.Split(urlString)
-	return strings.TrimSuffix(fileName, ".git")
-}
-
 func storerFromFs(fs billy.Filesystem) (storage.Storer, error) {
 	dot, err := fs.Chroot(".git")
 	if err != nil {

pkg/document/repo/repo_test.go

@@ -38,19 +38,6 @@ func (md mockBuilder) ToFetchOptions(transport.AuthMethod) *git.FetchOptions {
 }
 func (md mockBuilder) URL() string { return md.URLString }
 
-func TestNewRepositoryNegative(t *testing.T) {
-	err := fixtures.Init()
-	require.NoError(t, err)
-	defer testutil.CleanUpGitFixtures(t)
-
-	builder := &mockBuilder{
-		URLString: "",
-	}
-	repo, err := NewRepository(".", builder)
-	assert.Error(t, err)
-	assert.Nil(t, repo)
-}
-
 func TestDownload(t *testing.T) {
 	err := fixtures.Init()
 	require.NoError(t, err)
@@ -216,47 +203,3 @@ func TestCheckout(t *testing.T) {
 	err = repo.Checkout(true)
 	assert.Error(t, err)
 }
-
-func TestURLtoName(t *testing.T) {
-	tests := []struct {
-		input          string
-		expectedOutput string
-	}{
-		{
-			input:          "https://github.com/kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "git@github.com:kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "https://github.com/kubernetes/kube.somepath.ctl.git",
-			expectedOutput: "kube.somepath.ctl",
-		},
-		{
-			input:          "https://github.com/kubernetes/kubectl",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "git@github.com:kubernetes/kubectl",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "github.com:kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "/kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "/kubernetes/kubectl.git/",
-			expectedOutput: "",
-		},
-	}
-
-	for _, test := range tests {
-		assert.Equal(t, test.expectedOutput, nameFromURL(test.input))
-	}
-}

pkg/remote/remote_direct.go

@@ -63,10 +63,6 @@ func getRemoteDirectClient(remoteConfig *config.RemoteDirect, remoteURL string)
 
 func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.RemoteDirect, string, error) {
 	cfg := settings.Config()
-	manifest, err := cfg.CurrentContextManifest()
-	if err != nil {
-		return nil, "", err
-	}
 	bootstrapSettings, err := cfg.CurrentContextBootstrapInfo()
 	if err != nil {
 		return nil, "", err
@@ -77,14 +73,17 @@ func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.Re
 		return nil, "", config.ErrMissingConfig{What: "RemoteDirect options not defined in bootstrap config"}
 	}
 
-	// TODO (dukov) replace with the appropriate function once it's available
-	// in document module
-	docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "")
+	root, err := cfg.CurrentContextEntryPoint(config.Ephemeral, "")
 	if err != nil {
 		return nil, "", err
 	}
 
-	ls := document.EphemeralClusterSelector
+	docBundle, err := document.NewBundleByPath(root)
+	if err != nil {
+		return nil, "", err
+	}
+
+	ls := document.ControlNodeSelector
 	selector := document.NewSelector().
 		ByGvk("", "", AirshipHostKind).
 		ByLabel(ls)

pkg/remote/remote_direct_test.go

@@ -14,6 +14,7 @@ import (
 )
 
 func initSettings(t *testing.T, rd *config.RemoteDirect, testdata string) *environment.AirshipCTLSettings {
+	t.Helper()
 	settings := &environment.AirshipCTLSettings{}
 	settings.SetConfig(testutil.DummyConfig())
 	bi, err := settings.Config().CurrentContextBootstrapInfo()
@@ -36,7 +37,6 @@ func TestUnknownRemoteType(t *testing.T) {
 	)
 
 	err := DoRemoteDirect(s)
-
 	_, ok := err.(*GenericError)
 	assert.True(t, ok)
 }
@@ -52,7 +52,6 @@ func TestRedfishRemoteDirectWithEmptyURL(t *testing.T) {
 	)
 
 	err := DoRemoteDirect(s)
-
 	_, ok := err.(redfish.ErrRedfishMissingConfig)
 	assert.True(t, ok)
 }

pkg/remote/testdata/base/baremetal.yaml

@@ -1,49 +0,0 @@
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0
-spec:
-  online: true
-  bootMACAddress: 00:3b:8b:0c:ec:8b
-  bmc:
-    address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node
-    credentialsName: master-0-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1
-spec:
-  online: "true"
-  bootMACAddress: 01:3b:8b:0c:ec:8b
-  bmc:
-    address: ipmi://192.168.111.2:6230
-    credentialsName: master-1-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
-...

pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node
+    credentialsName: master-0-bmc-secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0-bmc-secret
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: cGFzc3dvcmQ=
+...

pkg/remote/testdata/emptyurl/baremetal.yaml

@@ -1,49 +0,0 @@
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0
-spec:
-  online: true
-  bootMACAddress: 00:3b:8b:0c:ec:8b
-  bmc:
-    address: ""
-    credentialsName: master-0-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1
-spec:
-  online: true
-  bootMACAddress: 01:3b:8b:0c:ec:8b
-  bmc:
-    address: ipmi://192.168.111.2:6230
-    credentialsName: master-1-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
-...

pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: ""
+    credentialsName: master-0-bmc-secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0-bmc-secret
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: cGFzc3dvcmQ=
+...

pkg/util/url.go

@@ -0,0 +1,12 @@
+package util
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// GitDirNameFromURL extract directory name of the repository from URL
+func GitDirNameFromURL(urlString string) string {
+	_, fileName := filepath.Split(urlString)
+	return strings.TrimSuffix(fileName, ".git")
+}

pkg/util/url_test.go

@@ -0,0 +1,51 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitDirNameFromURL(t *testing.T) {
+	tests := []struct {
+		input          string
+		expectedOutput string
+	}{
+		{
+			input:          "https://github.com/kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "git@github.com:kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "https://github.com/kubernetes/kube.somepath.ctl.git",
+			expectedOutput: "kube.somepath.ctl",
+		},
+		{
+			input:          "https://github.com/kubernetes/kubectl",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "git@github.com:kubernetes/kubectl",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "github.com:kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "/kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "/kubernetes/kubectl.git/",
+			expectedOutput: "",
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.expectedOutput, GitDirNameFromURL(test.input))
+	}
+}

playbooks/vars/test-config.yaml

@@ -1,7 +1,6 @@
 airship_config_action: generate
-airship_config_site_name: "test-bootstrap"
 airship_config_iso_gen_target_path: "{{ serve_dir }}"
-airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}/manifests/site/{{ airship_config_site_name }}"
+airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}"
 airship_config_ephemeral_ip: "{{ airship_gate_ipam.nat_network.ephemeral_ip }}"
 airship_config_iso_builder_docker_image: "kkalynovskyi/image-builder:latest"
 airship_config_ca_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

roles/airshipctl-test-configs/defaults/main.yaml

@@ -1,8 +1,6 @@
 airship_config_iso_gen_target_path: /srv/iso
 airship_config_manifest_directory: /tmp/airship
 airship_config_primary_repo_url: dummy.url.com
-airship_config_primary_repo_auth_type: ssh-key
-airship_config_primary_repo_key_path: ""
 airship_config_ephemeral_ip: "10.23.25.101"
 airship_config_iso_builder_docker_image: quay.io/airshipit/isogen:latest
 airship_config_iso_port: 8099

roles/airshipctl-test-configs/templates/airshipconfig.j2

@@ -13,16 +13,19 @@ current-context: dummy_cluster
 kind: Config
 manifests:
   dummy_manifest:
-    repository:
-      auth:
-        ssh-key: {{ airship_config_primary_repo_key_path | default("") }}
-        type: {{ airship_config_primary_repo_auth_type }}
+    primary-repository: primary
+    repositories:
+      primary:
         checkout:
           branch: "master"
           force: false
           remote-ref: ""
           tag: ""
         url: {{ airship_config_primary_repo_url }}
+    ## this is temporary hack, as soon as we use `document pull` command in gate process
+    ## this will subpath will be airshipctl/manifests/site/test-bootstrap, as airshipctl
+    ## will be primary repository
+    sub-path: "manifests/site/test-bootstrap"
     target-path: {{ airship_config_manifest_directory }}
 modules-config:
   bootstrapInfo:

testutil/testconfig.go

@@ -90,15 +90,17 @@ func DummyCluster() *config.Cluster {
 func DummyManifest() *config.Manifest {
 	m := config.NewManifest()
 	// Repositories is the map of repository adddressable by a name
-	m.Repository = DummyRepository()
+	m.Repositories = map[string]*config.Repository{"primary": DummyRepository()}
+	m.PrimaryRepositoryName = "primary"
 	m.TargetPath = "/var/tmp/"
+	m.SubPath = "manifests/site/test-site"
 	return m
 }
 
 // DummyRepository, utility function used for tests
 func DummyRepository() *config.Repository {
 	return &config.Repository{
-		URLString: "http://dummy.url.com",
+		URLString: "http://dummy.url.com/manifests.git",
 		CheckoutOptions: &config.RepoCheckout{
 			Tag:           "v1.0.1",
 			ForceCheckout: false,