Split document model, add entrypoints for repos

Add NewBundleByPath function, that would return bundle built from
the specified path argument

Add CurrentContextEntryPoint method of the config
object, that would allow easily get kustomize root path based on
clusterType and phase. You can also leave phase arg empty string,
which would try to return bundle for all phases

Introduce changes to config pakage objects:

- Manifest:
  SubPath: this is relative path to the root of the repository that
contains directories with sites (SiteNames)
    PrimaryRepositoryName: this is a string that must correspond to a key
of the Repositories map of manifest object, which is used to derive
primary repository
    Repositories object is a map, map keys correspond to names of the
directories where `document pull` command will download repositories
defined in manifest prepended by manifest.TargetPath.

Introduce new config method CurrentContextEntryPoint(), method takes
TargetPath, PrimaryRepo.URL, SubPath, and clusterType and phase
constructs a path to the entry point out of which the DocumentBundle
should be build, and returns it to the caller. After that caller can
build a bundle out of it, the bundle will contain documents relevant to
particular cluster-type and phase.

All objects that depend on bundle interface are updated to use the
CurrentContextEntryPoint() method of the config object

Relates-To: #99

Closes: #99

Change-Id: I99320c4cb626841d46f4c298b583e9af90b1dce4

cmd/document/pull_test.go

@@ -22,7 +22,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings {
 
 	fx := fixtures.Basic().One()
 
-	mfst.Repository = &config.Repository{
+	mfst.Repositories = map[string]*config.Repository{"primary": {
 		URLString: fx.DotGit().Root(),
 		CheckoutOptions: &config.RepoCheckout{
 			Branch:        "master",
@@ -31,6 +31,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings {
 		Auth: &config.RepoAuth{
 			Type: "http-basic",
 		},
+	},
 	}
 	settings.SetConfig(conf)
 	return settings

manifests/function/ephemeral/baremetal.yaml

@@ -2,7 +2,7 @@ apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   labels:
-    airshipit.org/ephemeral: "true"
+    airshipit.org/node-role: "control-plane"
   name: master-0
 spec:
   online: true

manifests/function/ephemeral/secret.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    airshipit.org/ephemeral: "true"
+    airshipit.org/node-role: "control-plane"
   name: node1-bmc-secret
 type: Opaque
 stringData:

manifests/site/test-bootstrap/ephemeral/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - ../../../type/test-bootstrap

manifests/site/test-bootstrap/kustomization.yaml

@@ -1,2 +0,0 @@
-resources:
-  - ../../type/test-bootstrap

pkg/bootstrap/cloudinit/cloud-init_test.go

@@ -8,12 +8,10 @@ import (
 	"sigs.k8s.io/kustomize/v3/pkg/types"
 
 	"opendev.org/airship/airshipctl/pkg/document"
-	"opendev.org/airship/airshipctl/testutil"
 )
 
 func TestGetCloudData(t *testing.T) {
-	fSys := testutil.SetupTestFs(t, "testdata")
-	bundle, err := document.NewBundle(fSys, "/", "/")
+	bundle, err := document.NewBundleByPath("testdata")
 	require.NoError(t, err, "Building Bundle Failed")
 
 	tests := []struct {

pkg/bootstrap/cloudinit/testdata/secret.yaml

@@ -0,0 +1,41 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: node1-bmc-secret
+type: Opaque
+data:
+  netconfig: bmV0Y29uZmlnCg==
+stringData:
+  userdata: cloud-init
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "worker"
+  name: node1-bmc-secret1
+type: Opaque
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    test: nodataforcfg
+  name: node1-bmc-secret2
+type: Opaque
+data:
+  foo: bmV0Y29uZmlnCg==
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    some-data: "True"
+  name: node1-bmc-in-secret2
+type: Opaque
+data:
+  netconfig: bmV0Y29uZmlnCg==
+stringData:
+  userdata: cloud-init

pkg/bootstrap/isogen/command.go

@@ -34,19 +34,17 @@ func GenerateBootstrapIso(settings *environment.AirshipCTLSettings) error {
 		return err
 	}
 
-	var manifest *config.Manifest
-	manifest, err = globalConf.CurrentContextManifest()
-	if err != nil {
-		return err
-	}
-
 	if err = verifyInputs(cfg); err != nil {
 		return err
 	}
 
 	// TODO (dukov) replace with the appropriate function once it's available
 	// in document module
-	docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "")
+	root, err := globalConf.CurrentContextEntryPoint(config.Ephemeral, "")
+	if err != nil {
+		return err
+	}
+	docBundle, err := document.NewBundleByPath(root)
 	if err != nil {
 		return err
 	}

pkg/bootstrap/isogen/command_test.go

@@ -45,8 +45,7 @@ func (mc *mockContainer) GetID() string {
 }
 
 func TestBootstrapIso(t *testing.T) {
-	fSys := testutil.SetupTestFs(t, "testdata")
-	bundle, err := document.NewBundle(fSys, "/", "/")
+	bundle, err := document.NewBundleByPath("testdata/primary/site/test-site/ephemeral")
 	require.NoError(t, err, "Building Bundle Failed")
 
 	tempVol, cleanup := testutil.TempDir(t, "bootstrap-test")

pkg/cluster/initinfra/infra.go

@@ -39,7 +39,6 @@ func (infra *Infra) Run() error {
 // Deploy method deploys documents
 func (infra *Infra) Deploy() error {
 	kctl := infra.Client.Kubectl()
-	var err error
 	ao, err := kctl.ApplyOptions()
 	if err != nil {
 		return err
@@ -56,29 +55,23 @@ func (infra *Infra) Deploy() error {
 		return err
 	}
 
-	var manifest *config.Manifest
-	manifest, err = globalConf.CurrentContextManifest()
+	kustomizePath, err := globalConf.CurrentContextEntryPoint(infra.ClusterType, config.Initinfra)
 	if err != nil {
 		return err
 	}
 
-	b, err := document.NewBundle(infra.FileSystem, manifest.TargetPath, "")
+	b, err := document.NewBundleByPath(kustomizePath)
 	if err != nil {
 		return err
 	}
 
-	ls := document.EphemeralClusterSelector
-	selector := document.NewSelector().ByLabel(ls)
-
-	// Get documents that are annotated to belong to initinfra
-	docs, err := b.Select(selector)
+	// TODO (kkalynovskyi) Add Selector that would filter by label indicating wether to deploy it to k8s
+	docs, err := b.GetAllDocuments()
 	if err != nil {
 		return err
 	}
 	if len(docs) == 0 {
-		return document.ErrDocNotFound{
-			Selector: selector,
-		}
+		return document.ErrDocNotFound{}
 	}
 
 	// Label every document indicating that it was deployed by initinfra module for further reference

pkg/cluster/initinfra/infra_test.go

@@ -28,7 +28,7 @@ func (tc TestClient) Kubectl() kubectl.Interface      { return tc.MockKubectl()
 
 const (
 	kubeconfigPath    = "testdata/kubeconfig.yaml"
-	filenameRC        = "testdata/replicationcontroller.yaml"
+	filenameRC        = "testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml"
 	airshipConfigFile = "testdata/config.yaml"
 )
 

pkg/cluster/initinfra/testdata/config.yaml

@@ -13,20 +13,19 @@ current-context: dummy_cluster
 kind: Config
 manifests:
   dummy_manifest:
+    primary-repository-name: primary
     repositories:
-      dummy:
-        target-path: dummy_targetpath
-        url:
-          ForceQuery: false
-          Fragment: ""
-          Host: dummy.url.com
-          Opaque: ""
-          Path: ""
-          RawPath: ""
-          RawQuery: ""
-          Scheme: http
-          User: null
-        username: dummy_user
+      primary:
+        auth:
+          ssh-key: testdata/test-key.pem
+          type: ssh-key
+        checkout:
+          branch: ""
+          force: false
+          remote-ref: ""
+          tag: v1.0.1
+        url: http://dummy.url.com/primary.git
+    sub-path: primary/site/test-site
     target-path: testdata
 modules-config:
   bootstrapInfo:

pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml

@@ -4,9 +4,7 @@ metadata:
   name: test-rc
   namespace: test
   labels:
-    airshipit.org/ephemeral: "true"
-    name: test-rc
-    airship-component: "initinfra"
+    airshipit.org/initinfra: "true"
 spec:
   replicas: 1
   template:

pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+  - initinfra

pkg/config/config.go

@@ -21,15 +21,15 @@ import (
 	"fmt"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"reflect"
 	"sort"
 	"strings"
 
-	"sigs.k8s.io/yaml"
-
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
+	"sigs.k8s.io/yaml"
 
 	"opendev.org/airship/airshipctl/pkg/util"
 )
@@ -634,6 +634,28 @@ func (c *Config) CurrentContextManifest() (*Manifest, error) {
 	return c.Manifests[currentContext.Manifest], nil
 }
 
+// CurrentContextEntryPoint returns path to build bundle based on clusterType and phase
+// example CurrentContextEntryPoint("ephemeral", "initinfra")
+func (c *Config) CurrentContextEntryPoint(clusterType string, phase string) (string, error) {
+	err := ValidClusterType(clusterType)
+	if err != nil {
+		return "", err
+	}
+	ccm, err := c.CurrentContextManifest()
+	if err != nil {
+		return "", err
+	}
+	_, exists := ccm.Repositories[ccm.PrimaryRepositoryName]
+	if !exists {
+		return "", ErrMissingPrimaryRepo{}
+	}
+	return path.Join(
+		ccm.TargetPath,
+		ccm.SubPath,
+		clusterType,
+		phase), nil
+}
+
 // Credential or AuthInfo related methods
 func (c *Config) GetAuthInfo(aiName string) (*AuthInfo, error) {
 	authinfo, exists := c.AuthInfos[aiName]
@@ -845,9 +867,8 @@ func (m *Manifest) Equal(n *Manifest) bool {
 	if n == nil {
 		return n == m
 	}
-	repositoryEq := reflect.DeepEqual(m.Repository, n.Repository)
-	extraReposEq := reflect.DeepEqual(m.ExtraRepositories, n.ExtraRepositories)
-	return repositoryEq && extraReposEq && m.TargetPath == n.TargetPath
+	reposEq := reflect.DeepEqual(m.Repositories, n.Repositories)
+	return reposEq && m.TargetPath == n.TargetPath && m.SubPath == n.SubPath
 }
 
 func (m *Manifest) String() string {

pkg/config/constants.go

@@ -11,6 +11,11 @@ const (
 	AirshipClusterDefaultType = Target
 )
 
+// Constants related to Phases
+const (
+	Initinfra = "initinfra"
+)
+
 // Sorted
 var AllClusterTypes = [2]string{Ephemeral, Target}
 
@@ -58,3 +63,8 @@ const (
 	FlagUsername = "username"
 	FlagCurrent  = "current"
 )
+
+// Constants related to filesystem
+const (
+	SiteDirectory = "site"
+)

pkg/config/errors.go

@@ -93,3 +93,11 @@ type ErrMissingCurrentContext struct {
 func (e ErrMissingCurrentContext) Error() string {
 	return "Current context must be set before using --current flag"
 }
+
+// ErrMissingPrimaryRepo returned when Primary Repository is not set in context manifest
+type ErrMissingPrimaryRepo struct {
+}
+
+func (e ErrMissingPrimaryRepo) Error() string {
+	return "Current context manifest must have primary repository set"
+}

pkg/config/testdata/config-string.yaml

@@ -16,16 +16,19 @@ current-context: dummy_context
 kind: Config
 manifests:
   dummy_manifest:
-    repository:
-      auth:
-        ssh-key: testdata/test-key.pem
-        type: ssh-key
-      checkout:
-        branch: ""
-        force: false
-        remote-ref: ""
-        tag: v1.0.1
-      url: http://dummy.url.com
+    primary-repository-name: primary
+    repositories:
+      primary:
+        auth:
+          ssh-key: testdata/test-key.pem
+          type: ssh-key
+        checkout:
+          branch: ""
+          force: false
+          remote-ref: ""
+          tag: v1.0.1
+        url: http://dummy.url.com/manifests.git
+    sub-path: manifests/site/test-site
     target-path: /var/tmp/
 modules-config:
   bootstrapInfo:

pkg/config/testdata/manifest-string.yaml

@@ -1,11 +1,14 @@
-repository:
-  auth:
-    ssh-key: testdata/test-key.pem
-    type: ssh-key
-  checkout:
-    branch: ""
-    force: false
-    remote-ref: ""
-    tag: v1.0.1
-  url: http://dummy.url.com
+primary-repository-name: primary
+repositories:
+  primary:
+    auth:
+      ssh-key: testdata/test-key.pem
+      type: ssh-key
+    checkout:
+      branch: ""
+      force: false
+      remote-ref: ""
+      tag: v1.0.1
+    url: http://dummy.url.com/manifests.git
+sub-path: manifests/site/test-site
 target-path: /var/tmp/

pkg/config/testdata/repository-string.yaml

@@ -6,4 +6,4 @@ checkout:
   force: false
   remote-ref: ""
   tag: v1.0.1
-url: http://dummy.url.com
+url: http://dummy.url.com/manifests.git

pkg/config/types.go

@@ -113,15 +113,23 @@ type AuthInfo struct {
 	authInfo *kubeconfig.AuthInfo
 }
 
-// Manifests is a tuple of references to a Manifest (how do Identify, collect ,
+// Manifest is a tuple of references to a Manifest (how do Identify, collect ,
 // find the yaml manifests that airship uses to perform its operations)
 type Manifest struct {
-	// Repositories is the map of repository adddressable by a name
-	Repository *Repository `json:"repository"`
+	// PrimaryRepositoryName is a name of the repo, that contains site/<site-name> directory
+	// and is a starting point for building document bundle
+	PrimaryRepositoryName string `json:"primary-repository-name"`
 	// ExtraRepositories is the map of extra repositories addressable by a name
-	ExtraRepositories map[string]*Repository `json:"extra-repositories,omitempty"`
+	Repositories map[string]*Repository `json:"repositories,omitempty"`
 	// TargetPath Local Target path for working or home dirctory for all Manifest Cloned/Returned/Generated
 	TargetPath string `json:"target-path"`
+	// SubPath is a path relative to TargetPath + Path where PrimaryRepository is cloned and contains
+	// directories with ClusterType and Phase bundles, example:
+	// Repositories[PrimaryRepositoryName].Url = 'https://github.com/airshipit/treasuremap'
+	// SubPath = "manifests"
+	// you would expect that at treasuremap/manifests you would have ephemeral/initinfra and
+	// ephemera/target directories, containing kustomize.yaml.
+	SubPath string `json:"sub-path"`
 }
 
 // Repository is a tuple that holds the information for the remote sources of manifest yaml documents.

pkg/config/utils.go

@@ -16,6 +16,10 @@ limitations under the License.
 
 package config
 
+const (
+	DefaultTestPrimaryRepo = "primary"
+)
+
 // NewConfig returns a newly initialized Config object
 func NewConfig() *Config {
 	return &Config{
@@ -30,15 +34,19 @@ func NewConfig() *Config {
 		},
 		Manifests: map[string]*Manifest{
 			AirshipDefaultManifest: {
-				Repository: &Repository{
-					URLString: AirshipDefaultManifestRepoLocation,
-					CheckoutOptions: &RepoCheckout{
-						CommitHash: "master",
-						Branch:     "master",
-						RemoteRef:  "master",
+				Repositories: map[string]*Repository{
+					DefaultTestPrimaryRepo: {
+						URLString: AirshipDefaultManifestRepoLocation,
+						CheckoutOptions: &RepoCheckout{
+							CommitHash: "master",
+							Branch:     "master",
+							RemoteRef:  "master",
+						},
 					},
 				},
-				TargetPath: "/tmp/" + AirshipDefaultManifest,
+				TargetPath:            "/tmp/" + AirshipDefaultManifest,
+				PrimaryRepositoryName: DefaultTestPrimaryRepo,
+				SubPath:               AirshipDefaultManifestRepo + "/manifests/site",
 			},
 		},
 		ModulesConfig: &Modules{
@@ -78,8 +86,8 @@ func NewCluster() *Cluster {
 // object with non-nil maps
 func NewManifest() *Manifest {
 	return &Manifest{
-		Repository:        NewRepository(),
-		ExtraRepositories: make(map[string]*Repository),
+		PrimaryRepositoryName: DefaultTestPrimaryRepo,
+		Repositories:          map[string]*Repository{DefaultTestPrimaryRepo: NewRepository()},
 	}
 }
 

pkg/document/bundle.go

@@ -57,6 +57,12 @@ type Bundle interface {
 	GetAllDocuments() ([]Document, error)
 }
 
+// NewBundleByPath helper function that returns new document.Bundle interface based on clusterType and
+// phase, example: helpers.NewBunde(airConfig, "ephemeral", "initinfra")
+func NewBundleByPath(rootPath string) (Bundle, error) {
+	return NewBundle(NewDocumentFs(), rootPath, "")
+}
+
 // NewBundle is a convenience function to create a new bundle
 // Over time, it will evolve to support allowing more control
 // for kustomize plugins

pkg/document/constants.go

@@ -2,8 +2,8 @@ package document
 
 const (
 	// Selectors
-	BaseAirshipSelector      = "airshipit.org"
-	EphemeralClusterSelector = BaseAirshipSelector + "/ephemeral in (True, true)"
+	BaseAirshipSelector = "airshipit.org"
+	ControlNodeSelector = BaseAirshipSelector + "/node-role=control-plane"
 
 	// Labels
 	DeployedByLabel = BaseAirshipSelector + "/deployed"

pkg/document/pull/pull.go

@@ -25,19 +25,8 @@ func (s *Settings) cloneRepositories() error {
 		return err
 	}
 
-	mainRepoConfig := currentManifest.Repository
-	repository, err := repo.NewRepository(currentManifest.TargetPath, mainRepoConfig)
-	if err != nil {
-		return err
-	}
-	err = repository.Download(mainRepoConfig.ToCheckoutOptions(true).Force)
-	if err != nil {
-		return err
-	}
-	repository.Driver.Close()
-
-	// Clone extra repositories
-	for _, extraRepoConfig := range currentManifest.ExtraRepositories {
+	// Clone repositories
+	for _, extraRepoConfig := range currentManifest.Repositories {
 		repository, err := repo.NewRepository(currentManifest.TargetPath, extraRepoConfig)
 		if err != nil {
 			return err

pkg/document/pull/pull_test.go

@@ -3,7 +3,6 @@ package pull
 import (
 	"io/ioutil"
 	"path"
-	"path/filepath"
 	"strings"
 	"testing"
 
@@ -16,6 +15,7 @@ import (
 
 	"opendev.org/airship/airshipctl/pkg/config"
 	"opendev.org/airship/airshipctl/pkg/environment"
+	"opendev.org/airship/airshipctl/pkg/util"
 	"opendev.org/airship/airshipctl/testutil"
 )
 
@@ -42,7 +42,7 @@ func TestPull(t *testing.T) {
 		fx := fixtures.Basic().One()
 
 		dummyGitDir := fx.DotGit().Root()
-		currentManifest.Repository = &config.Repository{
+		currentManifest.Repositories = map[string]*config.Repository{currentManifest.PrimaryRepositoryName: {
 			URLString: dummyGitDir,
 			CheckoutOptions: &config.RepoCheckout{
 				Branch:        "master",
@@ -51,6 +51,7 @@ func TestPull(t *testing.T) {
 			Auth: &config.RepoAuth{
 				Type: "http-basic",
 			},
+		},
 		}
 
 		tmpDir, cleanup := testutil.TempDir(t, "airshipctlPullTest-")
@@ -58,13 +59,13 @@ func TestPull(t *testing.T) {
 
 		currentManifest.TargetPath = tmpDir
 
-		_, err = repo2.NewRepository(".", currentManifest.Repository)
+		_, err = repo2.NewRepository(".", currentManifest.Repositories[currentManifest.PrimaryRepositoryName])
 		require.NoError(err)
 
 		err = dummyPullSettings.cloneRepositories()
 
 		require.NoError(err)
-		dummyRepoDirName := filepath.Base(dummyGitDir)
+		dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir)
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go"))
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
 		contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
@@ -82,14 +83,16 @@ func TestPull(t *testing.T) {
 
 		mfst := conf.Manifests["dummy_manifest"]
 		dummyGitDir := fx.DotGit().Root()
-		mfst.Repository = &config.Repository{
-			URLString: dummyGitDir,
-			CheckoutOptions: &config.RepoCheckout{
-				Branch:        "master",
-				ForceCheckout: false,
-			},
-			Auth: &config.RepoAuth{
-				Type: "http-basic",
+		mfst.Repositories = map[string]*config.Repository{
+			mfst.PrimaryRepositoryName: {
+				URLString: dummyGitDir,
+				CheckoutOptions: &config.RepoCheckout{
+					Branch:        "master",
+					ForceCheckout: false,
+				},
+				Auth: &config.RepoAuth{
+					Type: "http-basic",
+				},
 			},
 		}
 		dummyPullSettings.SetConfig(conf)
@@ -103,7 +106,7 @@ func TestPull(t *testing.T) {
 		err = dummyPullSettings.Pull()
 		require.NoError(err)
 
-		dummyRepoDirName := filepath.Base(dummyGitDir)
+		dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir)
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go"))
 		assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))
 		contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD"))

pkg/document/repo/adapter.go

@@ -28,7 +28,7 @@ type GitDriver struct {
 	Storer     storage.Storer
 }
 
-func NewGitDriver(fs billy.Filesystem, s storage.Storer) *GitDriver {
+func NewGitDriver(fs billy.Filesystem, s storage.Storer) Adapter {
 	return &GitDriver{Storer: s, Filesystem: fs}
 }
 

pkg/document/repo/repo.go

@@ -4,7 +4,6 @@ import (
 	"errors"
 	"fmt"
 	"path/filepath"
-	"strings"
 
 	"gopkg.in/src-d/go-billy.v4"
 	"gopkg.in/src-d/go-billy.v4/osfs"
@@ -15,6 +14,7 @@ import (
 	"gopkg.in/src-d/go-git.v4/storage/filesystem"
 
 	"opendev.org/airship/airshipctl/pkg/log"
+	"opendev.org/airship/airshipctl/pkg/util"
 )
 
 var (
@@ -41,7 +41,7 @@ type Repository struct {
 // NewRepository create repository object, with real filesystem on disk
 // basePath is used to calculate final path where to clone/open the repository
 func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error) {
-	dirName := nameFromURL(builder.URL())
+	dirName := util.GitDirNameFromURL(builder.URL())
 	if dirName == "" {
 		return nil, fmt.Errorf("URL: %s, original error: %w", builder.URL(), ErrCantParseURL)
 	}
@@ -60,11 +60,6 @@ func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error)
 	}, nil
 }
 
-func nameFromURL(urlString string) string {
-	_, fileName := filepath.Split(urlString)
-	return strings.TrimSuffix(fileName, ".git")
-}
-
 func storerFromFs(fs billy.Filesystem) (storage.Storer, error) {
 	dot, err := fs.Chroot(".git")
 	if err != nil {

pkg/document/repo/repo_test.go

@@ -38,19 +38,6 @@ func (md mockBuilder) ToFetchOptions(transport.AuthMethod) *git.FetchOptions {
 }
 func (md mockBuilder) URL() string { return md.URLString }
 
-func TestNewRepositoryNegative(t *testing.T) {
-	err := fixtures.Init()
-	require.NoError(t, err)
-	defer testutil.CleanUpGitFixtures(t)
-
-	builder := &mockBuilder{
-		URLString: "",
-	}
-	repo, err := NewRepository(".", builder)
-	assert.Error(t, err)
-	assert.Nil(t, repo)
-}
-
 func TestDownload(t *testing.T) {
 	err := fixtures.Init()
 	require.NoError(t, err)
@@ -216,47 +203,3 @@ func TestCheckout(t *testing.T) {
 	err = repo.Checkout(true)
 	assert.Error(t, err)
 }
-
-func TestURLtoName(t *testing.T) {
-	tests := []struct {
-		input          string
-		expectedOutput string
-	}{
-		{
-			input:          "https://github.com/kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "git@github.com:kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "https://github.com/kubernetes/kube.somepath.ctl.git",
-			expectedOutput: "kube.somepath.ctl",
-		},
-		{
-			input:          "https://github.com/kubernetes/kubectl",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "git@github.com:kubernetes/kubectl",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "github.com:kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "/kubernetes/kubectl.git",
-			expectedOutput: "kubectl",
-		},
-		{
-			input:          "/kubernetes/kubectl.git/",
-			expectedOutput: "",
-		},
-	}
-
-	for _, test := range tests {
-		assert.Equal(t, test.expectedOutput, nameFromURL(test.input))
-	}
-}

pkg/remote/remote_direct.go

@@ -63,10 +63,6 @@ func getRemoteDirectClient(remoteConfig *config.RemoteDirect, remoteURL string)
 
 func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.RemoteDirect, string, error) {
 	cfg := settings.Config()
-	manifest, err := cfg.CurrentContextManifest()
-	if err != nil {
-		return nil, "", err
-	}
 	bootstrapSettings, err := cfg.CurrentContextBootstrapInfo()
 	if err != nil {
 		return nil, "", err
@@ -77,14 +73,17 @@ func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.Re
 		return nil, "", config.ErrMissingConfig{What: "RemoteDirect options not defined in bootstrap config"}
 	}
 
-	// TODO (dukov) replace with the appropriate function once it's available
-	// in document module
-	docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "")
+	root, err := cfg.CurrentContextEntryPoint(config.Ephemeral, "")
 	if err != nil {
 		return nil, "", err
 	}
 
-	ls := document.EphemeralClusterSelector
+	docBundle, err := document.NewBundleByPath(root)
+	if err != nil {
+		return nil, "", err
+	}
+
+	ls := document.ControlNodeSelector
 	selector := document.NewSelector().
 		ByGvk("", "", AirshipHostKind).
 		ByLabel(ls)

pkg/remote/remote_direct_test.go

@@ -14,6 +14,7 @@ import (
 )
 
 func initSettings(t *testing.T, rd *config.RemoteDirect, testdata string) *environment.AirshipCTLSettings {
+	t.Helper()
 	settings := &environment.AirshipCTLSettings{}
 	settings.SetConfig(testutil.DummyConfig())
 	bi, err := settings.Config().CurrentContextBootstrapInfo()
@@ -36,7 +37,6 @@ func TestUnknownRemoteType(t *testing.T) {
 	)
 
 	err := DoRemoteDirect(s)
-
 	_, ok := err.(*GenericError)
 	assert.True(t, ok)
 }
@@ -52,7 +52,6 @@ func TestRedfishRemoteDirectWithEmptyURL(t *testing.T) {
 	)
 
 	err := DoRemoteDirect(s)
-
 	_, ok := err.(redfish.ErrRedfishMissingConfig)
 	assert.True(t, ok)
 }

pkg/remote/testdata/base/baremetal.yaml

@@ -1,49 +0,0 @@
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0
-spec:
-  online: true
-  bootMACAddress: 00:3b:8b:0c:ec:8b
-  bmc:
-    address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node
-    credentialsName: master-0-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1
-spec:
-  online: "true"
-  bootMACAddress: 01:3b:8b:0c:ec:8b
-  bmc:
-    address: ipmi://192.168.111.2:6230
-    credentialsName: master-1-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
-...

pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node
+    credentialsName: master-0-bmc-secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0-bmc-secret
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: cGFzc3dvcmQ=
+...

pkg/remote/testdata/emptyurl/baremetal.yaml

@@ -1,49 +0,0 @@
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0
-spec:
-  online: true
-  bootMACAddress: 00:3b:8b:0c:ec:8b
-  bmc:
-    address: ""
-    credentialsName: master-0-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/ephemeral: "true"
-  name: master-0-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
----
-apiVersion: metal3.io/v1alpha1
-kind: BareMetalHost
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1
-spec:
-  online: true
-  bootMACAddress: 01:3b:8b:0c:ec:8b
-  bmc:
-    address: ipmi://192.168.111.2:6230
-    credentialsName: master-1-bmc-secret
----
-apiVersion: v1
-kind: Secret
-metadata:
-  labels:
-    airshipit.org/target: "true"
-  name: master-1-bmc-secret
-type: Opaque
-data:
-  username: YWRtaW4=
-  password: cGFzc3dvcmQ=
-...

pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -0,0 +1,25 @@
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: ""
+    credentialsName: master-0-bmc-secret
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/node-role: "control-plane"
+  name: master-0-bmc-secret
+type: Opaque
+data:
+  username: YWRtaW4=
+  password: cGFzc3dvcmQ=
+...

pkg/util/url.go

@@ -0,0 +1,12 @@
+package util
+
+import (
+	"path/filepath"
+	"strings"
+)
+
+// GitDirNameFromURL extract directory name of the repository from URL
+func GitDirNameFromURL(urlString string) string {
+	_, fileName := filepath.Split(urlString)
+	return strings.TrimSuffix(fileName, ".git")
+}

pkg/util/url_test.go

@@ -0,0 +1,51 @@
+package util
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGitDirNameFromURL(t *testing.T) {
+	tests := []struct {
+		input          string
+		expectedOutput string
+	}{
+		{
+			input:          "https://github.com/kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "git@github.com:kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "https://github.com/kubernetes/kube.somepath.ctl.git",
+			expectedOutput: "kube.somepath.ctl",
+		},
+		{
+			input:          "https://github.com/kubernetes/kubectl",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "git@github.com:kubernetes/kubectl",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "github.com:kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "/kubernetes/kubectl.git",
+			expectedOutput: "kubectl",
+		},
+		{
+			input:          "/kubernetes/kubectl.git/",
+			expectedOutput: "",
+		},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.expectedOutput, GitDirNameFromURL(test.input))
+	}
+}

playbooks/vars/test-config.yaml

@@ -1,7 +1,6 @@
 airship_config_action: generate
-airship_config_site_name: "test-bootstrap"
 airship_config_iso_gen_target_path: "{{ serve_dir }}"
-airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}/manifests/site/{{ airship_config_site_name }}"
+airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}"
 airship_config_ephemeral_ip: "{{ airship_gate_ipam.nat_network.ephemeral_ip }}"
 airship_config_iso_builder_docker_image: "kkalynovskyi/image-builder:latest"
 airship_config_ca_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo=

roles/airshipctl-test-configs/defaults/main.yaml

@@ -1,8 +1,6 @@
 airship_config_iso_gen_target_path: /srv/iso
 airship_config_manifest_directory: /tmp/airship
 airship_config_primary_repo_url: dummy.url.com
-airship_config_primary_repo_auth_type: ssh-key
-airship_config_primary_repo_key_path: ""
 airship_config_ephemeral_ip: "10.23.25.101"
 airship_config_iso_builder_docker_image: quay.io/airshipit/isogen:latest
 airship_config_iso_port: 8099

roles/airshipctl-test-configs/templates/airshipconfig.j2

@@ -13,16 +13,19 @@ current-context: dummy_cluster
 kind: Config
 manifests:
   dummy_manifest:
-    repository:
-      auth:
-        ssh-key: {{ airship_config_primary_repo_key_path | default("") }}
-        type: {{ airship_config_primary_repo_auth_type }}
-      checkout:
-        branch: "master"
-        force: false
-        remote-ref: ""
-        tag: ""
-      url: {{ airship_config_primary_repo_url }}
+    primary-repository: primary
+    repositories:
+      primary:
+        checkout:
+          branch: "master"
+          force: false
+          remote-ref: ""
+          tag: ""
+        url: {{ airship_config_primary_repo_url }}
+    ## this is temporary hack, as soon as we use `document pull` command in gate process
+    ## this will subpath will be airshipctl/manifests/site/test-bootstrap, as airshipctl
+    ## will be primary repository
+    sub-path: "manifests/site/test-bootstrap"
     target-path: {{ airship_config_manifest_directory }}
 modules-config:
   bootstrapInfo:

testutil/testconfig.go

@@ -90,15 +90,17 @@ func DummyCluster() *config.Cluster {
 func DummyManifest() *config.Manifest {
 	m := config.NewManifest()
 	// Repositories is the map of repository adddressable by a name
-	m.Repository = DummyRepository()
+	m.Repositories = map[string]*config.Repository{"primary": DummyRepository()}
+	m.PrimaryRepositoryName = "primary"
 	m.TargetPath = "/var/tmp/"
+	m.SubPath = "manifests/site/test-site"
 	return m
 }
 
 // DummyRepository, utility function used for tests
 func DummyRepository() *config.Repository {
 	return &config.Repository{
-		URLString: "http://dummy.url.com",
+		URLString: "http://dummy.url.com/manifests.git",
 		CheckoutOptions: &config.RepoCheckout{
 			Tag:           "v1.0.1",
 			ForceCheckout: false,