diff --git a/cmd/document/pull_test.go b/cmd/document/pull_test.go index a5f18d478..e811b5480 100644 --- a/cmd/document/pull_test.go +++ b/cmd/document/pull_test.go @@ -22,7 +22,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings { fx := fixtures.Basic().One() - mfst.Repository = &config.Repository{ + mfst.Repositories = map[string]*config.Repository{"primary": { URLString: fx.DotGit().Root(), CheckoutOptions: &config.RepoCheckout{ Branch: "master", @@ -31,6 +31,7 @@ func getDummyAirshipSettings(t *testing.T) *environment.AirshipCTLSettings { Auth: &config.RepoAuth{ Type: "http-basic", }, + }, } settings.SetConfig(conf) return settings diff --git a/manifests/function/ephemeral/baremetal.yaml b/manifests/function/ephemeral/baremetal.yaml index a96cbc8eb..04094685b 100644 --- a/manifests/function/ephemeral/baremetal.yaml +++ b/manifests/function/ephemeral/baremetal.yaml @@ -2,7 +2,7 @@ apiVersion: metal3.io/v1alpha1 kind: BareMetalHost metadata: labels: - airshipit.org/ephemeral: "true" + airshipit.org/node-role: "control-plane" name: master-0 spec: online: true diff --git a/manifests/function/ephemeral/secret.yaml b/manifests/function/ephemeral/secret.yaml index 77933923e..d6b94f045 100644 --- a/manifests/function/ephemeral/secret.yaml +++ b/manifests/function/ephemeral/secret.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Secret metadata: labels: - airshipit.org/ephemeral: "true" + airshipit.org/node-role: "control-plane" name: node1-bmc-secret type: Opaque stringData: diff --git a/manifests/site/test-bootstrap/ephemeral/kustomization.yaml b/manifests/site/test-bootstrap/ephemeral/kustomization.yaml new file mode 100644 index 000000000..8572e662f --- /dev/null +++ b/manifests/site/test-bootstrap/ephemeral/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - ../../../type/test-bootstrap diff --git a/manifests/site/test-bootstrap/kustomization.yaml b/manifests/site/test-bootstrap/kustomization.yaml deleted file mode 100644 index c4acabaf5..000000000 --- a/manifests/site/test-bootstrap/kustomization.yaml +++ /dev/null @@ -1,2 +0,0 @@ -resources: - - ../../type/test-bootstrap diff --git a/pkg/bootstrap/cloudinit/cloud-init_test.go b/pkg/bootstrap/cloudinit/cloud-init_test.go index f843bb388..46d87665c 100644 --- a/pkg/bootstrap/cloudinit/cloud-init_test.go +++ b/pkg/bootstrap/cloudinit/cloud-init_test.go @@ -8,12 +8,10 @@ import ( "sigs.k8s.io/kustomize/v3/pkg/types" "opendev.org/airship/airshipctl/pkg/document" - "opendev.org/airship/airshipctl/testutil" ) func TestGetCloudData(t *testing.T) { - fSys := testutil.SetupTestFs(t, "testdata") - bundle, err := document.NewBundle(fSys, "/", "/") + bundle, err := document.NewBundleByPath("testdata") require.NoError(t, err, "Building Bundle Failed") tests := []struct { diff --git a/pkg/bootstrap/cloudinit/testdata/secret.yaml b/pkg/bootstrap/cloudinit/testdata/secret.yaml new file mode 100644 index 000000000..173efb7a0 --- /dev/null +++ b/pkg/bootstrap/cloudinit/testdata/secret.yaml @@ -0,0 +1,41 @@ +apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/node-role: "control-plane" + name: node1-bmc-secret +type: Opaque +data: + netconfig: bmV0Y29uZmlnCg== +stringData: + userdata: cloud-init +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/node-role: "worker" + name: node1-bmc-secret1 +type: Opaque +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + test: nodataforcfg + name: node1-bmc-secret2 +type: Opaque +data: + foo: bmV0Y29uZmlnCg== +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + some-data: "True" + name: node1-bmc-in-secret2 +type: Opaque +data: + netconfig: bmV0Y29uZmlnCg== +stringData: + userdata: cloud-init diff --git a/pkg/bootstrap/isogen/command.go b/pkg/bootstrap/isogen/command.go index 7ebfdac75..d633adeb5 100644 --- a/pkg/bootstrap/isogen/command.go +++ b/pkg/bootstrap/isogen/command.go @@ -34,19 +34,17 @@ func GenerateBootstrapIso(settings *environment.AirshipCTLSettings) error { return err } - var manifest *config.Manifest - manifest, err = globalConf.CurrentContextManifest() - if err != nil { - return err - } - if err = verifyInputs(cfg); err != nil { return err } // TODO (dukov) replace with the appropriate function once it's available // in document module - docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "") + root, err := globalConf.CurrentContextEntryPoint(config.Ephemeral, "") + if err != nil { + return err + } + docBundle, err := document.NewBundleByPath(root) if err != nil { return err } diff --git a/pkg/bootstrap/isogen/command_test.go b/pkg/bootstrap/isogen/command_test.go index 4969263c5..1627d8e07 100644 --- a/pkg/bootstrap/isogen/command_test.go +++ b/pkg/bootstrap/isogen/command_test.go @@ -45,8 +45,7 @@ func (mc *mockContainer) GetID() string { } func TestBootstrapIso(t *testing.T) { - fSys := testutil.SetupTestFs(t, "testdata") - bundle, err := document.NewBundle(fSys, "/", "/") + bundle, err := document.NewBundleByPath("testdata/primary/site/test-site/ephemeral") require.NoError(t, err, "Building Bundle Failed") tempVol, cleanup := testutil.TempDir(t, "bootstrap-test") diff --git a/pkg/bootstrap/isogen/testdata/kustomization.yaml b/pkg/bootstrap/isogen/testdata/primary/site/test-site/ephemeral/kustomization.yaml similarity index 100% rename from pkg/bootstrap/isogen/testdata/kustomization.yaml rename to pkg/bootstrap/isogen/testdata/primary/site/test-site/ephemeral/kustomization.yaml diff --git a/pkg/bootstrap/isogen/testdata/secret.yaml b/pkg/bootstrap/isogen/testdata/primary/site/test-site/ephemeral/secret.yaml similarity index 100% rename from pkg/bootstrap/isogen/testdata/secret.yaml rename to pkg/bootstrap/isogen/testdata/primary/site/test-site/ephemeral/secret.yaml diff --git a/pkg/cluster/initinfra/infra.go b/pkg/cluster/initinfra/infra.go index 886b9dbe5..173f770b7 100644 --- a/pkg/cluster/initinfra/infra.go +++ b/pkg/cluster/initinfra/infra.go @@ -39,7 +39,6 @@ func (infra *Infra) Run() error { // Deploy method deploys documents func (infra *Infra) Deploy() error { kctl := infra.Client.Kubectl() - var err error ao, err := kctl.ApplyOptions() if err != nil { return err @@ -56,29 +55,23 @@ func (infra *Infra) Deploy() error { return err } - var manifest *config.Manifest - manifest, err = globalConf.CurrentContextManifest() + kustomizePath, err := globalConf.CurrentContextEntryPoint(infra.ClusterType, config.Initinfra) if err != nil { return err } - b, err := document.NewBundle(infra.FileSystem, manifest.TargetPath, "") + b, err := document.NewBundleByPath(kustomizePath) if err != nil { return err } - ls := document.EphemeralClusterSelector - selector := document.NewSelector().ByLabel(ls) - - // Get documents that are annotated to belong to initinfra - docs, err := b.Select(selector) + // TODO (kkalynovskyi) Add Selector that would filter by label indicating wether to deploy it to k8s + docs, err := b.GetAllDocuments() if err != nil { return err } if len(docs) == 0 { - return document.ErrDocNotFound{ - Selector: selector, - } + return document.ErrDocNotFound{} } // Label every document indicating that it was deployed by initinfra module for further reference diff --git a/pkg/cluster/initinfra/infra_test.go b/pkg/cluster/initinfra/infra_test.go index 4edbddd0f..5d636c49a 100644 --- a/pkg/cluster/initinfra/infra_test.go +++ b/pkg/cluster/initinfra/infra_test.go @@ -28,7 +28,7 @@ func (tc TestClient) Kubectl() kubectl.Interface { return tc.MockKubectl() const ( kubeconfigPath = "testdata/kubeconfig.yaml" - filenameRC = "testdata/replicationcontroller.yaml" + filenameRC = "testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml" airshipConfigFile = "testdata/config.yaml" ) diff --git a/pkg/cluster/initinfra/testdata/config.yaml b/pkg/cluster/initinfra/testdata/config.yaml index be1b20a2f..6d0c76e94 100644 --- a/pkg/cluster/initinfra/testdata/config.yaml +++ b/pkg/cluster/initinfra/testdata/config.yaml @@ -13,20 +13,19 @@ current-context: dummy_cluster kind: Config manifests: dummy_manifest: + primary-repository-name: primary repositories: - dummy: - target-path: dummy_targetpath - url: - ForceQuery: false - Fragment: "" - Host: dummy.url.com - Opaque: "" - Path: "" - RawPath: "" - RawQuery: "" - Scheme: http - User: null - username: dummy_user + primary: + auth: + ssh-key: testdata/test-key.pem + type: ssh-key + checkout: + branch: "" + force: false + remote-ref: "" + tag: v1.0.1 + url: http://dummy.url.com/primary.git + sub-path: primary/site/test-site target-path: testdata modules-config: bootstrapInfo: diff --git a/pkg/cluster/initinfra/testdata/kustomization.yaml b/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/kustomization.yaml similarity index 100% rename from pkg/cluster/initinfra/testdata/kustomization.yaml rename to pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/kustomization.yaml diff --git a/pkg/cluster/initinfra/testdata/replicationcontroller.yaml b/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml similarity index 77% rename from pkg/cluster/initinfra/testdata/replicationcontroller.yaml rename to pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml index be90542b6..7616611ba 100644 --- a/pkg/cluster/initinfra/testdata/replicationcontroller.yaml +++ b/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/initinfra/replicationcontroller.yaml @@ -4,9 +4,7 @@ metadata: name: test-rc namespace: test labels: - airshipit.org/ephemeral: "true" - name: test-rc - airship-component: "initinfra" + airshipit.org/initinfra: "true" spec: replicas: 1 template: diff --git a/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/kustomization.yaml b/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/kustomization.yaml new file mode 100644 index 000000000..3789004c6 --- /dev/null +++ b/pkg/cluster/initinfra/testdata/primary/site/test-site/ephemeral/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - initinfra diff --git a/pkg/config/config.go b/pkg/config/config.go index d01badeff..46f49734a 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -21,15 +21,15 @@ import ( "fmt" "io/ioutil" "os" + "path" "path/filepath" "reflect" "sort" "strings" - "sigs.k8s.io/yaml" - "k8s.io/client-go/tools/clientcmd" clientcmdapi "k8s.io/client-go/tools/clientcmd/api" + "sigs.k8s.io/yaml" "opendev.org/airship/airshipctl/pkg/util" ) @@ -634,6 +634,28 @@ func (c *Config) CurrentContextManifest() (*Manifest, error) { return c.Manifests[currentContext.Manifest], nil } +// CurrentContextEntryPoint returns path to build bundle based on clusterType and phase +// example CurrentContextEntryPoint("ephemeral", "initinfra") +func (c *Config) CurrentContextEntryPoint(clusterType string, phase string) (string, error) { + err := ValidClusterType(clusterType) + if err != nil { + return "", err + } + ccm, err := c.CurrentContextManifest() + if err != nil { + return "", err + } + _, exists := ccm.Repositories[ccm.PrimaryRepositoryName] + if !exists { + return "", ErrMissingPrimaryRepo{} + } + return path.Join( + ccm.TargetPath, + ccm.SubPath, + clusterType, + phase), nil +} + // Credential or AuthInfo related methods func (c *Config) GetAuthInfo(aiName string) (*AuthInfo, error) { authinfo, exists := c.AuthInfos[aiName] @@ -845,9 +867,8 @@ func (m *Manifest) Equal(n *Manifest) bool { if n == nil { return n == m } - repositoryEq := reflect.DeepEqual(m.Repository, n.Repository) - extraReposEq := reflect.DeepEqual(m.ExtraRepositories, n.ExtraRepositories) - return repositoryEq && extraReposEq && m.TargetPath == n.TargetPath + reposEq := reflect.DeepEqual(m.Repositories, n.Repositories) + return reposEq && m.TargetPath == n.TargetPath && m.SubPath == n.SubPath } func (m *Manifest) String() string { diff --git a/pkg/config/constants.go b/pkg/config/constants.go index 333b336d9..874407926 100644 --- a/pkg/config/constants.go +++ b/pkg/config/constants.go @@ -11,6 +11,11 @@ const ( AirshipClusterDefaultType = Target ) +// Constants related to Phases +const ( + Initinfra = "initinfra" +) + // Sorted var AllClusterTypes = [2]string{Ephemeral, Target} @@ -58,3 +63,8 @@ const ( FlagUsername = "username" FlagCurrent = "current" ) + +// Constants related to filesystem +const ( + SiteDirectory = "site" +) diff --git a/pkg/config/errors.go b/pkg/config/errors.go index a4c7f33da..34406b77e 100644 --- a/pkg/config/errors.go +++ b/pkg/config/errors.go @@ -93,3 +93,11 @@ type ErrMissingCurrentContext struct { func (e ErrMissingCurrentContext) Error() string { return "Current context must be set before using --current flag" } + +// ErrMissingPrimaryRepo returned when Primary Repository is not set in context manifest +type ErrMissingPrimaryRepo struct { +} + +func (e ErrMissingPrimaryRepo) Error() string { + return "Current context manifest must have primary repository set" +} diff --git a/pkg/config/testdata/config-string.yaml b/pkg/config/testdata/config-string.yaml index acd723b1d..caf8b0e90 100644 --- a/pkg/config/testdata/config-string.yaml +++ b/pkg/config/testdata/config-string.yaml @@ -16,16 +16,19 @@ current-context: dummy_context kind: Config manifests: dummy_manifest: - repository: - auth: - ssh-key: testdata/test-key.pem - type: ssh-key - checkout: - branch: "" - force: false - remote-ref: "" - tag: v1.0.1 - url: http://dummy.url.com + primary-repository-name: primary + repositories: + primary: + auth: + ssh-key: testdata/test-key.pem + type: ssh-key + checkout: + branch: "" + force: false + remote-ref: "" + tag: v1.0.1 + url: http://dummy.url.com/manifests.git + sub-path: manifests/site/test-site target-path: /var/tmp/ modules-config: bootstrapInfo: diff --git a/pkg/config/testdata/manifest-string.yaml b/pkg/config/testdata/manifest-string.yaml index 94a227771..2e817499a 100644 --- a/pkg/config/testdata/manifest-string.yaml +++ b/pkg/config/testdata/manifest-string.yaml @@ -1,11 +1,14 @@ -repository: - auth: - ssh-key: testdata/test-key.pem - type: ssh-key - checkout: - branch: "" - force: false - remote-ref: "" - tag: v1.0.1 - url: http://dummy.url.com +primary-repository-name: primary +repositories: + primary: + auth: + ssh-key: testdata/test-key.pem + type: ssh-key + checkout: + branch: "" + force: false + remote-ref: "" + tag: v1.0.1 + url: http://dummy.url.com/manifests.git +sub-path: manifests/site/test-site target-path: /var/tmp/ diff --git a/pkg/config/testdata/repository-string.yaml b/pkg/config/testdata/repository-string.yaml index c51d79b15..97b3772ba 100644 --- a/pkg/config/testdata/repository-string.yaml +++ b/pkg/config/testdata/repository-string.yaml @@ -6,4 +6,4 @@ checkout: force: false remote-ref: "" tag: v1.0.1 -url: http://dummy.url.com +url: http://dummy.url.com/manifests.git diff --git a/pkg/config/types.go b/pkg/config/types.go index be03579a3..c61296321 100644 --- a/pkg/config/types.go +++ b/pkg/config/types.go @@ -113,15 +113,23 @@ type AuthInfo struct { authInfo *kubeconfig.AuthInfo } -// Manifests is a tuple of references to a Manifest (how do Identify, collect , +// Manifest is a tuple of references to a Manifest (how do Identify, collect , // find the yaml manifests that airship uses to perform its operations) type Manifest struct { - // Repositories is the map of repository adddressable by a name - Repository *Repository `json:"repository"` + // PrimaryRepositoryName is a name of the repo, that contains site/ directory + // and is a starting point for building document bundle + PrimaryRepositoryName string `json:"primary-repository-name"` // ExtraRepositories is the map of extra repositories addressable by a name - ExtraRepositories map[string]*Repository `json:"extra-repositories,omitempty"` + Repositories map[string]*Repository `json:"repositories,omitempty"` // TargetPath Local Target path for working or home dirctory for all Manifest Cloned/Returned/Generated TargetPath string `json:"target-path"` + // SubPath is a path relative to TargetPath + Path where PrimaryRepository is cloned and contains + // directories with ClusterType and Phase bundles, example: + // Repositories[PrimaryRepositoryName].Url = 'https://github.com/airshipit/treasuremap' + // SubPath = "manifests" + // you would expect that at treasuremap/manifests you would have ephemeral/initinfra and + // ephemera/target directories, containing kustomize.yaml. + SubPath string `json:"sub-path"` } // Repository is a tuple that holds the information for the remote sources of manifest yaml documents. diff --git a/pkg/config/utils.go b/pkg/config/utils.go index f88806e1e..bdf6ffd3f 100644 --- a/pkg/config/utils.go +++ b/pkg/config/utils.go @@ -16,6 +16,10 @@ limitations under the License. package config +const ( + DefaultTestPrimaryRepo = "primary" +) + // NewConfig returns a newly initialized Config object func NewConfig() *Config { return &Config{ @@ -30,15 +34,19 @@ func NewConfig() *Config { }, Manifests: map[string]*Manifest{ AirshipDefaultManifest: { - Repository: &Repository{ - URLString: AirshipDefaultManifestRepoLocation, - CheckoutOptions: &RepoCheckout{ - CommitHash: "master", - Branch: "master", - RemoteRef: "master", + Repositories: map[string]*Repository{ + DefaultTestPrimaryRepo: { + URLString: AirshipDefaultManifestRepoLocation, + CheckoutOptions: &RepoCheckout{ + CommitHash: "master", + Branch: "master", + RemoteRef: "master", + }, }, }, - TargetPath: "/tmp/" + AirshipDefaultManifest, + TargetPath: "/tmp/" + AirshipDefaultManifest, + PrimaryRepositoryName: DefaultTestPrimaryRepo, + SubPath: AirshipDefaultManifestRepo + "/manifests/site", }, }, ModulesConfig: &Modules{ @@ -78,8 +86,8 @@ func NewCluster() *Cluster { // object with non-nil maps func NewManifest() *Manifest { return &Manifest{ - Repository: NewRepository(), - ExtraRepositories: make(map[string]*Repository), + PrimaryRepositoryName: DefaultTestPrimaryRepo, + Repositories: map[string]*Repository{DefaultTestPrimaryRepo: NewRepository()}, } } diff --git a/pkg/document/bundle.go b/pkg/document/bundle.go index a647e8379..a57584979 100644 --- a/pkg/document/bundle.go +++ b/pkg/document/bundle.go @@ -57,6 +57,12 @@ type Bundle interface { GetAllDocuments() ([]Document, error) } +// NewBundleByPath helper function that returns new document.Bundle interface based on clusterType and +// phase, example: helpers.NewBunde(airConfig, "ephemeral", "initinfra") +func NewBundleByPath(rootPath string) (Bundle, error) { + return NewBundle(NewDocumentFs(), rootPath, "") +} + // NewBundle is a convenience function to create a new bundle // Over time, it will evolve to support allowing more control // for kustomize plugins diff --git a/pkg/document/constants.go b/pkg/document/constants.go index c12a0e954..38e52ea55 100644 --- a/pkg/document/constants.go +++ b/pkg/document/constants.go @@ -2,8 +2,8 @@ package document const ( // Selectors - BaseAirshipSelector = "airshipit.org" - EphemeralClusterSelector = BaseAirshipSelector + "/ephemeral in (True, true)" + BaseAirshipSelector = "airshipit.org" + ControlNodeSelector = BaseAirshipSelector + "/node-role=control-plane" // Labels DeployedByLabel = BaseAirshipSelector + "/deployed" diff --git a/pkg/document/pull/pull.go b/pkg/document/pull/pull.go index be7f7970b..8a48e1178 100644 --- a/pkg/document/pull/pull.go +++ b/pkg/document/pull/pull.go @@ -25,19 +25,8 @@ func (s *Settings) cloneRepositories() error { return err } - mainRepoConfig := currentManifest.Repository - repository, err := repo.NewRepository(currentManifest.TargetPath, mainRepoConfig) - if err != nil { - return err - } - err = repository.Download(mainRepoConfig.ToCheckoutOptions(true).Force) - if err != nil { - return err - } - repository.Driver.Close() - - // Clone extra repositories - for _, extraRepoConfig := range currentManifest.ExtraRepositories { + // Clone repositories + for _, extraRepoConfig := range currentManifest.Repositories { repository, err := repo.NewRepository(currentManifest.TargetPath, extraRepoConfig) if err != nil { return err diff --git a/pkg/document/pull/pull_test.go b/pkg/document/pull/pull_test.go index 86f722b62..9aa0af38a 100644 --- a/pkg/document/pull/pull_test.go +++ b/pkg/document/pull/pull_test.go @@ -3,7 +3,6 @@ package pull import ( "io/ioutil" "path" - "path/filepath" "strings" "testing" @@ -16,6 +15,7 @@ import ( "opendev.org/airship/airshipctl/pkg/config" "opendev.org/airship/airshipctl/pkg/environment" + "opendev.org/airship/airshipctl/pkg/util" "opendev.org/airship/airshipctl/testutil" ) @@ -42,7 +42,7 @@ func TestPull(t *testing.T) { fx := fixtures.Basic().One() dummyGitDir := fx.DotGit().Root() - currentManifest.Repository = &config.Repository{ + currentManifest.Repositories = map[string]*config.Repository{currentManifest.PrimaryRepositoryName: { URLString: dummyGitDir, CheckoutOptions: &config.RepoCheckout{ Branch: "master", @@ -51,6 +51,7 @@ func TestPull(t *testing.T) { Auth: &config.RepoAuth{ Type: "http-basic", }, + }, } tmpDir, cleanup := testutil.TempDir(t, "airshipctlPullTest-") @@ -58,13 +59,13 @@ func TestPull(t *testing.T) { currentManifest.TargetPath = tmpDir - _, err = repo2.NewRepository(".", currentManifest.Repository) + _, err = repo2.NewRepository(".", currentManifest.Repositories[currentManifest.PrimaryRepositoryName]) require.NoError(err) err = dummyPullSettings.cloneRepositories() require.NoError(err) - dummyRepoDirName := filepath.Base(dummyGitDir) + dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir) assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go")) assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD")) contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD")) @@ -82,14 +83,16 @@ func TestPull(t *testing.T) { mfst := conf.Manifests["dummy_manifest"] dummyGitDir := fx.DotGit().Root() - mfst.Repository = &config.Repository{ - URLString: dummyGitDir, - CheckoutOptions: &config.RepoCheckout{ - Branch: "master", - ForceCheckout: false, - }, - Auth: &config.RepoAuth{ - Type: "http-basic", + mfst.Repositories = map[string]*config.Repository{ + mfst.PrimaryRepositoryName: { + URLString: dummyGitDir, + CheckoutOptions: &config.RepoCheckout{ + Branch: "master", + ForceCheckout: false, + }, + Auth: &config.RepoAuth{ + Type: "http-basic", + }, }, } dummyPullSettings.SetConfig(conf) @@ -103,7 +106,7 @@ func TestPull(t *testing.T) { err = dummyPullSettings.Pull() require.NoError(err) - dummyRepoDirName := filepath.Base(dummyGitDir) + dummyRepoDirName := util.GitDirNameFromURL(dummyGitDir) assert.FileExists(path.Join(tmpDir, dummyRepoDirName, "go/example.go")) assert.FileExists(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD")) contents, err := ioutil.ReadFile(path.Join(tmpDir, dummyRepoDirName, ".git/HEAD")) diff --git a/pkg/document/repo/adapter.go b/pkg/document/repo/adapter.go index fdca2dcc4..71803fde5 100644 --- a/pkg/document/repo/adapter.go +++ b/pkg/document/repo/adapter.go @@ -28,7 +28,7 @@ type GitDriver struct { Storer storage.Storer } -func NewGitDriver(fs billy.Filesystem, s storage.Storer) *GitDriver { +func NewGitDriver(fs billy.Filesystem, s storage.Storer) Adapter { return &GitDriver{Storer: s, Filesystem: fs} } diff --git a/pkg/document/repo/repo.go b/pkg/document/repo/repo.go index ac24b5b6c..e838fde1d 100644 --- a/pkg/document/repo/repo.go +++ b/pkg/document/repo/repo.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "path/filepath" - "strings" "gopkg.in/src-d/go-billy.v4" "gopkg.in/src-d/go-billy.v4/osfs" @@ -15,6 +14,7 @@ import ( "gopkg.in/src-d/go-git.v4/storage/filesystem" "opendev.org/airship/airshipctl/pkg/log" + "opendev.org/airship/airshipctl/pkg/util" ) var ( @@ -41,7 +41,7 @@ type Repository struct { // NewRepository create repository object, with real filesystem on disk // basePath is used to calculate final path where to clone/open the repository func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error) { - dirName := nameFromURL(builder.URL()) + dirName := util.GitDirNameFromURL(builder.URL()) if dirName == "" { return nil, fmt.Errorf("URL: %s, original error: %w", builder.URL(), ErrCantParseURL) } @@ -60,11 +60,6 @@ func NewRepository(basePath string, builder OptionsBuilder) (*Repository, error) }, nil } -func nameFromURL(urlString string) string { - _, fileName := filepath.Split(urlString) - return strings.TrimSuffix(fileName, ".git") -} - func storerFromFs(fs billy.Filesystem) (storage.Storer, error) { dot, err := fs.Chroot(".git") if err != nil { diff --git a/pkg/document/repo/repo_test.go b/pkg/document/repo/repo_test.go index c5b42383e..535c3bc0a 100644 --- a/pkg/document/repo/repo_test.go +++ b/pkg/document/repo/repo_test.go @@ -38,19 +38,6 @@ func (md mockBuilder) ToFetchOptions(transport.AuthMethod) *git.FetchOptions { } func (md mockBuilder) URL() string { return md.URLString } -func TestNewRepositoryNegative(t *testing.T) { - err := fixtures.Init() - require.NoError(t, err) - defer testutil.CleanUpGitFixtures(t) - - builder := &mockBuilder{ - URLString: "", - } - repo, err := NewRepository(".", builder) - assert.Error(t, err) - assert.Nil(t, repo) -} - func TestDownload(t *testing.T) { err := fixtures.Init() require.NoError(t, err) @@ -216,47 +203,3 @@ func TestCheckout(t *testing.T) { err = repo.Checkout(true) assert.Error(t, err) } - -func TestURLtoName(t *testing.T) { - tests := []struct { - input string - expectedOutput string - }{ - { - input: "https://github.com/kubernetes/kubectl.git", - expectedOutput: "kubectl", - }, - { - input: "git@github.com:kubernetes/kubectl.git", - expectedOutput: "kubectl", - }, - { - input: "https://github.com/kubernetes/kube.somepath.ctl.git", - expectedOutput: "kube.somepath.ctl", - }, - { - input: "https://github.com/kubernetes/kubectl", - expectedOutput: "kubectl", - }, - { - input: "git@github.com:kubernetes/kubectl", - expectedOutput: "kubectl", - }, - { - input: "github.com:kubernetes/kubectl.git", - expectedOutput: "kubectl", - }, - { - input: "/kubernetes/kubectl.git", - expectedOutput: "kubectl", - }, - { - input: "/kubernetes/kubectl.git/", - expectedOutput: "", - }, - } - - for _, test := range tests { - assert.Equal(t, test.expectedOutput, nameFromURL(test.input)) - } -} diff --git a/pkg/remote/remote_direct.go b/pkg/remote/remote_direct.go index c3c52440b..2f431e4c0 100644 --- a/pkg/remote/remote_direct.go +++ b/pkg/remote/remote_direct.go @@ -63,10 +63,6 @@ func getRemoteDirectClient(remoteConfig *config.RemoteDirect, remoteURL string) func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.RemoteDirect, string, error) { cfg := settings.Config() - manifest, err := cfg.CurrentContextManifest() - if err != nil { - return nil, "", err - } bootstrapSettings, err := cfg.CurrentContextBootstrapInfo() if err != nil { return nil, "", err @@ -77,14 +73,17 @@ func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.Re return nil, "", config.ErrMissingConfig{What: "RemoteDirect options not defined in bootstrap config"} } - // TODO (dukov) replace with the appropriate function once it's available - // in document module - docBundle, err := document.NewBundle(document.NewDocumentFs(), manifest.TargetPath, "") + root, err := cfg.CurrentContextEntryPoint(config.Ephemeral, "") if err != nil { return nil, "", err } - ls := document.EphemeralClusterSelector + docBundle, err := document.NewBundleByPath(root) + if err != nil { + return nil, "", err + } + + ls := document.ControlNodeSelector selector := document.NewSelector(). ByGvk("", "", AirshipHostKind). ByLabel(ls) diff --git a/pkg/remote/remote_direct_test.go b/pkg/remote/remote_direct_test.go index 0654073bb..e770372cc 100644 --- a/pkg/remote/remote_direct_test.go +++ b/pkg/remote/remote_direct_test.go @@ -14,6 +14,7 @@ import ( ) func initSettings(t *testing.T, rd *config.RemoteDirect, testdata string) *environment.AirshipCTLSettings { + t.Helper() settings := &environment.AirshipCTLSettings{} settings.SetConfig(testutil.DummyConfig()) bi, err := settings.Config().CurrentContextBootstrapInfo() @@ -36,7 +37,6 @@ func TestUnknownRemoteType(t *testing.T) { ) err := DoRemoteDirect(s) - _, ok := err.(*GenericError) assert.True(t, ok) } @@ -52,7 +52,6 @@ func TestRedfishRemoteDirectWithEmptyURL(t *testing.T) { ) err := DoRemoteDirect(s) - _, ok := err.(redfish.ErrRedfishMissingConfig) assert.True(t, ok) } diff --git a/pkg/remote/testdata/base/baremetal.yaml b/pkg/remote/testdata/base/baremetal.yaml deleted file mode 100644 index f5806fe95..000000000 --- a/pkg/remote/testdata/base/baremetal.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - labels: - airshipit.org/ephemeral: "true" - name: master-0 -spec: - online: true - bootMACAddress: 00:3b:8b:0c:ec:8b - bmc: - address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node - credentialsName: master-0-bmc-secret ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - airshipit.org/ephemeral: "true" - name: master-0-bmc-secret -type: Opaque -data: - username: YWRtaW4= - password: cGFzc3dvcmQ= ---- -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - labels: - airshipit.org/target: "true" - name: master-1 -spec: - online: "true" - bootMACAddress: 01:3b:8b:0c:ec:8b - bmc: - address: ipmi://192.168.111.2:6230 - credentialsName: master-1-bmc-secret ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - airshipit.org/target: "true" - name: master-1-bmc-secret -type: Opaque -data: - username: YWRtaW4= - password: cGFzc3dvcmQ= -... diff --git a/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml b/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml new file mode 100644 index 000000000..5e3f238e8 --- /dev/null +++ b/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + labels: + airshipit.org/node-role: "control-plane" + name: master-0 +spec: + online: true + bootMACAddress: 00:3b:8b:0c:ec:8b + bmc: + address: redfish+http://nolocalhost:8888/redfish/v1/Systems/test-node + credentialsName: master-0-bmc-secret +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/node-role: "control-plane" + name: master-0-bmc-secret +type: Opaque +data: + username: YWRtaW4= + password: cGFzc3dvcmQ= +... diff --git a/pkg/remote/testdata/base/kustomization.yaml b/pkg/remote/testdata/base/manifests/site/test-site/ephemeral/kustomization.yaml similarity index 100% rename from pkg/remote/testdata/base/kustomization.yaml rename to pkg/remote/testdata/base/manifests/site/test-site/ephemeral/kustomization.yaml diff --git a/pkg/remote/testdata/emptyurl/baremetal.yaml b/pkg/remote/testdata/emptyurl/baremetal.yaml deleted file mode 100644 index 51be180ea..000000000 --- a/pkg/remote/testdata/emptyurl/baremetal.yaml +++ /dev/null @@ -1,49 +0,0 @@ ---- -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - labels: - airshipit.org/ephemeral: "true" - name: master-0 -spec: - online: true - bootMACAddress: 00:3b:8b:0c:ec:8b - bmc: - address: "" - credentialsName: master-0-bmc-secret ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - airshipit.org/ephemeral: "true" - name: master-0-bmc-secret -type: Opaque -data: - username: YWRtaW4= - password: cGFzc3dvcmQ= ---- -apiVersion: metal3.io/v1alpha1 -kind: BareMetalHost -metadata: - labels: - airshipit.org/target: "true" - name: master-1 -spec: - online: true - bootMACAddress: 01:3b:8b:0c:ec:8b - bmc: - address: ipmi://192.168.111.2:6230 - credentialsName: master-1-bmc-secret ---- -apiVersion: v1 -kind: Secret -metadata: - labels: - airshipit.org/target: "true" - name: master-1-bmc-secret -type: Opaque -data: - username: YWRtaW4= - password: cGFzc3dvcmQ= -... diff --git a/pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml b/pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml new file mode 100644 index 000000000..ef9904ec1 --- /dev/null +++ b/pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml @@ -0,0 +1,25 @@ +--- +apiVersion: metal3.io/v1alpha1 +kind: BareMetalHost +metadata: + labels: + airshipit.org/node-role: "control-plane" + name: master-0 +spec: + online: true + bootMACAddress: 00:3b:8b:0c:ec:8b + bmc: + address: "" + credentialsName: master-0-bmc-secret +--- +apiVersion: v1 +kind: Secret +metadata: + labels: + airshipit.org/node-role: "control-plane" + name: master-0-bmc-secret +type: Opaque +data: + username: YWRtaW4= + password: cGFzc3dvcmQ= +... diff --git a/pkg/remote/testdata/emptyurl/kustomization.yaml b/pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/kustomization.yaml similarity index 100% rename from pkg/remote/testdata/emptyurl/kustomization.yaml rename to pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/kustomization.yaml diff --git a/pkg/util/url.go b/pkg/util/url.go new file mode 100644 index 000000000..d246599a3 --- /dev/null +++ b/pkg/util/url.go @@ -0,0 +1,12 @@ +package util + +import ( + "path/filepath" + "strings" +) + +// GitDirNameFromURL extract directory name of the repository from URL +func GitDirNameFromURL(urlString string) string { + _, fileName := filepath.Split(urlString) + return strings.TrimSuffix(fileName, ".git") +} diff --git a/pkg/util/url_test.go b/pkg/util/url_test.go new file mode 100644 index 000000000..1190748bc --- /dev/null +++ b/pkg/util/url_test.go @@ -0,0 +1,51 @@ +package util + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestGitDirNameFromURL(t *testing.T) { + tests := []struct { + input string + expectedOutput string + }{ + { + input: "https://github.com/kubernetes/kubectl.git", + expectedOutput: "kubectl", + }, + { + input: "git@github.com:kubernetes/kubectl.git", + expectedOutput: "kubectl", + }, + { + input: "https://github.com/kubernetes/kube.somepath.ctl.git", + expectedOutput: "kube.somepath.ctl", + }, + { + input: "https://github.com/kubernetes/kubectl", + expectedOutput: "kubectl", + }, + { + input: "git@github.com:kubernetes/kubectl", + expectedOutput: "kubectl", + }, + { + input: "github.com:kubernetes/kubectl.git", + expectedOutput: "kubectl", + }, + { + input: "/kubernetes/kubectl.git", + expectedOutput: "kubectl", + }, + { + input: "/kubernetes/kubectl.git/", + expectedOutput: "", + }, + } + + for _, test := range tests { + assert.Equal(t, test.expectedOutput, GitDirNameFromURL(test.input)) + } +} diff --git a/playbooks/vars/test-config.yaml b/playbooks/vars/test-config.yaml index da694f7fa..09a8e4b4a 100644 --- a/playbooks/vars/test-config.yaml +++ b/playbooks/vars/test-config.yaml @@ -1,7 +1,6 @@ airship_config_action: generate -airship_config_site_name: "test-bootstrap" airship_config_iso_gen_target_path: "{{ serve_dir }}" -airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}/manifests/site/{{ airship_config_site_name }}" +airship_config_manifest_directory: "{{ remote_work_dir | default(zuul.project.src_dir) }}" airship_config_ephemeral_ip: "{{ airship_gate_ipam.nat_network.ephemeral_ip }}" airship_config_iso_builder_docker_image: "kkalynovskyi/image-builder:latest" airship_config_ca_data: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUN5RENDQWJDZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRFNU1USXlOakE0TWpneU5Gb1hEVEk1TVRJeU16QTRNamd5TkZvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTTFSClM0d3lnajNpU0JBZjlCR0JUS1p5VTFwYmdDaGQ2WTdJektaZWRoakM2K3k1ZEJpWm81ZUx6Z2tEc2gzOC9YQ1MKenFPS2V5cE5RcDN5QVlLdmJKSHg3ODZxSFZZNjg1ZDVYVDNaOHNyVVRzVDR5WmNzZHAzV3lHdDM0eXYzNi9BSQoxK1NlUFErdU5JemN6bzNEdWhXR0ZoQjk3VjZwRitFUTBlVWN5bk05c2hkL3AwWVFzWDR1ZlhxaENENVpzZnZUCnBka3UvTWkyWnVGUldUUUtNeGpqczV3Z2RBWnBsNnN0L2ZkbmZwd1Q5cC9WTjRuaXJnMEsxOURTSFFJTHVrU2MKb013bXNBeDJrZmxITWhPazg5S3FpMEloL2cyczRFYTRvWURZemt0Y2JRZ24wd0lqZ2dmdnVzM3pRbEczN2lwYQo4cVRzS2VmVGdkUjhnZkJDNUZNQ0F3RUFBYU1qTUNFd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0RRWUpLb1pJaHZjTkFRRUxCUUFEZ2dFQkFJek9BL00xWmRGUElzd2VoWjFuemJ0VFNURG4KRHMyVnhSV0VnclFFYzNSYmV3a1NkbTlBS3MwVGR0ZHdEbnBEL2tRYkNyS2xEeFF3RWg3NFZNSFZYYkFadDdsVwpCSm90T21xdXgxYThKYklDRTljR0FHRzFvS0g5R29jWERZY0JzOTA3ckxIdStpVzFnL0xVdG5hN1dSampqZnBLCnFGelFmOGdJUHZIM09BZ3B1RVVncUx5QU8ya0VnelZwTjZwQVJxSnZVRks2TUQ0YzFmMnlxWGxwNXhrN2dFSnIKUzQ4WmF6d0RmWUVmV3Jrdld1YWdvZ1M2SktvbjVEZ0Z1ZHhINXM2Snl6R3lPVnZ0eG1TY2FvOHNxaCs3UXkybgoyLzFVcU5ZK0hlN0x4d04rYkhwYkIxNUtIMTU5ZHNuS3BRbjRORG1jSTZrVnJ3MDVJMUg5ZGRBbGF0bz0KLS0tLS1FTkQgQ0VSVElGSUNBVEUtLS0tLQo= diff --git a/roles/airshipctl-test-configs/defaults/main.yaml b/roles/airshipctl-test-configs/defaults/main.yaml index c87ea209f..f5d2d2dce 100644 --- a/roles/airshipctl-test-configs/defaults/main.yaml +++ b/roles/airshipctl-test-configs/defaults/main.yaml @@ -1,8 +1,6 @@ airship_config_iso_gen_target_path: /srv/iso airship_config_manifest_directory: /tmp/airship airship_config_primary_repo_url: dummy.url.com -airship_config_primary_repo_auth_type: ssh-key -airship_config_primary_repo_key_path: "" airship_config_ephemeral_ip: "10.23.25.101" airship_config_iso_builder_docker_image: quay.io/airshipit/isogen:latest airship_config_iso_port: 8099 diff --git a/roles/airshipctl-test-configs/templates/airshipconfig.j2 b/roles/airshipctl-test-configs/templates/airshipconfig.j2 index afb96afad..cf587e61d 100644 --- a/roles/airshipctl-test-configs/templates/airshipconfig.j2 +++ b/roles/airshipctl-test-configs/templates/airshipconfig.j2 @@ -13,16 +13,19 @@ current-context: dummy_cluster kind: Config manifests: dummy_manifest: - repository: - auth: - ssh-key: {{ airship_config_primary_repo_key_path | default("") }} - type: {{ airship_config_primary_repo_auth_type }} - checkout: - branch: "master" - force: false - remote-ref: "" - tag: "" - url: {{ airship_config_primary_repo_url }} + primary-repository: primary + repositories: + primary: + checkout: + branch: "master" + force: false + remote-ref: "" + tag: "" + url: {{ airship_config_primary_repo_url }} + ## this is temporary hack, as soon as we use `document pull` command in gate process + ## this will subpath will be airshipctl/manifests/site/test-bootstrap, as airshipctl + ## will be primary repository + sub-path: "manifests/site/test-bootstrap" target-path: {{ airship_config_manifest_directory }} modules-config: bootstrapInfo: diff --git a/testutil/testconfig.go b/testutil/testconfig.go index 4c959eb9a..73d517349 100644 --- a/testutil/testconfig.go +++ b/testutil/testconfig.go @@ -90,15 +90,17 @@ func DummyCluster() *config.Cluster { func DummyManifest() *config.Manifest { m := config.NewManifest() // Repositories is the map of repository adddressable by a name - m.Repository = DummyRepository() + m.Repositories = map[string]*config.Repository{"primary": DummyRepository()} + m.PrimaryRepositoryName = "primary" m.TargetPath = "/var/tmp/" + m.SubPath = "manifests/site/test-site" return m } // DummyRepository, utility function used for tests func DummyRepository() *config.Repository { return &config.Repository{ - URLString: "http://dummy.url.com", + URLString: "http://dummy.url.com/manifests.git", CheckoutOptions: &config.RepoCheckout{ Tag: "v1.0.1", ForceCheckout: false,