Add document pull step to gate script runner

Currently we don't test document pull step by rewriting
manifest_directory variable as current source path, which
makes a mess in target_path, manifest_path and related paths
(concatenation of target_path+manifest_path won't return
an existing and proper location). This patch organizes
mentioned variables and enables document pull step which
starts working properly. The latest repo state used as
git source for this command.

Change-Id: I5abce73877441c7529f2f77add79cf410e2226d8
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>
This commit is contained in:
Ruslan Aliev 2020-09-17 18:33:18 -05:00
parent 233b87a103
commit d8a6139e81
26 changed files with 76 additions and 150 deletions

View File

@ -33,11 +33,6 @@ func TestDocument(t *testing.T) {
CmdLine: "-h",
Cmd: document.NewPluginCommand(),
},
{
Name: "document-pull-with-help",
CmdLine: "-h",
Cmd: document.NewPullCommand(nil),
},
}
for _, tt := range tests {
testutil.RunTest(t, tt)

View File

@ -23,13 +23,17 @@ import (
// NewPullCommand creates a new command for pulling airship document repositories
func NewPullCommand(cfgFactory config.Factory) *cobra.Command {
var noCheckout bool
documentPullCmd := &cobra.Command{
Use: "pull",
Short: "Pulls documents from remote git repository",
RunE: func(cmd *cobra.Command, args []string) error {
return pull.Pull(cfgFactory)
return pull.Pull(cfgFactory, noCheckout)
},
}
documentPullCmd.Flags().BoolVarP(&noCheckout, "no-checkout", "n", false,
"No checkout is performed after the clone is complete.")
return documentPullCmd
}

View File

@ -1,7 +0,0 @@
Pulls documents from remote git repository
Usage:
pull [flags]
Flags:
-h, --help help for pull

View File

@ -5,3 +5,4 @@ Usage:
Flags:
-h, --help help for pull
-n, --no-checkout No checkout is performed after the clone is complete.

View File

@ -14,6 +14,7 @@ airshipctl document pull [flags]
```
-h, --help help for pull
-n, --no-checkout No checkout is performed after the clone is complete.
```
### Options inherited from parent commands

View File

@ -3,7 +3,7 @@ kind: Clusterctl
metadata:
labels:
airshipit.org/deploy-k8s: "false"
name: clusterctl-v1
name: clusterctl_init
init-options:
core-provider: "cluster-api:v0.3.7"
bootstrap-providers:
@ -17,22 +17,22 @@ providers:
type: "InfrastructureProvider"
variable-substitution: true
versions:
v0.3.2: manifests/function/capm3/v0.3.2
v0.3.2: airshipctl/manifests/function/capm3/v0.3.2
- name: "kubeadm"
type: "BootstrapProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cabpk/v0.3.7
v0.3.7: airshipctl/manifests/function/cabpk/v0.3.7
- name: "cluster-api"
type: "CoreProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/capi/v0.3.7
v0.3.7: airshipctl/manifests/function/capi/v0.3.7
- name: "kubeadm"
type: "ControlPlaneProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cacpk/v0.3.7
v0.3.7: airshipctl/manifests/function/cacpk/v0.3.7
# These default images can be overridden via the `replacements/` entrypoint
additional-vars:
CONTAINER_CAPM3_MANAGER: quay.io/metal3-io/cluster-api-provider-metal3:v0.3.2

View File

@ -1,2 +1,2 @@
phase:
path: manifests/phases
path: airshipctl/manifests/phases

View File

@ -0,0 +1,6 @@
---
apiVersion: airshipit.org/v1alpha1
kind: Clusterctl
metadata:
name: clusterctl_init
action: init

View File

@ -19,53 +19,6 @@ move-options: {}
action: move
---
apiVersion: airshipit.org/v1alpha1
kind: Clusterctl
metadata:
labels:
airshipit.org/deploy-k8s: "false"
name: clusterctl_init
init-options:
core-provider: "cluster-api:v0.3.7"
bootstrap-providers:
- "kubeadm:v0.3.7"
infrastructure-providers:
- "metal3:v0.3.2"
control-plane-providers:
- "kubeadm:v0.3.7"
action: init
providers:
- name: "metal3"
type: "InfrastructureProvider"
variable-substitution: true
versions:
v0.3.2: manifests/function/capm3/v0.3.2
- name: "kubeadm"
type: "BootstrapProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cabpk/v0.3.7
- name: "cluster-api"
type: "CoreProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/capi/v0.3.7
- name: "kubeadm"
type: "ControlPlaneProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cacpk/v0.3.7
# These default images can be overridden via the `replacements/` entrypoint
additional-vars:
CONTAINER_CAPM3_MANAGER: quay.io/metal3-io/cluster-api-provider-metal3:v0.3.2
CONTAINER_CACPK_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/kubeadm-control-plane-controller:v0.3.7
CONTAINER_CABPK_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/kubeadm-bootstrap-controller:v0.3.7
CONTAINER_CAPI_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/cluster-api-controller:v0.3.7
CONTAINER_CAPM3_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0
CONTAINER_CACPK_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
CONTAINER_CABPK_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
CONTAINER_CAPI_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
---
apiVersion: airshipit.org/v1alpha1
kind: ImageConfiguration
metadata:
name: isogen

View File

@ -4,3 +4,6 @@ resources:
- executors.yaml
- cluster-map.yaml
- kubeconfig.yaml
- ../function/clusterctl
patchesStrategicMerge:
- clusterctl_init_options.yaml

View File

@ -7,7 +7,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: ImageConfiguration
name: isogen
documentEntryPoint: manifests/site/test-site/ephemeral/bootstrap
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/bootstrap
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -19,7 +19,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/ephemeral/initinfra
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/initinfra
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -31,7 +31,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/ephemeral/controlplane
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/controlplane
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -44,7 +44,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/initinfra
documentEntryPoint: airshipctl/manifests/site/test-site/target/initinfra
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -57,7 +57,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/workers
documentEntryPoint: airshipctl/manifests/site/test-site/target/workers
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -102,4 +102,4 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/workload
documentEntryPoint: airshipctl/manifests/site/test-site/target/workload

View File

@ -16,16 +16,16 @@ providers:
- name: "docker"
type: "InfrastructureProvider"
versions:
v0.3.7: manifests/function/capd/v0.3.7
v0.3.7: airshipctl/manifests/function/capd/v0.3.7
- name: "kubeadm"
type: "BootstrapProvider"
versions:
v0.3.3: manifests/function/cabpk/v0.3.3
v0.3.3: airshipctl/manifests/function/cabpk/v0.3.3
- name: "cluster-api"
type: "CoreProvider"
versions:
v0.3.3: manifests/function/capi/v0.3.3
v0.3.3: airshipctl/manifests/function/capi/v0.3.3
- name: "kubeadm"
type: "ControlPlaneProvider"
versions:
v0.3.3: manifests/function/cacpk/v0.3.3
v0.3.3: airshipctl/manifests/function/cacpk/v0.3.3

View File

@ -217,11 +217,10 @@ func (repo *Repository) ToAuth() (transport.AuthMethod, error) {
// ToCheckoutOptions returns an instance of git.CheckoutOptions with
// respective values(Branch/Tag/Hash) in checkout options initialized
// CheckoutOptions describes how a checkout operation should be performed
func (repo *Repository) ToCheckoutOptions(force bool) *git.CheckoutOptions {
co := &git.CheckoutOptions{
Force: force,
}
func (repo *Repository) ToCheckoutOptions() *git.CheckoutOptions {
co := &git.CheckoutOptions{}
if repo.CheckoutOptions != nil {
co.Force = repo.CheckoutOptions.ForceCheckout
switch {
case repo.CheckoutOptions.Branch != "":
co.Branch = plumbing.NewBranchReferenceName(repo.CheckoutOptions.Branch)
@ -238,19 +237,10 @@ func (repo *Repository) ToCheckoutOptions(force bool) *git.CheckoutOptions {
// authentication and URL set
// CloneOptions describes how a clone should be performed
func (repo *Repository) ToCloneOptions(auth transport.AuthMethod) *git.CloneOptions {
cl := &git.CloneOptions{
return &git.CloneOptions{
Auth: auth,
URL: repo.URLString,
}
if repo.CheckoutOptions != nil {
switch {
case repo.CheckoutOptions.Branch != "":
cl.ReferenceName = plumbing.NewBranchReferenceName(repo.CheckoutOptions.Branch)
case repo.CheckoutOptions.Tag != "":
cl.ReferenceName = plumbing.NewTagReferenceName(repo.CheckoutOptions.Tag)
}
}
return cl
}
// ToFetchOptions returns an instance of git.FetchOptions for given authentication

View File

@ -182,7 +182,7 @@ func TestToCheckout(t *testing.T) {
for _, name := range testCase.dataMapEntry {
repo := data.TestData[name]
require.NotNil(t, repo)
co := repo.ToCheckoutOptions(false)
co := repo.ToCheckoutOptions()
if testCase.expectedNil {
assert.Nil(t, co)
} else {

View File

@ -20,30 +20,20 @@ import (
"opendev.org/airship/airshipctl/pkg/log"
)
// Settings is a reference to environment.AirshipCTLSettings
// AirshipCTLSettings is a container for all of the settings needed by airshipctl
type Settings struct {
*config.Config
}
// Pull clones repositories
func Pull(cfgFactory config.Factory) error {
func Pull(cfgFactory config.Factory, noCheckout bool) error {
cfg, err := cfgFactory()
if err != nil {
return err
}
settings := &Settings{cfg}
if err = settings.cloneRepositories(); err != nil {
return err
}
return nil
return cloneRepositories(cfg, noCheckout)
}
func (s *Settings) cloneRepositories() error {
func cloneRepositories(cfg *config.Config, noCheckout bool) error {
// Clone main repository
currentManifest, err := s.CurrentContextManifest()
log.Debugf("Reading current context manifest information from %s", s.LoadedConfigPath())
currentManifest, err := cfg.CurrentContextManifest()
log.Debugf("Reading current context manifest information from %s", cfg.LoadedConfigPath())
if err != nil {
return err
}
@ -60,7 +50,7 @@ func (s *Settings) cloneRepositories() error {
}
log.Printf("Downloading %s repository %s from %s into %s",
repoName, repository.Name, extraRepoConfig.URL(), currentManifest.TargetPath)
err = repository.Download(extraRepoConfig.ToCheckoutOptions(true).Force)
err = repository.Download(noCheckout)
if err != nil {
return err
}

View File

@ -81,14 +81,6 @@ func TestPull(t *testing.T) {
name: "TestCloneRepositoriesMissingCheckoutOptions",
error: nil,
},
{
name: "TestCloneRepositoriesNonMasterBranch",
checkoutOpts: &config.RepoCheckout{
Branch: "branch",
ForceCheckout: false,
},
error: nil,
},
{
name: "TestCloneRepositoriesInvalidOpts",
checkoutOpts: &config.RepoCheckout{
@ -116,7 +108,7 @@ func TestPull(t *testing.T) {
currentManifest, err := cfg.CurrentContextManifest()
require.NoError(err)
err = pull.Pull(cfgFactory)
err = pull.Pull(cfgFactory, false)
if expectedErr != nil {
assert.NotNil(err)
assert.Equal(expectedErr, err)

View File

@ -35,7 +35,7 @@ import (
type OptionsBuilder interface {
ToAuth() (transport.AuthMethod, error)
ToCloneOptions(auth transport.AuthMethod) *git.CloneOptions
ToCheckoutOptions(force bool) *git.CheckoutOptions
ToCheckoutOptions() *git.CheckoutOptions
ToFetchOptions(auth transport.AuthMethod) *git.FetchOptions
URL() string
}
@ -79,7 +79,7 @@ func storerFromFs(fs billy.Filesystem) (storage.Storer, error) {
}
// Update fetches new refs, and checkout according to checkout options
func (repo *Repository) Update(force bool) error {
func (repo *Repository) Update() error {
log.Debugf("Updating repository %s", repo.Name)
if !repo.Driver.IsOpen() {
return ErrNoOpenRepo{}
@ -92,15 +92,15 @@ func (repo *Repository) Update(force bool) error {
if err != nil && err != git.NoErrAlreadyUpToDate {
return fmt.Errorf("failed to fetch refs for repository %v: %w", repo.Name, err)
}
return repo.Checkout(force)
return repo.Checkout()
}
// Checkout git repository, ToCheckoutOptions method will be used go get CheckoutOptions
func (repo *Repository) Checkout(enforce bool) error {
func (repo *Repository) Checkout() error {
if !repo.Driver.IsOpen() {
return ErrNoOpenRepo{}
}
co := repo.ToCheckoutOptions(enforce)
co := repo.ToCheckoutOptions()
var branchHash string
if co.Hash == plumbing.ZeroHash {
branchHash = fmt.Sprintf("branch %s", co.Branch.String())
@ -137,7 +137,7 @@ func (repo *Repository) Clone() error {
// no remotes will be modified in this case, also no refs will be updated.
// enforce parameter is used to simulate git reset --hard option.
// If you want to enforce state of the repository, please delete current git repository before downloading.
func (repo *Repository) Download(enforceCheckout bool) error {
func (repo *Repository) Download(noCheckout bool) error {
log.Debugf("Attempting to download the repository %s", repo.Name)
if !repo.Driver.IsOpen() {
@ -152,5 +152,8 @@ func (repo *Repository) Download(enforceCheckout bool) error {
}
}
return repo.Checkout(enforceCheckout)
if noCheckout {
return nil
}
return repo.Checkout()
}

View File

@ -44,7 +44,7 @@ func (md mockBuilder) ToAuth() (transport.AuthMethod, error) {
func (md mockBuilder) ToCloneOptions(transport.AuthMethod) *git.CloneOptions {
return md.CloneOptions
}
func (md mockBuilder) ToCheckoutOptions(bool) *git.CheckoutOptions {
func (md mockBuilder) ToCheckoutOptions() *git.CheckoutOptions {
return md.CheckoutOptions
}
func (md mockBuilder) ToFetchOptions(transport.AuthMethod) *git.FetchOptions {
@ -131,15 +131,15 @@ func TestUpdate(t *testing.T) {
require.NotEqual(t, prevCommitHash.String(), headHash.String())
builder.CheckoutOptions = &git.CheckoutOptions{Hash: *prevCommitHash}
// Checkout previous commit
err = repo.Checkout(true)
err = repo.Checkout()
require.NoError(t, err)
// Set checkout back to master
builder.CheckoutOptions = checkout
err = repo.Checkout(true)
err = repo.Checkout()
assert.NoError(t, err)
// update repository
require.NoError(t, repo.Update(true))
require.NoError(t, repo.Update())
currentHash, err := repo.Driver.Head()
assert.NoError(t, err)
@ -147,7 +147,7 @@ func TestUpdate(t *testing.T) {
assert.Equal(t, headHash.String(), currentHash.Hash().String())
repo.Driver.Close()
updateError := repo.Update(true)
updateError := repo.Update()
assert.Error(t, updateError)
}
@ -206,6 +206,6 @@ func TestCheckout(t *testing.T) {
repo, err := NewRepository(".", builder)
require.NoError(t, err)
err = repo.Checkout(true)
err = repo.Checkout()
assert.Error(t, err)
}

View File

@ -19,6 +19,5 @@
vars:
default_zuul_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
environment:
AIRSHIP_CONFIG_MANIFEST_DIRECTORY: "{{ remote_work_dir | default(local_src_dir) | default(default_zuul_dir) }}"
AIRSHIP_SITE_NAME: "manifests/site/{{ site | default('test-site') }}"
AIRSHIP_CONFIG_PRIMARY_REPO_URL: "{{ remote_work_dir | default(local_src_dir) | default(default_zuul_dir) }}"
SITE_NAME: "{{ site | default('test-site') }}"

View File

@ -34,8 +34,8 @@ export SYSTEM_ACTION_RETRIES=30
export SYSTEM_REBOOT_DELAY=30
export AIRSHIP_CONFIG_PRIMARY_REPO_BRANCH=${BRANCH:-"master"}
# the git repo url or local file system path to a cloned repo, e.g., /home/stack/airshipctl
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${REPO:-"https://review.opendev.org/airship/airshipctl"}
export AIRSHIP_SITE_NAME=${AIRSHIP_SITE_NAME:-"manifests/site/test-site"}
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${AIRSHIP_CONFIG_PRIMARY_REPO_URL:-"https://review.opendev.org/airship/airshipctl"}
export AIRSHIP_CONFIG_PRIMARY_REPO_NAME=${AIRSHIP_CONFIG_PRIMARY_REPO_NAME:-"airshipctl"}
export AIRSHIP_CONFIG_MANIFEST_DIRECTORY=${AIRSHIP_CONFIG_MANIFEST_DIRECTORY:-"/tmp/airship"}
export EPHEMERAL_CONFIG_CA_DATA=$(cat tools/deployment/certificates/ephemeral_config_ca_data| base64 -w0)
export EPHEMERAL_IP=${EPHEMERAL_IP:-"10.23.25.101"}

View File

@ -15,4 +15,4 @@
set -xe
echo "Pull site documents using airshipctl"
airshipctl document pull --debug
airshipctl document pull -n --debug

View File

@ -28,6 +28,5 @@ manifests:
remoteRef: ""
tag: ""
url: ${AIRSHIP_CONFIG_PRIMARY_REPO_URL}
metadataPath: manifests/metadata.yaml
subPath: ${AIRSHIP_SITE_NAME}
metadataPath: airshipctl/manifests/metadata.yaml
targetPath: ${AIRSHIP_CONFIG_MANIFEST_DIRECTORY}

View File

@ -16,11 +16,10 @@ set -xe
# The root of the manifest structure to be validated.
# This corresponds to the targetPath in an airshipctl config
: ${MANIFEST_ROOT:="${PWD}"}
: ${MANIFEST_ROOT:="$(dirname "${PWD}")"}
# The location of sites whose manifests should be validated.
# This are relative to MANIFEST_ROOT above, and correspond to
# the base of the subPath in an airshipctl config
: ${SITE_ROOT:="manifests/site"}
# This are relative to MANIFEST_ROOT above
: ${SITE_ROOT:="$(basename "${PWD}")/manifests/site"}
: ${SITE:="test-workload"}
: ${CONTEXT:="kind-airship"}
@ -78,7 +77,6 @@ manifests:
force: false
tag: ""
url: https://opendev.org/airship/treasuremap
subPath: ${SITE_ROOT}/${SITE}
targetPath: ${MANIFEST_ROOT}
EOL
}
@ -103,8 +101,6 @@ for cluster in ephemeral target; do
sed -i "s/${CONTEXT}/${CONTEXT}_${cluster}/" ${AIRSHIPKUBECONFIG}
generate_airshipconf ${cluster}
${ACTL} cluster init
# A sequential list of potential phases. A fancier attempt at this has been
# removed since it was choking in certain cases and got to be more trouble than was worth.
# This should be removed once we have a phase map that is smarter.

View File

@ -23,6 +23,7 @@ TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"}
ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"}
PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"}
export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD}
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${AIRSHIP_CONFIG_PRIMARY_REPO_URL:-$PWD}
sudo --preserve-env=AIRSHIPCTL_WS ansible-playbook -i "$ANSIBLE_HOSTS" \
playbooks/airshipctl-gate-runner.yaml \

View File

@ -18,11 +18,10 @@ set -xe
# The root of the manifest structure to be validated.
# This corresponds to the targetPath in an airshipctl config
: ${MANIFEST_ROOT:="${PWD}"}
: ${MANIFEST_ROOT:="$(dirname "${PWD}")"}
# The space-separated locations of sites whose manifests should be validated.
# These are relative to MANIFEST_ROOT above, and correspond to
# the base of the subPath in an airshipctl config
: ${SITE_ROOTS:="manifests/site"}
# These are relative to MANIFEST_ROOT above
: ${SITE_ROOTS:="$(basename "${PWD}")/manifests/site"}
# get kind
echo "Fetching kind from ${KIND_URL}..."

View File

@ -91,6 +91,7 @@
- ./tools/deployment/02_install_clusterctl.sh
# 21_systemwide_executable.sh is run in the build-gate pre-run above
- ./tools/deployment/22_test_configs.sh
- ./tools/deployment/23_pull_documents.sh
- ./tools/deployment/24_build_ephemeral_iso.sh
- ./tools/deployment/25_deploy_ephemeral_node.sh
- ./tools/deployment/26_deploy_metal3_capi_ephemeral_node.sh