Merge "Add document pull step to gate script runner"

This commit is contained in:
Zuul 2020-09-28 22:45:02 +00:00 committed by Gerrit Code Review
commit 7502da5c34
26 changed files with 76 additions and 150 deletions

View File

@ -33,11 +33,6 @@ func TestDocument(t *testing.T) {
CmdLine: "-h",
Cmd: document.NewPluginCommand(),
},
{
Name: "document-pull-with-help",
CmdLine: "-h",
Cmd: document.NewPullCommand(nil),
},
}
for _, tt := range tests {
testutil.RunTest(t, tt)

View File

@ -23,13 +23,17 @@ import (
// NewPullCommand creates a new command for pulling airship document repositories
func NewPullCommand(cfgFactory config.Factory) *cobra.Command {
var noCheckout bool
documentPullCmd := &cobra.Command{
Use: "pull",
Short: "Pulls documents from remote git repository",
RunE: func(cmd *cobra.Command, args []string) error {
return pull.Pull(cfgFactory)
return pull.Pull(cfgFactory, noCheckout)
},
}
documentPullCmd.Flags().BoolVarP(&noCheckout, "no-checkout", "n", false,
"No checkout is performed after the clone is complete.")
return documentPullCmd
}

View File

@ -1,7 +0,0 @@
Pulls documents from remote git repository
Usage:
pull [flags]
Flags:
-h, --help help for pull

View File

@ -5,3 +5,4 @@ Usage:
Flags:
-h, --help help for pull
-n, --no-checkout No checkout is performed after the clone is complete.

View File

@ -14,6 +14,7 @@ airshipctl document pull [flags]
```
-h, --help help for pull
-n, --no-checkout No checkout is performed after the clone is complete.
```
### Options inherited from parent commands

View File

@ -3,7 +3,7 @@ kind: Clusterctl
metadata:
labels:
airshipit.org/deploy-k8s: "false"
name: clusterctl-v1
name: clusterctl_init
init-options:
core-provider: "cluster-api:v0.3.7"
bootstrap-providers:
@ -17,22 +17,22 @@ providers:
type: "InfrastructureProvider"
variable-substitution: true
versions:
v0.3.2: manifests/function/capm3/v0.3.2
v0.3.2: airshipctl/manifests/function/capm3/v0.3.2
- name: "kubeadm"
type: "BootstrapProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cabpk/v0.3.7
v0.3.7: airshipctl/manifests/function/cabpk/v0.3.7
- name: "cluster-api"
type: "CoreProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/capi/v0.3.7
v0.3.7: airshipctl/manifests/function/capi/v0.3.7
- name: "kubeadm"
type: "ControlPlaneProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cacpk/v0.3.7
v0.3.7: airshipctl/manifests/function/cacpk/v0.3.7
# These default images can be overridden via the `replacements/` entrypoint
additional-vars:
CONTAINER_CAPM3_MANAGER: quay.io/metal3-io/cluster-api-provider-metal3:v0.3.2

View File

@ -1,2 +1,2 @@
phase:
path: manifests/phases
path: airshipctl/manifests/phases

View File

@ -0,0 +1,6 @@
---
apiVersion: airshipit.org/v1alpha1
kind: Clusterctl
metadata:
name: clusterctl_init
action: init

View File

@ -19,53 +19,6 @@ move-options: {}
action: move
---
apiVersion: airshipit.org/v1alpha1
kind: Clusterctl
metadata:
labels:
airshipit.org/deploy-k8s: "false"
name: clusterctl_init
init-options:
core-provider: "cluster-api:v0.3.7"
bootstrap-providers:
- "kubeadm:v0.3.7"
infrastructure-providers:
- "metal3:v0.3.2"
control-plane-providers:
- "kubeadm:v0.3.7"
action: init
providers:
- name: "metal3"
type: "InfrastructureProvider"
variable-substitution: true
versions:
v0.3.2: manifests/function/capm3/v0.3.2
- name: "kubeadm"
type: "BootstrapProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cabpk/v0.3.7
- name: "cluster-api"
type: "CoreProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/capi/v0.3.7
- name: "kubeadm"
type: "ControlPlaneProvider"
variable-substitution: true
versions:
v0.3.7: manifests/function/cacpk/v0.3.7
# These default images can be overridden via the `replacements/` entrypoint
additional-vars:
CONTAINER_CAPM3_MANAGER: quay.io/metal3-io/cluster-api-provider-metal3:v0.3.2
CONTAINER_CACPK_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/kubeadm-control-plane-controller:v0.3.7
CONTAINER_CABPK_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/kubeadm-bootstrap-controller:v0.3.7
CONTAINER_CAPI_MANAGER: us.gcr.io/k8s-artifacts-prod/cluster-api/cluster-api-controller:v0.3.7
CONTAINER_CAPM3_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.0
CONTAINER_CACPK_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
CONTAINER_CABPK_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
CONTAINER_CAPI_AUTH_PROXY: gcr.io/kubebuilder/kube-rbac-proxy:v0.4.1
---
apiVersion: airshipit.org/v1alpha1
kind: ImageConfiguration
metadata:
name: isogen

View File

@ -4,3 +4,6 @@ resources:
- executors.yaml
- cluster-map.yaml
- kubeconfig.yaml
- ../function/clusterctl
patchesStrategicMerge:
- clusterctl_init_options.yaml

View File

@ -7,7 +7,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: ImageConfiguration
name: isogen
documentEntryPoint: manifests/site/test-site/ephemeral/bootstrap
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/bootstrap
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -19,7 +19,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/ephemeral/initinfra
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/initinfra
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -31,7 +31,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/ephemeral/controlplane
documentEntryPoint: airshipctl/manifests/site/test-site/ephemeral/controlplane
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -44,7 +44,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/initinfra
documentEntryPoint: airshipctl/manifests/site/test-site/target/initinfra
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -57,7 +57,7 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/workers
documentEntryPoint: airshipctl/manifests/site/test-site/target/workers
---
apiVersion: airshipit.org/v1alpha1
kind: Phase
@ -102,4 +102,4 @@ config:
apiVersion: airshipit.org/v1alpha1
kind: KubernetesApply
name: kubernetes-apply
documentEntryPoint: manifests/site/test-site/target/workload
documentEntryPoint: airshipctl/manifests/site/test-site/target/workload

View File

@ -16,16 +16,16 @@ providers:
- name: "docker"
type: "InfrastructureProvider"
versions:
v0.3.7: manifests/function/capd/v0.3.7
v0.3.7: airshipctl/manifests/function/capd/v0.3.7
- name: "kubeadm"
type: "BootstrapProvider"
versions:
v0.3.3: manifests/function/cabpk/v0.3.3
v0.3.3: airshipctl/manifests/function/cabpk/v0.3.3
- name: "cluster-api"
type: "CoreProvider"
versions:
v0.3.3: manifests/function/capi/v0.3.3
v0.3.3: airshipctl/manifests/function/capi/v0.3.3
- name: "kubeadm"
type: "ControlPlaneProvider"
versions:
v0.3.3: manifests/function/cacpk/v0.3.3
v0.3.3: airshipctl/manifests/function/cacpk/v0.3.3

View File

@ -217,11 +217,10 @@ func (repo *Repository) ToAuth() (transport.AuthMethod, error) {
// ToCheckoutOptions returns an instance of git.CheckoutOptions with
// respective values(Branch/Tag/Hash) in checkout options initialized
// CheckoutOptions describes how a checkout operation should be performed
func (repo *Repository) ToCheckoutOptions(force bool) *git.CheckoutOptions {
co := &git.CheckoutOptions{
Force: force,
}
func (repo *Repository) ToCheckoutOptions() *git.CheckoutOptions {
co := &git.CheckoutOptions{}
if repo.CheckoutOptions != nil {
co.Force = repo.CheckoutOptions.ForceCheckout
switch {
case repo.CheckoutOptions.Branch != "":
co.Branch = plumbing.NewBranchReferenceName(repo.CheckoutOptions.Branch)
@ -238,19 +237,10 @@ func (repo *Repository) ToCheckoutOptions(force bool) *git.CheckoutOptions {
// authentication and URL set
// CloneOptions describes how a clone should be performed
func (repo *Repository) ToCloneOptions(auth transport.AuthMethod) *git.CloneOptions {
cl := &git.CloneOptions{
return &git.CloneOptions{
Auth: auth,
URL: repo.URLString,
}
if repo.CheckoutOptions != nil {
switch {
case repo.CheckoutOptions.Branch != "":
cl.ReferenceName = plumbing.NewBranchReferenceName(repo.CheckoutOptions.Branch)
case repo.CheckoutOptions.Tag != "":
cl.ReferenceName = plumbing.NewTagReferenceName(repo.CheckoutOptions.Tag)
}
}
return cl
}
// ToFetchOptions returns an instance of git.FetchOptions for given authentication

View File

@ -182,7 +182,7 @@ func TestToCheckout(t *testing.T) {
for _, name := range testCase.dataMapEntry {
repo := data.TestData[name]
require.NotNil(t, repo)
co := repo.ToCheckoutOptions(false)
co := repo.ToCheckoutOptions()
if testCase.expectedNil {
assert.Nil(t, co)
} else {

View File

@ -20,30 +20,20 @@ import (
"opendev.org/airship/airshipctl/pkg/log"
)
// Settings is a reference to environment.AirshipCTLSettings
// AirshipCTLSettings is a container for all of the settings needed by airshipctl
type Settings struct {
*config.Config
}
// Pull clones repositories
func Pull(cfgFactory config.Factory) error {
func Pull(cfgFactory config.Factory, noCheckout bool) error {
cfg, err := cfgFactory()
if err != nil {
return err
}
settings := &Settings{cfg}
if err = settings.cloneRepositories(); err != nil {
return err
}
return nil
return cloneRepositories(cfg, noCheckout)
}
func (s *Settings) cloneRepositories() error {
func cloneRepositories(cfg *config.Config, noCheckout bool) error {
// Clone main repository
currentManifest, err := s.CurrentContextManifest()
log.Debugf("Reading current context manifest information from %s", s.LoadedConfigPath())
currentManifest, err := cfg.CurrentContextManifest()
log.Debugf("Reading current context manifest information from %s", cfg.LoadedConfigPath())
if err != nil {
return err
}
@ -60,7 +50,7 @@ func (s *Settings) cloneRepositories() error {
}
log.Printf("Downloading %s repository %s from %s into %s",
repoName, repository.Name, extraRepoConfig.URL(), currentManifest.TargetPath)
err = repository.Download(extraRepoConfig.ToCheckoutOptions(true).Force)
err = repository.Download(noCheckout)
if err != nil {
return err
}

View File

@ -81,14 +81,6 @@ func TestPull(t *testing.T) {
name: "TestCloneRepositoriesMissingCheckoutOptions",
error: nil,
},
{
name: "TestCloneRepositoriesNonMasterBranch",
checkoutOpts: &config.RepoCheckout{
Branch: "branch",
ForceCheckout: false,
},
error: nil,
},
{
name: "TestCloneRepositoriesInvalidOpts",
checkoutOpts: &config.RepoCheckout{
@ -116,7 +108,7 @@ func TestPull(t *testing.T) {
currentManifest, err := cfg.CurrentContextManifest()
require.NoError(err)
err = pull.Pull(cfgFactory)
err = pull.Pull(cfgFactory, false)
if expectedErr != nil {
assert.NotNil(err)
assert.Equal(expectedErr, err)

View File

@ -35,7 +35,7 @@ import (
type OptionsBuilder interface {
ToAuth() (transport.AuthMethod, error)
ToCloneOptions(auth transport.AuthMethod) *git.CloneOptions
ToCheckoutOptions(force bool) *git.CheckoutOptions
ToCheckoutOptions() *git.CheckoutOptions
ToFetchOptions(auth transport.AuthMethod) *git.FetchOptions
URL() string
}
@ -79,7 +79,7 @@ func storerFromFs(fs billy.Filesystem) (storage.Storer, error) {
}
// Update fetches new refs, and checkout according to checkout options
func (repo *Repository) Update(force bool) error {
func (repo *Repository) Update() error {
log.Debugf("Updating repository %s", repo.Name)
if !repo.Driver.IsOpen() {
return ErrNoOpenRepo{}
@ -92,15 +92,15 @@ func (repo *Repository) Update(force bool) error {
if err != nil && err != git.NoErrAlreadyUpToDate {
return fmt.Errorf("failed to fetch refs for repository %v: %w", repo.Name, err)
}
return repo.Checkout(force)
return repo.Checkout()
}
// Checkout git repository, ToCheckoutOptions method will be used go get CheckoutOptions
func (repo *Repository) Checkout(enforce bool) error {
func (repo *Repository) Checkout() error {
if !repo.Driver.IsOpen() {
return ErrNoOpenRepo{}
}
co := repo.ToCheckoutOptions(enforce)
co := repo.ToCheckoutOptions()
var branchHash string
if co.Hash == plumbing.ZeroHash {
branchHash = fmt.Sprintf("branch %s", co.Branch.String())
@ -137,7 +137,7 @@ func (repo *Repository) Clone() error {
// no remotes will be modified in this case, also no refs will be updated.
// enforce parameter is used to simulate git reset --hard option.
// If you want to enforce state of the repository, please delete current git repository before downloading.
func (repo *Repository) Download(enforceCheckout bool) error {
func (repo *Repository) Download(noCheckout bool) error {
log.Debugf("Attempting to download the repository %s", repo.Name)
if !repo.Driver.IsOpen() {
@ -152,5 +152,8 @@ func (repo *Repository) Download(enforceCheckout bool) error {
}
}
return repo.Checkout(enforceCheckout)
if noCheckout {
return nil
}
return repo.Checkout()
}

View File

@ -44,7 +44,7 @@ func (md mockBuilder) ToAuth() (transport.AuthMethod, error) {
func (md mockBuilder) ToCloneOptions(transport.AuthMethod) *git.CloneOptions {
return md.CloneOptions
}
func (md mockBuilder) ToCheckoutOptions(bool) *git.CheckoutOptions {
func (md mockBuilder) ToCheckoutOptions() *git.CheckoutOptions {
return md.CheckoutOptions
}
func (md mockBuilder) ToFetchOptions(transport.AuthMethod) *git.FetchOptions {
@ -131,15 +131,15 @@ func TestUpdate(t *testing.T) {
require.NotEqual(t, prevCommitHash.String(), headHash.String())
builder.CheckoutOptions = &git.CheckoutOptions{Hash: *prevCommitHash}
// Checkout previous commit
err = repo.Checkout(true)
err = repo.Checkout()
require.NoError(t, err)
// Set checkout back to master
builder.CheckoutOptions = checkout
err = repo.Checkout(true)
err = repo.Checkout()
assert.NoError(t, err)
// update repository
require.NoError(t, repo.Update(true))
require.NoError(t, repo.Update())
currentHash, err := repo.Driver.Head()
assert.NoError(t, err)
@ -147,7 +147,7 @@ func TestUpdate(t *testing.T) {
assert.Equal(t, headHash.String(), currentHash.Hash().String())
repo.Driver.Close()
updateError := repo.Update(true)
updateError := repo.Update()
assert.Error(t, updateError)
}
@ -206,6 +206,6 @@ func TestCheckout(t *testing.T) {
repo, err := NewRepository(".", builder)
require.NoError(t, err)
err = repo.Checkout(true)
err = repo.Checkout()
assert.Error(t, err)
}

View File

@ -19,6 +19,5 @@
vars:
default_zuul_dir: "{{ ansible_user_dir }}/{{ zuul.project.src_dir }}"
environment:
AIRSHIP_CONFIG_MANIFEST_DIRECTORY: "{{ remote_work_dir | default(local_src_dir) | default(default_zuul_dir) }}"
AIRSHIP_SITE_NAME: "manifests/site/{{ site | default('test-site') }}"
AIRSHIP_CONFIG_PRIMARY_REPO_URL: "{{ remote_work_dir | default(local_src_dir) | default(default_zuul_dir) }}"
SITE_NAME: "{{ site | default('test-site') }}"

View File

@ -34,8 +34,8 @@ export SYSTEM_ACTION_RETRIES=30
export SYSTEM_REBOOT_DELAY=30
export AIRSHIP_CONFIG_PRIMARY_REPO_BRANCH=${BRANCH:-"master"}
# the git repo url or local file system path to a cloned repo, e.g., /home/stack/airshipctl
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${REPO:-"https://review.opendev.org/airship/airshipctl"}
export AIRSHIP_SITE_NAME=${AIRSHIP_SITE_NAME:-"manifests/site/test-site"}
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${AIRSHIP_CONFIG_PRIMARY_REPO_URL:-"https://review.opendev.org/airship/airshipctl"}
export AIRSHIP_CONFIG_PRIMARY_REPO_NAME=${AIRSHIP_CONFIG_PRIMARY_REPO_NAME:-"airshipctl"}
export AIRSHIP_CONFIG_MANIFEST_DIRECTORY=${AIRSHIP_CONFIG_MANIFEST_DIRECTORY:-"/tmp/airship"}
export EPHEMERAL_CONFIG_CA_DATA=$(cat tools/deployment/certificates/ephemeral_config_ca_data| base64 -w0)
export EPHEMERAL_IP=${EPHEMERAL_IP:-"10.23.25.101"}

View File

@ -15,4 +15,4 @@
set -xe
echo "Pull site documents using airshipctl"
airshipctl document pull --debug
airshipctl document pull -n --debug

View File

@ -28,6 +28,5 @@ manifests:
remoteRef: ""
tag: ""
url: ${AIRSHIP_CONFIG_PRIMARY_REPO_URL}
metadataPath: manifests/metadata.yaml
subPath: ${AIRSHIP_SITE_NAME}
metadataPath: airshipctl/manifests/metadata.yaml
targetPath: ${AIRSHIP_CONFIG_MANIFEST_DIRECTORY}

View File

@ -16,11 +16,10 @@ set -xe
# The root of the manifest structure to be validated.
# This corresponds to the targetPath in an airshipctl config
: ${MANIFEST_ROOT:="${PWD}"}
: ${MANIFEST_ROOT:="$(dirname "${PWD}")"}
# The location of sites whose manifests should be validated.
# This are relative to MANIFEST_ROOT above, and correspond to
# the base of the subPath in an airshipctl config
: ${SITE_ROOT:="manifests/site"}
# This are relative to MANIFEST_ROOT above
: ${SITE_ROOT:="$(basename "${PWD}")/manifests/site"}
: ${SITE:="test-workload"}
: ${CONTEXT:="kind-airship"}
@ -78,7 +77,6 @@ manifests:
force: false
tag: ""
url: https://opendev.org/airship/treasuremap
subPath: ${SITE_ROOT}/${SITE}
targetPath: ${MANIFEST_ROOT}
EOL
}
@ -103,8 +101,6 @@ for cluster in ephemeral target; do
sed -i "s/${CONTEXT}/${CONTEXT}_${cluster}/" ${AIRSHIPKUBECONFIG}
generate_airshipconf ${cluster}
${ACTL} cluster init
# A sequential list of potential phases. A fancier attempt at this has been
# removed since it was choking in certain cases and got to be more trouble than was worth.
# This should be removed once we have a phase map that is smarter.

View File

@ -23,6 +23,7 @@ TMP_DIR=${TMP_DIR:-"$(dirname $(mktemp -u))"}
ANSIBLE_HOSTS=${ANSIBLE_HOSTS:-"${TMP_DIR}/ansible_hosts"}
PLAYBOOK_CONFIG=${PLAYBOOK_CONFIG:-"${TMP_DIR}/config.yaml"}
export AIRSHIPCTL_WS=${AIRSHIPCTL_WS:-$PWD}
export AIRSHIP_CONFIG_PRIMARY_REPO_URL=${AIRSHIP_CONFIG_PRIMARY_REPO_URL:-$PWD}
sudo --preserve-env=AIRSHIPCTL_WS ansible-playbook -i "$ANSIBLE_HOSTS" \
playbooks/airshipctl-gate-runner.yaml \

View File

@ -18,11 +18,10 @@ set -xe
# The root of the manifest structure to be validated.
# This corresponds to the targetPath in an airshipctl config
: ${MANIFEST_ROOT:="${PWD}"}
: ${MANIFEST_ROOT:="$(dirname "${PWD}")"}
# The space-separated locations of sites whose manifests should be validated.
# These are relative to MANIFEST_ROOT above, and correspond to
# the base of the subPath in an airshipctl config
: ${SITE_ROOTS:="manifests/site"}
# These are relative to MANIFEST_ROOT above
: ${SITE_ROOTS:="$(basename "${PWD}")/manifests/site"}
# get kind
echo "Fetching kind from ${KIND_URL}..."

View File

@ -91,6 +91,7 @@
- ./tools/deployment/02_install_clusterctl.sh
# 21_systemwide_executable.sh is run in the build-gate pre-run above
- ./tools/deployment/22_test_configs.sh
- ./tools/deployment/23_pull_documents.sh
- ./tools/deployment/24_build_ephemeral_iso.sh
- ./tools/deployment/25_deploy_ephemeral_node.sh
- ./tools/deployment/26_deploy_metal3_capi_ephemeral_node.sh