Add single source of Seletors

Also this commit, adds condition to setup testbundle, that ignores
directories

Commit adds new set of functions that allow easy selection of
documents from bundle for different modules.

Relates-To: #61
Closes: #61

Change-Id: I6011203a1f573cbb847e9f57c04aa60bf8278ef1

manifests/function/ephemeral/baremetal.yaml

@@ -2,11 +2,12 @@ apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-node: "true"
   name: master-0
 spec:
   online: true
   bootMACAddress: 00:3b:8b:0c:ec:8b
+  networkData: "somedata: working data"
   bmc:
     address: redfish+http://localhost:8000/redfish/v1/Systems/air-ephemeral
     credentialsName: master-0-bmc-secret

manifests/function/ephemeral/secret.yaml

@@ -2,7 +2,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-user-data: "true"
   name: node1-bmc-secret
 type: Opaque
 stringData:

pkg/bootstrap/cloudinit/cloud-init.go

@@ -7,13 +7,8 @@ import (
 )
 
 const (
-	UserDataKind           = "Secret"
-	NetworkDataKind        = "Secret"
-	BareMetalHostKind      = "BareMetalHost"
-	EphemeralHostLabel     = "airshipit.org/ephemeral-node=true"
-	EphemeralUserDataLabel = "airshipit.org/ephemeral-user-data=true"
-	networkDataKey         = "networkData"
-	userDataKey            = "userData"
+	networkDataKey = "networkData"
+	userDataKey    = "userData"
 )
 
 // GetCloudData reads YAML document input and generates cloud-init data for
@@ -36,7 +31,7 @@ func GetCloudData(docBundle document.Bundle) (userData []byte, netConf []byte, e
 
 func getUserData(docBundle document.Bundle) ([]byte, error) {
 	// find the user-data document
-	selector := document.NewSelector().ByKind(UserDataKind).ByLabel(EphemeralUserDataLabel)
+	selector := document.NewEphemeralCloudDataSelector()
 	docs, err := docBundle.Select(selector)
 	if err != nil {
 		return nil, err
@@ -62,7 +57,7 @@ func getUserData(docBundle document.Bundle) ([]byte, error) {
 
 func getNetworkData(docBundle document.Bundle) ([]byte, error) {
 	// find the baremetal host indicated as the ephemeral node
-	selector := document.NewSelector().ByKind(BareMetalHostKind).ByLabel(EphemeralHostLabel)
+	selector := document.NewEphemeralBMHSelector()
 	docs, err := docBundle.Select(selector)
 	if err != nil {
 		return nil, err
@@ -78,18 +73,11 @@ func getNetworkData(docBundle document.Bundle) ([]byte, error) {
 		bmhDoc = docs[0]
 	}
 
-	// extract the network data document pointer from the bmh document
-	netConfDocName, err := bmhDoc.GetString("spec.networkData.name")
-	if err != nil {
-		return nil, err
-	}
-	netConfDocNamespace, err := bmhDoc.GetString("spec.networkData.namespace")
-	if err != nil {
-		return nil, err
-	}
-
 	// try and find these documents in our bundle
-	selector = document.NewSelector().ByKind(NetworkDataKind).ByNamespace(netConfDocNamespace).ByName(netConfDocName)
+	selector, err = document.NewEphemeralNetworkDataSelector(bmhDoc)
+	if err != nil {
+		return nil, err
+	}
 	docs, err = docBundle.Select(selector)
 
 	if err != nil {

pkg/bootstrap/cloudinit/cloud-init_test.go

@@ -32,7 +32,7 @@ func TestGetCloudData(t *testing.T) {
 			expectedNetData:  nil,
 			expectedErr: document.ErrDocNotFound{
 				Selector: document.NewSelector().
-					ByLabel("airshipit.org/ephemeral-node=true").
+					ByLabel(document.EphemeralHostSelector).
 					ByKind("BareMetalHost"),
 			},
 		},
@@ -42,7 +42,7 @@ func TestGetCloudData(t *testing.T) {
 			expectedNetData:  nil,
 			expectedErr: document.ErrMultipleDocsFound{
 				Selector: document.NewSelector().
-					ByLabel("airshipit.org/ephemeral-node=true").
+					ByLabel(document.EphemeralHostSelector).
 					ByKind("BareMetalHost"),
 			},
 		},
@@ -82,7 +82,7 @@ func TestGetCloudData(t *testing.T) {
 			expectedErr: document.ErrDocNotFound{
 				Selector: document.NewSelector().
 					ByKind("Secret").
-					ByLabel("airshipit.org/ephemeral-user-data=true"),
+					ByLabel(document.EphemeralUserDataSelector),
 			},
 		},
 	}

pkg/cluster/initinfra/infra.go

@@ -65,8 +65,9 @@ func (infra *Infra) Deploy() error {
 		return err
 	}
 
+	selector := document.NewInintInfraSelector()
 	// TODO (kkalynovskyi) Add Selector that would filter by label indicating wether to deploy it to k8s
-	docs, err := b.GetAllDocuments()
+	docs, err := b.Select(selector)
 	if err != nil {
 		return err
 	}

pkg/document/bundle_test.go

@@ -14,7 +14,7 @@ import (
 func TestNewBundle(t *testing.T) {
 	require := require.New(t)
 
-	bundle := testutil.NewTestBundle(t, "testdata")
+	bundle := testutil.NewTestBundle(t, "testdata/common")
 	require.NotNil(bundle)
 }
 
@@ -22,7 +22,7 @@ func TestBundleDocumentFiltering(t *testing.T) {
 	assert := assert.New(t)
 	require := require.New(t)
 
-	bundle := testutil.NewTestBundle(t, "testdata")
+	bundle := testutil.NewTestBundle(t, "testdata/common")
 
 	t.Run("GetKustomizeResourceMap", func(t *testing.T) {
 		r := bundle.GetKustomizeResourceMap()

pkg/document/constants.go

@@ -2,8 +2,9 @@ package document
 
 const (
 	// Selectors
-	BaseAirshipSelector = "airshipit.org"
-	ControlNodeSelector = BaseAirshipSelector + "/node-role=control-plane"
+	BaseAirshipSelector       = "airshipit.org"
+	EphemeralHostSelector     = BaseAirshipSelector + "/ephemeral-node in (True, true)"
+	EphemeralUserDataSelector = BaseAirshipSelector + "/ephemeral-user-data in (True, true)"
 
 	// Labels
 	DeployedByLabel = BaseAirshipSelector + "/deployed"
@@ -11,3 +12,9 @@ const (
 	// Identifiers (Static label values)
 	InitinfraIdentifier = "initinfra"
 )
+
+// Kinds
+const (
+	SecretKind        = "Secret"
+	BareMetalHostKind = "BareMetalHost"
+)

pkg/document/document_test.go

@@ -21,7 +21,7 @@ func TestDocument(t *testing.T) {
 	// alanmeadows(TODO): at some point
 	// refactoring this so there isn't a reliance
 	// on a bundle might be useful
-	fSys := testutil.SetupTestFs(t, "testdata")
+	fSys := testutil.SetupTestFs(t, "testdata/common")
 	bundle, err := document.NewBundle(fSys, "/", "/")
 	require.NoError(err, "Building Bundle Failed")
 	require.NotNil(bundle)

pkg/document/selectors.go

@@ -63,3 +63,46 @@ func (s Selector) ByAnnotation(annotationSelector string) Selector {
 	}
 	return s
 }
+
+// EphemeralCloudDataSelector returns selector to get BaremetalHost for ephemeral node
+func NewEphemeralCloudDataSelector() Selector {
+	return NewSelector().ByKind(SecretKind).ByLabel(EphemeralUserDataSelector)
+}
+
+// NewEphemeralBMHSelector returns selector to get BaremetalHost for ephemeral node
+func NewEphemeralBMHSelector() Selector {
+	return NewSelector().ByKind(BareMetalHostKind).ByLabel(EphemeralHostSelector)
+}
+
+// NewEphemeralNetworkDataSelector returns selector that can be used to get secret with
+// network data bmhDoc argument is a document interface, that should hold fields
+// spec.networkData.name and spec.networkData.namespace where to find the secret,
+// if either of these fields are not defined in Document error will be returned
+func NewEphemeralNetworkDataSelector(bmhDoc Document) (Selector, error) {
+	selector := NewSelector()
+	// extract the network data document pointer from the bmh document
+	netConfDocName, err := bmhDoc.GetString("spec.networkData.name")
+	if err != nil {
+		return selector, err
+	}
+	netConfDocNamespace, err := bmhDoc.GetString("spec.networkData.namespace")
+	if err != nil {
+		return selector, err
+	}
+
+	// try and find these documents in our bundle
+	selector = selector.
+		ByKind(SecretKind).
+		ByNamespace(netConfDocNamespace).
+		ByName(netConfDocName)
+
+	return selector, nil
+}
+
+// NewInintInfraSelector returns selector of all initinfra documents
+// TODO (kkalynovskyi) add selector that would specify if document
+// should be deployed to kubernetes cluster when appropriate label
+// is added
+func NewInintInfraSelector() Selector {
+	return NewSelector()
+}

pkg/document/selectors_test.go

@@ -0,0 +1,62 @@
+package document_test
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
+	"opendev.org/airship/airshipctl/pkg/document"
+	"opendev.org/airship/airshipctl/testutil"
+)
+
+func TestSelectorsPositive(t *testing.T) {
+	bundle := testutil.NewTestBundle(t, "testdata/selectors/valid")
+
+	t.Run("TestEphemeralCloudDataSelector", func(t *testing.T) {
+		doc, err := bundle.Select(document.NewEphemeralCloudDataSelector())
+		require.NoError(t, err)
+		assert.Len(t, doc, 1)
+	})
+
+	t.Run("TestEphemeralNetworkDataSelector", func(t *testing.T) {
+		docs, err := bundle.Select(document.NewEphemeralBMHSelector())
+		require.NoError(t, err)
+		assert.Len(t, docs, 1)
+		bmhDoc := docs[0]
+		selector, err := document.NewEphemeralNetworkDataSelector(bmhDoc)
+		require.NoError(t, err)
+		assert.Equal(t, "validName", selector.Name)
+	})
+
+	t.Run("TestEphemeralCloudDataSelector", func(t *testing.T) {
+		doc, err := bundle.Select(document.NewEphemeralCloudDataSelector())
+		require.NoError(t, err)
+		assert.Len(t, doc, 1)
+	})
+}
+
+func TestSelectorsNegative(t *testing.T) {
+	// These two tests take bundle with two malformed documents
+	// each of the documents will fail at different locations providing higher
+	// test coverage
+	bundle := testutil.NewTestBundle(t, "testdata/selectors/invalid")
+
+	t.Run("TestNewEphemeralNetworkDataSelectorErr", func(t *testing.T) {
+		docs, err := bundle.Select(document.NewEphemeralBMHSelector())
+		require.NoError(t, err)
+		assert.Len(t, docs, 2)
+		bmhDoc := docs[0]
+		_, err = document.NewEphemeralNetworkDataSelector(bmhDoc)
+		assert.Error(t, err)
+	})
+
+	t.Run("TestEphemeralNetworkDataSelectorErr", func(t *testing.T) {
+		docs, err := bundle.Select(document.NewEphemeralBMHSelector())
+		require.NoError(t, err)
+		assert.Len(t, docs, 2)
+		bmhDoc := docs[1]
+		_, err = document.NewEphemeralNetworkDataSelector(bmhDoc)
+		assert.Error(t, err)
+	})
+}

pkg/document/testdata/selectors/invalid/baremetal.yaml

@@ -0,0 +1,36 @@
+## this file provides two invalid documents that would lead to errors in
+## different places, see more details in each document comment
+---
+## this document will lead to failure when trying to get string data
+## by field (spec.networkData.name) and return error
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/ephemeral-node: "true"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: ipmi://192.168.111.1:6230
+    credentialsName: master-0-bmc-secret
+  networkData:
+    namespace: validNamespace
+---
+## this document will lead to failure when trying to get string data
+## by field (spec.networkData.namespace) and return error
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/ephemeral-node: "true"
+  name: master-1
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: ipmi://192.168.111.1:6230
+    credentialsName: master-0-bmc-secret
+  networkData:
+    name: validName

pkg/document/testdata/selectors/invalid/kustomization.yaml

@@ -0,0 +1,2 @@
+resources:
+ - baremetal.yaml

pkg/document/testdata/selectors/valid/argo.yaml

@@ -0,0 +1,291 @@
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+  airshipit.org/clustertype: target
+  name: workflows.argoproj.io
+spec:
+  group: argoproj.io
+  names:
+    kind: Workflow
+    plural: workflows
+    shortNames:
+    - wf
+  scope: Namespaced
+  version: v1alpha1
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-ui
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-admin: "true"
+  name: argo-aggregate-to-admin
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-edit: "true"
+  name: argo-aggregate-to-edit
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  verbs:
+  - create
+  - delete
+  - deletecollection
+  - get
+  - list
+  - patch
+  - update
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  labels:
+    rbac.authorization.k8s.io/aggregate-to-view: "true"
+  name: argo-aggregate-to-view
+rules:
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-cluster-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  verbs:
+  - create
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - get
+  - watch
+  - list
+- apiGroups:
+  - ""
+  resources:
+  - persistentvolumeclaims
+  verbs:
+  - create
+  - delete
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  - workflows/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - delete
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-ui-cluster-role
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - pods
+  - pods/exec
+  - pods/log
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - get
+- apiGroups:
+  - argoproj.io
+  resources:
+  - workflows
+  verbs:
+  - get
+  - list
+  - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argo-cluster-role
+subjects:
+- kind: ServiceAccount
+  name: argo
+  namespace: argo
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-ui-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: argo-ui-cluster-role
+subjects:
+- kind: ServiceAccount
+  name: argo-ui
+  namespace: argo
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: workflow-controller-configmap
+---
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  name: argo-ui
+spec:
+  ports:
+  - port: 80
+    targetPort: 8001
+  selector:
+    app: argo-ui
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+    airshipit.org/random-payload: random
+  name: argo-ui
+spec:
+  selector:
+    matchLabels:
+      app: argo-ui
+  template:
+    metadata:
+      labels:
+        app: argo-ui
+    spec:
+      containers:
+      - env:
+        - name: ARGO_NAMESPACE
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.namespace
+        - name: IN_CLUSTER
+          value: "true"
+        - name: ENABLE_WEB_CONSOLE
+          value: "false"
+        - name: BASE_HREF
+          value: /
+        image: argoproj/argoui:v2.3.0
+        name: argo-ui
+      serviceAccountName: argo-ui
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  annotations:
+    airshipit.org/clustertype: target
+  labels:
+    app: workflow-controller
+    arbitrary-label: some-label
+  name: workflow-controller
+spec:
+  selector:
+    matchLabels:
+      app: workflow-controller
+  template:
+    metadata:
+      labels:
+        app: workflow-controller
+    spec:
+      containers:
+      - args:
+        - --configmap
+        - workflow-controller-configmap
+        - --executor-image
+        - argoproj/argoexec:v2.3.0
+        command:
+        - workflow-controller
+        image: argoproj/workflow-controller:v2.3.0
+        name: workflow-controller
+  serviceAccountName: argo
+...

pkg/document/testdata/selectors/valid/baremetal.yaml

@@ -0,0 +1,16 @@
+---
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  labels:
+    airshipit.org/ephemeral-node: "true"
+  name: master-0
+spec:
+  online: true
+  bootMACAddress: 00:3b:8b:0c:ec:8b
+  bmc:
+    address: ipmi://192.168.111.1:6230
+    credentialsName: master-0-bmc-secret
+  networkData:
+    name: validName
+    namespace: validNamespace

pkg/document/testdata/selectors/valid/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+ - baremetal.yaml
+ - secret.yaml
+ - argo.yaml

pkg/document/testdata/selectors/valid/secret.yaml

@@ -0,0 +1,21 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  labels:
+    airshipit.org/ephemeral-user-data: "true"
+    test: validdocset
+  name: airship-isogen-userdata
+type: Opaque
+stringData:
+  userData: cloud-init
+---
+
+apiVersion: v1
+kind: Secret
+metadata:
+  name: validName
+  namespace: validNamespace
+type: Opaque
+stringData:
+  userData: cloud-init
+

pkg/remote/remote_direct.go

@@ -83,10 +83,7 @@ func getRemoteDirectConfig(settings *environment.AirshipCTLSettings) (*config.Re
 		return nil, "", err
 	}
 
-	ls := document.ControlNodeSelector
-	selector := document.NewSelector().
-		ByGvk("", "", AirshipHostKind).
-		ByLabel(ls)
+	selector := document.NewEphemeralBMHSelector()
 	docs, err := docBundle.Select(selector)
 	if err != nil {
 		return nil, "", err

pkg/remote/testdata/base/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -3,7 +3,7 @@ apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-node: "true"
   name: master-0
 spec:
   online: true
@@ -16,7 +16,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-node: "true"
   name: master-0-bmc-secret
 type: Opaque
 data:

pkg/remote/testdata/emptyurl/manifests/site/test-site/ephemeral/baremetal.yaml

@@ -3,7 +3,7 @@ apiVersion: metal3.io/v1alpha1
 kind: BareMetalHost
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-node: "true"
   name: master-0
 spec:
   online: true
@@ -16,7 +16,7 @@ apiVersion: v1
 kind: Secret
 metadata:
   labels:
-    airshipit.org/node-role: "control-plane"
+    airshipit.org/ephemeral-node: "true"
   name: master-0-bmc-secret
 type: Opaque
 data: