This commit is contained in:
Rodolfo Pacheco 2020-10-25 16:31:55 -04:00
parent 4ae1b473e5
commit 1178764fa6
45 changed files with 1665 additions and 0 deletions

BIN
bin/manager Executable file

Binary file not shown.

View File

@ -0,0 +1,26 @@
# The following manifests contain a self-signed issuer CR and a certificate CR.
# More document can be found at https://docs.cert-manager.io
# WARNING: Targets CertManager 0.11 check https://docs.cert-manager.io/en/latest/tasks/upgrading/index.html for
# breaking changes
apiVersion: cert-manager.io/v1alpha2
kind: Issuer
metadata:
name: selfsigned-issuer
namespace: system
spec:
selfSigned: {}
---
apiVersion: cert-manager.io/v1alpha2
kind: Certificate
metadata:
name: serving-cert # this name should match the one appeared in kustomizeconfig.yaml
namespace: system
spec:
# $(SERVICE_NAME) and $(SERVICE_NAMESPACE) will be substituted by kustomize
dnsNames:
- $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc
- $(SERVICE_NAME).$(SERVICE_NAMESPACE).svc.cluster.local
issuerRef:
kind: Issuer
name: selfsigned-issuer
secretName: webhook-server-cert # this secret will not be prefixed, since it's not managed by kustomize

View File

@ -0,0 +1,5 @@
resources:
- certificate.yaml
configurations:
- kustomizeconfig.yaml

View File

@ -0,0 +1,16 @@
# This configuration is for teaching kustomize how to update name ref and var substitution
nameReference:
- kind: Issuer
group: cert-manager.io
fieldSpecs:
- kind: Certificate
group: cert-manager.io
path: spec/issuerRef/name
varReference:
- kind: Certificate
group: cert-manager.io
path: spec/commonName
- kind: Certificate
group: cert-manager.io
path: spec/dnsNames

View File

@ -0,0 +1,114 @@
---
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.2.5
creationTimestamp: null
name: sipclusters.airship.airshipit.org
spec:
group: airship.airshipit.org
names:
kind: SIPCluster
listKind: SIPClusterList
plural: sipclusters
singular: sipcluster
scope: Namespaced
validation:
openAPIV3Schema:
description: SIPCluster is the Schema for the sipclusters API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: SIPClusterSpec defines the desired state of SIPCluster
properties:
infra:
description: Infra is the collection of expeected configuration details
for the multiple infrastructure services or pods that SIP manages
properties:
jumphost:
properties:
image:
type: string
nodeInterfaceId:
type: string
nodePort:
type: integer
nodelabels:
additionalProperties:
type: string
type: object
sshkey:
type: string
type: object
loadbalancer:
properties:
clusterIp:
type: string
image:
type: string
nodeInterfaceId:
type: string
nodePort:
type: integer
nodelabels:
additionalProperties:
type: string
type: object
type: object
type: object
nodes:
description: Nodes are the list of Nodes objects workers, or master
that definee eexpectations of the Tenant cluster
properties:
count:
description: VmCount
properties:
active:
description: 'INSERT ADDITIONAL STATUS FIELD - define observed
state of cluster Important: Run "make" to regenerate code
after modifying this file'
type: integer
standby:
type: integer
type: object
scheduling-constraints:
description: PlaceHolder until we define the real expected Implementation
items:
type: string
type: array
vm-flavor:
type: string
vm-role:
description: VmRoles defines the states the provisioner will report
the tenant has having.
type: string
type: object
type: object
status:
description: SIPClusterStatus defines the observed state of SIPCluster
type: object
type: object
version: v1
versions:
- name: v1
served: true
storage: true
status:
acceptedNames:
kind: ""
plural: ""
conditions: []
storedVersions: []

View File

@ -0,0 +1,21 @@
# This kustomization.yaml is not intended to be run by itself,
# since it depends on service name and namespace that are out of this kustomize package.
# It should be run by config/default
resources:
- bases/airship.airshipit.org_sipclusters.yaml
# +kubebuilder:scaffold:crdkustomizeresource
patchesStrategicMerge:
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
# patches here are for enabling the conversion webhook for each CRD
#- patches/webhook_in_sipclusters.yaml
# +kubebuilder:scaffold:crdkustomizewebhookpatch
# [CERTMANAGER] To enable webhook, uncomment all the sections with [CERTMANAGER] prefix.
# patches here are for enabling the CA injection for each CRD
#- patches/cainjection_in_sipclusters.yaml
# +kubebuilder:scaffold:crdkustomizecainjectionpatch
# the following config is for teaching kustomize how to do kustomization for CRDs.
configurations:
- kustomizeconfig.yaml

View File

@ -0,0 +1,17 @@
# This file is for teaching kustomize how to substitute name and namespace reference in CRD
nameReference:
- kind: Service
version: v1
fieldSpecs:
- kind: CustomResourceDefinition
group: apiextensions.k8s.io
path: spec/conversion/webhookClientConfig/service/name
namespace:
- kind: CustomResourceDefinition
group: apiextensions.k8s.io
path: spec/conversion/webhookClientConfig/service/namespace
create: false
varReference:
- path: metadata/annotations

View File

@ -0,0 +1,8 @@
# The following patch adds a directive for certmanager to inject CA into the CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
name: sipclusters.airship.airshipit.org

View File

@ -0,0 +1,17 @@
# The following patch enables conversion webhook for CRD
# CRD conversion requires k8s 1.13 or later.
apiVersion: apiextensions.k8s.io/v1beta1
kind: CustomResourceDefinition
metadata:
name: sipclusters.airship.airshipit.org
spec:
conversion:
strategy: Webhook
webhookClientConfig:
# this is "\n" used as a placeholder, otherwise it will be rejected by the apiserver for being blank,
# but we're going to set it later using the cert-manager (or potentially a patch if not using cert-manager)
caBundle: Cg==
service:
namespace: system
name: webhook-service
path: /convert

View File

@ -0,0 +1,70 @@
# Adds namespace to all resources.
namespace: sipcluster-system
# Value of this field is prepended to the
# names of all resources, e.g. a deployment named
# "wordpress" becomes "alices-wordpress".
# Note that it should also match with the prefix (text before '-') of the namespace
# field above.
namePrefix: sipcluster-
# Labels to add to all resources and selectors.
#commonLabels:
# someName: someValue
bases:
- ../crd
- ../rbac
- ../manager
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
#- ../webhook
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
#- ../certmanager
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
#- ../prometheus
patchesStrategicMerge:
# Protect the /metrics endpoint by putting it behind auth.
# If you want your controller-manager to expose the /metrics
# endpoint w/o any authn/z, please comment the following line.
- manager_auth_proxy_patch.yaml
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
# crd/kustomization.yaml
#- manager_webhook_patch.yaml
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'.
# Uncomment 'CERTMANAGER' sections in crd/kustomization.yaml to enable the CA injection in the admission webhooks.
# 'CERTMANAGER' needs to be enabled to use ca injection
#- webhookcainjection_patch.yaml
# the following config is for teaching kustomize how to do var substitution
vars:
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER' prefix.
#- name: CERTIFICATE_NAMESPACE # namespace of the certificate CR
# objref:
# kind: Certificate
# group: cert-manager.io
# version: v1alpha2
# name: serving-cert # this name should match the one in certificate.yaml
# fieldref:
# fieldpath: metadata.namespace
#- name: CERTIFICATE_NAME
# objref:
# kind: Certificate
# group: cert-manager.io
# version: v1alpha2
# name: serving-cert # this name should match the one in certificate.yaml
#- name: SERVICE_NAMESPACE # namespace of the service
# objref:
# kind: Service
# version: v1
# name: webhook-service
# fieldref:
# fieldpath: metadata.namespace
#- name: SERVICE_NAME
# objref:
# kind: Service
# version: v1
# name: webhook-service

View File

@ -0,0 +1,25 @@
# This patch inject a sidecar container which is a HTTP proxy for the
# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews.
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
containers:
- name: kube-rbac-proxy
image: gcr.io/kubebuilder/kube-rbac-proxy:v0.5.0
args:
- "--secure-listen-address=0.0.0.0:8443"
- "--upstream=http://127.0.0.1:8080/"
- "--logtostderr=true"
- "--v=10"
ports:
- containerPort: 8443
name: https
- name: manager
args:
- "--metrics-addr=127.0.0.1:8080"
- "--enable-leader-election"

View File

@ -0,0 +1,23 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
spec:
template:
spec:
containers:
- name: manager
ports:
- containerPort: 9443
name: webhook-server
protocol: TCP
volumeMounts:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: cert
readOnly: true
volumes:
- name: cert
secret:
defaultMode: 420
secretName: webhook-server-cert

View File

@ -0,0 +1,15 @@
# This patch add annotation to admission webhook config and
# the variables $(CERTIFICATE_NAMESPACE) and $(CERTIFICATE_NAME) will be substituted by kustomize.
apiVersion: admissionregistration.k8s.io/v1beta1
kind: MutatingWebhookConfiguration
metadata:
name: mutating-webhook-configuration
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)
---
apiVersion: admissionregistration.k8s.io/v1beta1
kind: ValidatingWebhookConfiguration
metadata:
name: validating-webhook-configuration
annotations:
cert-manager.io/inject-ca-from: $(CERTIFICATE_NAMESPACE)/$(CERTIFICATE_NAME)

View File

@ -0,0 +1,2 @@
resources:
- manager.yaml

View File

@ -0,0 +1,39 @@
apiVersion: v1
kind: Namespace
metadata:
labels:
control-plane: controller-manager
name: system
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
namespace: system
labels:
control-plane: controller-manager
spec:
selector:
matchLabels:
control-plane: controller-manager
replicas: 1
template:
metadata:
labels:
control-plane: controller-manager
spec:
containers:
- command:
- /manager
args:
- --enable-leader-election
image: controller:latest
name: manager
resources:
limits:
cpu: 100m
memory: 30Mi
requests:
cpu: 100m
memory: 20Mi
terminationGracePeriodSeconds: 10

View File

@ -0,0 +1,2 @@
resources:
- monitor.yaml

View File

@ -0,0 +1,16 @@
# Prometheus Monitor Service (Metrics)
apiVersion: monitoring.coreos.com/v1
kind: ServiceMonitor
metadata:
labels:
control-plane: controller-manager
name: controller-manager-metrics-monitor
namespace: system
spec:
endpoints:
- path: /metrics
port: https
selector:
matchLabels:
control-plane: controller-manager

View File

@ -0,0 +1,7 @@
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: metrics-reader
rules:
- nonResourceURLs: ["/metrics"]
verbs: ["get"]

View File

@ -0,0 +1,13 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: proxy-role
rules:
- apiGroups: ["authentication.k8s.io"]
resources:
- tokenreviews
verbs: ["create"]
- apiGroups: ["authorization.k8s.io"]
resources:
- subjectaccessreviews
verbs: ["create"]

View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: proxy-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: proxy-role
subjects:
- kind: ServiceAccount
name: default
namespace: system

View File

@ -0,0 +1,14 @@
apiVersion: v1
kind: Service
metadata:
labels:
control-plane: controller-manager
name: controller-manager-metrics-service
namespace: system
spec:
ports:
- name: https
port: 8443
targetPort: https
selector:
control-plane: controller-manager

View File

@ -0,0 +1,12 @@
resources:
- role.yaml
- role_binding.yaml
- leader_election_role.yaml
- leader_election_role_binding.yaml
# Comment the following 4 lines if you want to disable
# the auth proxy (https://github.com/brancz/kube-rbac-proxy)
# which protects your /metrics endpoint.
- auth_proxy_service.yaml
- auth_proxy_role.yaml
- auth_proxy_role_binding.yaml
- auth_proxy_client_clusterrole.yaml

View File

@ -0,0 +1,32 @@
# permissions to do leader election.
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: leader-election-role
rules:
- apiGroups:
- ""
resources:
- configmaps
verbs:
- get
- list
- watch
- create
- update
- patch
- delete
- apiGroups:
- ""
resources:
- configmaps/status
verbs:
- get
- update
- patch
- apiGroups:
- ""
resources:
- events
verbs:
- create

View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: leader-election-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: leader-election-role
subjects:
- kind: ServiceAccount
name: default
namespace: system

28
config/rbac/role.yaml Normal file
View File

@ -0,0 +1,28 @@
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
creationTimestamp: null
name: manager-role
rules:
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters/status
verbs:
- get
- patch
- update

View File

@ -0,0 +1,12 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: manager-rolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: manager-role
subjects:
- kind: ServiceAccount
name: default
namespace: system

View File

@ -0,0 +1,24 @@
# permissions for end users to edit sipclusters.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: sipcluster-editor-role
rules:
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters/status
verbs:
- get

View File

@ -0,0 +1,20 @@
# permissions for end users to view sipclusters.
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: sipcluster-viewer-role
rules:
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters
verbs:
- get
- list
- watch
- apiGroups:
- airship.airshipit.org
resources:
- sipclusters/status
verbs:
- get

View File

@ -0,0 +1,35 @@
apiVersion: airship.airshipit.org/v1
kind: SIPCluster
metadata:
name: sipcluster-sample
namespace: clusterA
spec:
nodes:
- vm-role: 'worker'
vm-flavor: 'foobar'
scheduling-constraints: ['per-node'] # Support dont'care option.
count:
active: 3 #driven by capi node number
standby: 1 #slew for upgrades etc
- vm-role: 'master'
vm-flavor: 'goober'
scheduling-constraints: ['per-node','per-rack']
count:
active: 3
standby: 1
infra:
loadbalancer:
clusterIp: 1.2.3.4 #<-- this aligns to the VIP IP for undercloud k8s
image: haproxy:foo
nodeLabels:
- airship-masters
nodePort: 7000
nodeInterfaceId: oam-ipv4
jump:
sshKey: rsa.... #<-- this needs to align to the ssh keys provided to cluster api objects
image: sshpod:foo
nodeLabels:
- airship-masters
nodePort: 7022
nodeInterfaceId: oam-ipv4

View File

@ -0,0 +1,6 @@
resources:
- manifests.yaml
- service.yaml
configurations:
- kustomizeconfig.yaml

View File

@ -0,0 +1,25 @@
# the following config is for teaching kustomize where to look at when substituting vars.
# It requires kustomize v2.1.0 or newer to work properly.
nameReference:
- kind: Service
version: v1
fieldSpecs:
- kind: MutatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/name
- kind: ValidatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/name
namespace:
- kind: MutatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/namespace
create: true
- kind: ValidatingWebhookConfiguration
group: admissionregistration.k8s.io
path: webhooks/clientConfig/service/namespace
create: true
varReference:
- path: metadata/annotations

View File

@ -0,0 +1,12 @@
apiVersion: v1
kind: Service
metadata:
name: webhook-service
namespace: system
spec:
ports:
- port: 443
targetPort: 9443
selector:
control-plane: controller-manager

15
hack/boilerplate.go.txt Normal file
View File

@ -0,0 +1,15 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

View File

@ -0,0 +1,36 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package v1 contains API Schema definitions for the airship v1 API group
// +kubebuilder:object:generate=true
// +groupName=airship.airshipit.org
package v1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)
var (
// GroupVersion is group version used to register these objects
GroupVersion = schema.GroupVersion{Group: "airship.airshipit.org", Version: "v1"}
// SchemeBuilder is used to add go types to the GroupVersionKind scheme
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)

View File

@ -0,0 +1,179 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
// SIPClusterSpec defines the desired state of SIPCluster
type SIPClusterSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file
// Nodes are the list of Nodes objects workers, or master that definee eexpectations
// of the Tenant cluster
Nodes *NodeSet `json:"nodes,omitempty"`
// Infra is the collection of expeected configuration details
// for the multiple infrastructure services or pods that SIP manages
//Infra *InfraSet `json:"infra,omitempty"`
// List of Infrastructure Services
InfraServices map[InfraService]InfraConfig `json:"infra"`
}
// VmRoles defines the states the provisioner will report
// the tenant has having.
type InfraService string
// Possible Infra Structure Services
const (
// LoadBalancer Service
LoadBalancerService InfraService = "LoadBalancer"
// JumpHostService Service
JumpHostService InfraService = "JumpHost"
// AuthHostService Service
AuthHostService InfraService = "AuthHost"
)
// NodeSet are the the list of Nodes objects workers,
// or master that definee eexpectations
// for the Tenant Clusters
// Includes artifacts to associate with each defined namespace
// Such as :
// - Roles for the Nodes
// - Flavor for theh Nodes image
// - Scheduling expectations
// - Scale of the group of Nodes
//
type NodeSet struct {
// VmRole is either Control or Workers
VmRole VmRoles `json:"vm-role,omitempty"`
// VmFlavor is essentially a Flavor label identifying the
// type of Node that meets the construction reqirements
VmFlavor string `json:"vm-flavor,omitempty"`
// PlaceHolder until we define the real expected
// Implementation
// Scheduling define constraints the allows the SIP Scheduler
// to identify the required BMH's to allow CAPI to build a cluster
Scheduling []string `json:"scheduling-constraints,omitempty"`
// Count defines the scale expectations for the Nodes
Count *VmCount `json:"count,omitempty"`
}
/*
type InfraSet struct {
// Load Balancer Infrastrcture Service that fronts a Tenant Cluster
// Kubernetes API Servers
// These provides the configuration expectations
LoadBalancer *LoadBalancerConfig `json:"loadbalancer,omitempty"`
// Jump Host is an Operation POD that allows operators to
// Manage the VM Node infrastcuture
// These provides the configuration expectations
JumpHost *JumpHostConfig `json:"jumphost,omitempty"`
// AuthHost is an Service POD that provides an OIDC provider
// interface to the Tenant Cluster
// These provides the configuration expectations
AuthHost *AuthConfig `json:"authhost,omitempty"`
}
*/
/*
type LoadBalancerConfig struct {
ClusterIp string `json:"clusterIp,omitempty"`
Config *InfraConfig `json:"config,omitempty"`
}
type JumpHostConfig struct {
SshKey string `json:"sshkey,omitempty"`
Config *InfraConfig `json:"config,omitempty"`
}
type AuthConfig struct {
Config *InfraConfig `json:"config,omitempty"`
}
*/
type InfraConfig struct {
OptionalData *OptsConfig `json:"optional,omitempty"`
Image string `json:"image,omitempty"`
NodeLabels map[string]string `json:"nodelabels,omitempty"`
NodePort int `json:"nodePort,omitempty"`
NodeInterface string `json:"nodeInterfaceId,omitempty"`
}
type OptsConfig struct {
SshKey string `json:"sshkey,omitempty"`
ClusterIp string `json:"clusterIp,omitempty"`
}
// VmRoles defines the states the provisioner will report
// the tenant has having.
type VmRoles string
// Possible Node or VM Roles for a Tenant
const (
// VmMaster means the state is unknown
VmMaster VmRoles = "Master"
// VmWorker means the state is unknown
VmWorker VmRoles = "Worker"
)
// VmCount
type VmCount struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
Active int `json:"active,omitempty"`
Standby int `json:"standby,omitempty"`
}
// SIPClusterStatus defines the observed state of SIPCluster
type SIPClusterStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
}
// +kubebuilder:object:root=true
// SIPCluster is the Schema for the sipclusters API
type SIPCluster struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec SIPClusterSpec `json:"spec,omitempty"`
Status SIPClusterStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// SIPClusterList contains a list of SIPCluster
type SIPClusterList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []SIPCluster `json:"items"`
}
func init() {
SchemeBuilder.Register(&SIPCluster{}, &SIPClusterList{})
}

View File

@ -0,0 +1,208 @@
// +build !ignore_autogenerated
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by controller-gen. DO NOT EDIT.
package v1
import (
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *InfraConfig) DeepCopyInto(out *InfraConfig) {
*out = *in
if in.OptionalData != nil {
in, out := &in.OptionalData, &out.OptionalData
*out = new(OptsConfig)
**out = **in
}
if in.NodeLabels != nil {
in, out := &in.NodeLabels, &out.NodeLabels
*out = make(map[string]string, len(*in))
for key, val := range *in {
(*out)[key] = val
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InfraConfig.
func (in *InfraConfig) DeepCopy() *InfraConfig {
if in == nil {
return nil
}
out := new(InfraConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *NodeSet) DeepCopyInto(out *NodeSet) {
*out = *in
if in.Scheduling != nil {
in, out := &in.Scheduling, &out.Scheduling
*out = make([]string, len(*in))
copy(*out, *in)
}
if in.Count != nil {
in, out := &in.Count, &out.Count
*out = new(VmCount)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NodeSet.
func (in *NodeSet) DeepCopy() *NodeSet {
if in == nil {
return nil
}
out := new(NodeSet)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *OptsConfig) DeepCopyInto(out *OptsConfig) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OptsConfig.
func (in *OptsConfig) DeepCopy() *OptsConfig {
if in == nil {
return nil
}
out := new(OptsConfig)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SIPCluster) DeepCopyInto(out *SIPCluster) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Status = in.Status
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SIPCluster.
func (in *SIPCluster) DeepCopy() *SIPCluster {
if in == nil {
return nil
}
out := new(SIPCluster)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *SIPCluster) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SIPClusterList) DeepCopyInto(out *SIPClusterList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]SIPCluster, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SIPClusterList.
func (in *SIPClusterList) DeepCopy() *SIPClusterList {
if in == nil {
return nil
}
out := new(SIPClusterList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *SIPClusterList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SIPClusterSpec) DeepCopyInto(out *SIPClusterSpec) {
*out = *in
if in.Nodes != nil {
in, out := &in.Nodes, &out.Nodes
*out = new(NodeSet)
(*in).DeepCopyInto(*out)
}
if in.InfraServices != nil {
in, out := &in.InfraServices, &out.InfraServices
*out = make(map[InfraService]InfraConfig, len(*in))
for key, val := range *in {
(*out)[key] = *val.DeepCopy()
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SIPClusterSpec.
func (in *SIPClusterSpec) DeepCopy() *SIPClusterSpec {
if in == nil {
return nil
}
out := new(SIPClusterSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *SIPClusterStatus) DeepCopyInto(out *SIPClusterStatus) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SIPClusterStatus.
func (in *SIPClusterStatus) DeepCopy() *SIPClusterStatus {
if in == nil {
return nil
}
out := new(SIPClusterStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *VmCount) DeepCopyInto(out *VmCount) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VmCount.
func (in *VmCount) DeepCopy() *VmCount {
if in == nil {
return nil
}
out := new(VmCount)
in.DeepCopyInto(out)
return out
}

View File

@ -0,0 +1,142 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"context"
"github.com/go-logr/logr"
// "github.com/prometheus/common/log"
"k8s.io/apimachinery/pkg/runtime"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
airshipv1 "sipcluster/pkg/api/v1"
airshipsvc "sipcluster/pkg/services"
airshipvms "sipcluster/pkg/vbmh"
)
// SIPClusterReconciler reconciles a SIPCluster object
type SIPClusterReconciler struct {
client.Client
Log logr.Logger
Scheme *runtime.Scheme
}
// +kubebuilder:rbac:groups=airship.airshipit.org,resources=sipclusters,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=airship.airshipit.org,resources=sipclusters/status,verbs=get;update;patch
func (r *SIPClusterReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
ctx := context.Background()
log := r.Log.WithValues("sipcluster", req.NamespacedName)
// Lets retrieve the SIPCluster
sip := airshipv1.SIPCluster{}
if err := r.Get(ctx, req.NamespacedName, &sip); err != nil {
log.Error(err, "unable to fetch SIP Cluster")
// we'll ignore not-found errors, since they can't be fixed by an immediate
// requeue (we'll need to wait for a new notification), and we can get them
// on deleted requests.
return ctrl.Result{}, nil
}
// machines
err, machines := r.gatherVM(sip)
if err != nil {
return ctrl.Result{}, err
}
// Do we extra the information in a generic way
// So that LB and Jump Host can both leverage
err, machineData := r.extractFromVM(sip, machines)
if err != nil {
return ctrl.Result{}, err
}
err = r.deployInfra(sip, machines, machineData)
if err != nil {
return ctrl.Result{}, err
}
return ctrl.Result{}, nil
}
func (r *SIPClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&airshipv1.SIPCluster{}).
Complete(r)
}
/*
### Gather Phase
#### Identity BMH VM's
- Gather BMH's that meet the criteria expected for the groups
- Check for existing labeled BMH's
- Complete the expected scheduling contraints :
- If master
- collect into list of bmh's to label
- If worker
- collect into list of bmh's to label
#### Extract Info from Identified BMH
- identify and extract the IP address ands other info as needed (***)
- Use it as part of the service infrastucture configuration
- At this point I have a list of BMH's, and I have the extrapolated data I need for configuring services.
### Service Infrastructure Deploy Phase
- Create or Updated the [LB|admin pod] with the appropriate configuration
### Label Phase
- Label the collected hosts.
- At this point SIPCluster is done processing a given CR, and can move on the next.
*/
// machines
func (r *SIPClusterReconciler) gatherVM(sip airshipv1.SIPCluster) (error, airshipvms.MachineList) {
// 1- Let me retrieve all BMH that are unlabeled or already labeled with the target Tenant/CNF
// 2- Let me now select the one's that meet teh scheduling criteria
// If I schedule successfully then
// If Not complete schedule , then throw an error.
return nil, airshipvms.MachineList{}
}
/*
*/
func (r *SIPClusterReconciler) extractFromVM(sip airshipv1.SIPCluster, machines airshipvms.MachineList) (error, airshipvms.MachineData) {
return nil, airshipvms.MachineData{}
}
func (r *SIPClusterReconciler) deployInfra(sip airshipv1.SIPCluster, machines airshipvms.MachineList, machineData airshipvms.MachineData) error {
for sName, sConfig := range sip.Spec.InfraServices {
// Instantiate
service, err := airshipsvc.NewService(sName, sConfig)
if err != nil {
return err
}
// Lets deploy the Service
err = service.Deploy(machines, machineData)
if err != nil {
return err
}
// Did it deploy correctly, letcs check
err = service.Validate()
if err != nil {
return err
}
}
return nil
}

View File

@ -0,0 +1,81 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package controllers
import (
"path/filepath"
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"k8s.io/client-go/kubernetes/scheme"
"k8s.io/client-go/rest"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/envtest"
"sigs.k8s.io/controller-runtime/pkg/envtest/printer"
logf "sigs.k8s.io/controller-runtime/pkg/log"
"sigs.k8s.io/controller-runtime/pkg/log/zap"
airshipv1 "sipcluster/pkg/api/v1"
// +kubebuilder:scaffold:imports
)
// These tests use Ginkgo (BDD-style Go testing framework). Refer to
// http://onsi.github.io/ginkgo/ to learn more about Ginkgo.
var cfg *rest.Config
var k8sClient client.Client
var testEnv *envtest.Environment
func TestAPIs(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecsWithDefaultAndCustomReporters(t,
"Controller Suite",
[]Reporter{printer.NewlineReporter{}})
}
var _ = BeforeSuite(func(done Done) {
logf.SetLogger(zap.LoggerTo(GinkgoWriter, true))
By("bootstrapping test environment")
testEnv = &envtest.Environment{
CRDDirectoryPaths: []string{filepath.Join("..", "config", "crd", "bases")},
}
var err error
cfg, err = testEnv.Start()
Expect(err).ToNot(HaveOccurred())
Expect(cfg).ToNot(BeNil())
err = airshipv1.AddToScheme(scheme.Scheme)
Expect(err).NotTo(HaveOccurred())
// +kubebuilder:scaffold:scheme
k8sClient, err = client.New(cfg, client.Options{Scheme: scheme.Scheme})
Expect(err).ToNot(HaveOccurred())
Expect(k8sClient).ToNot(BeNil())
close(done)
}, 60)
var _ = AfterSuite(func() {
By("tearing down the test environment")
err := testEnv.Stop()
Expect(err).ToNot(HaveOccurred())
})

32
pkg/services/authhost.go Normal file
View File

@ -0,0 +1,32 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
airshipv1 "sipcluster/pkg/api/v1"
)
type AuthHost struct {
Service
}
func newAuthHost(infraCfg airshipv1.InfraConfig) InfrastructureService {
return &AuthHost{
Service: Service{
serviceName: airshipv1.AuthHostService,
config: infraCfg,
},
}
}

25
pkg/services/errors.go Normal file
View File

@ -0,0 +1,25 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import ()
// ErrAuthTypeNotSupported is returned when wrong AuthType is provided
type ErrInfraServiceNotSupported struct {
}
func (e ErrInfraServiceNotSupported) Error() string {
return "Invalid Infrastructure Service"
}

View File

@ -0,0 +1,61 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
airshipv1 "sipcluster/pkg/api/v1"
airshipvms "sipcluster/pkg/vbmh"
)
// Infrastructure interface should be implemented by each Tenant Required
// Infrastructure Service
// Init : prepares the Service
// Deploy : deploys the service
// Validate : will make sure that the deployment is successfull
type InfrastructureService interface {
//
Deploy(airshipvms.MachineList, airshipvms.MachineData) error
Validate() error
}
// Generic Service Factory
type Service struct {
serviceName airshipv1.InfraService
config airshipv1.InfraConfig
}
func (s *Service) Deploy(machines airshipvms.MachineList, machineData airshipvms.MachineData) error {
// do something, might decouple this a bit
return nil
}
func (s *Service) Validate() error {
// do something, might decouple this a bit
return nil
}
// Service Factory
func NewService(infraName airshipv1.InfraService, infraCfg airshipv1.InfraConfig) (InfrastructureService, error) {
if infraName == airshipv1.LoadBalancerService {
return newLoadBalancer(infraCfg), nil
} else if infraName == airshipv1.JumpHostService {
return newJumpHost(infraCfg), nil
} else if infraName == airshipv1.AuthHostService {
return newAuthHost(infraCfg), nil
}
return nil, ErrInfraServiceNotSupported{}
}

51
pkg/services/jumphost.go Normal file
View File

@ -0,0 +1,51 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
airshipv1 "sipcluster/pkg/api/v1"
)
type JumpHost struct {
Service
}
func newJumpHost(infraCfg airshipv1.InfraConfig) InfrastructureService {
return &JumpHost{
Service: Service{
serviceName: airshipv1.JumpHostService,
config: infraCfg,
},
}
}
/*
The SIP Cluster operator will manufacture a jump host pod specifically for this tenant cluster. Much like we did above for master nodes by extracting IP addresses, we would need to extract the `oam-ipv4` ip address for all nodes and create a configmap to bind mount into the pod so it understands what host IPs represent the clusters.
The expectation is the Jump Pod runs `sshd` protected by `uam` to allow operators to SSH directly to the Jump Pod and authenticate via UAM to immediately access their cluster.
It will provide the following functionality over SSH:
- The Jump Pod will be fronted by a `NodePort` service to allow incoming ssh.
- The Jump Pod will be UAM secured (for SSH)
- Bind mount in cluster-specific SSH key for cluster
- Ability to Power Cycle the cluster VMs
- A kubectl binary and kubeconfig (cluster-admin) for the cluster
- SSH access to the cluster node VMs
- Libvirt console logs for the VMs
- We will secure libvirt with tls and provide keys to every jump host with curated interfaces to extract logs remotely for all VMs for their clusters.
*/

View File

@ -0,0 +1,74 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package services
import (
airshipv1 "sipcluster/pkg/api/v1"
)
type LoadBalancer struct {
Service
}
func newLoadBalancer(infraCfg airshipv1.InfraConfig) InfrastructureService {
return &LoadBalancer{
Service: Service{
serviceName: airshipv1.LoadBalancerService,
config: infraCfg,
},
}
}
/*
:::warning
For the loadbalanced interface a **static asignment** via network data is required. For now, we will not support updates to this field without manual intervention. In other words, there is no expectation that the SIP operator watches `BareMetalHost` objects and reacts to changes in the future. The expectation would instead to re-deliver the `SIPCluster` object to force a no-op update to load balancer configuration is updated.
:::
By extracting these IP address from the appropriate/defined interface for each master node, we can build our loadbalancer service endpoint list to feed to haproxy. In other words, the SIP Cluster will now manufacture an haproxy configuration file that directs traffic to all IP endpoints found above over port 6443. For example:
``` gotpl
global
log /dev/stdout local0
log /dev/stdout local1 notice
daemon
defaults
log global
mode tcp
option dontlognull
# TODO: tune these
timeout connect 5000
timeout client 50000
timeout server 50000
frontend control-plane
bind *:6443
default_backend kube-apiservers
backend kube-apiservers
option httpchk GET /healthz
{% for i in range(1, number_masters) %}
server {{ cluster_name }}-{{ i }} {{ vm_master_ip }}:6443 check check-ssl verify none
{% end %}
```
This will be saved as a configmap and mounted into the cluster specific haproxy daemonset across all undercloud control nodes.
We will then create a Kubernetes NodePort `Service` that will direct traffic on the infrastructure `nodePort` defined in the SIP Cluster definition to these haproxy workloads.
At this point, the SIP Cluster controller can now label the VMs appropriately so they'll be scheduled by the Cluster-API process.
*/

25
pkg/vbmh/machinedata.go Normal file
View File

@ -0,0 +1,25 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vms
import (
// metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
)
type MachineData struct {
MachData *Machine
}

56
pkg/vbmh/machines.go Normal file
View File

@ -0,0 +1,56 @@
/*
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package vms
import (
metal3 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
)
// ScheduledState
type ScheduledState string
// Possible Node or VM Roles for a Tenant
const (
// ToBeScheduled means that the VM was identified by the scheduler to be selected
ToBeScheduled ScheduledState = "Selected"
// Scheduled means the BMH / VM already has a label implying it
// was previously scheduled
Scheduled ScheduledState = "Scheduled"
// NotScheduled, This BMH was not selected to be scheduled.
// Either because we didnt meet the criteria
// or simple we reached the count
NotScheduled ScheduledState = "NotSelected"
)
// MAchine represents an individual BMH CR, and teh appropriate
// attributes required to manage the SIP Cluster scheduling and
// rocesing needs about thhem
type Machine struct {
Bmh metal3.BareMetalHostSpec
AcheduleState ScheduledState
// scheduleLabels
ScheduleLabels map[string]string
}
type MachineList struct {
bmhs []Machine
// I might have some index here of MAchines that have been actualy Selected to be Scheduled.
}