Containerize stackube controller

including
* build docker image for stackube-controller
* deploy it via kubernetes deployments

Change-Id: Id43529320ee47922fcee4d0d3d67e03eca9c8566
Implements: blueprint containerize-controller
Signed-off-by: mozhuli <21621232@zju.edu.cn>

Makefile

@@ -21,7 +21,8 @@
 GIT_HOST = git.openstack.org
 SHELL := /bin/bash
 
-KUBESTACK_VERSION=0.1
+STACKUBE_VERSION = 0.1
+KUBESTACK_VERSION = 0.1
 
 PWD := $(shell pwd)
 BASE_DIR := $(shell basename $(PWD))
@@ -68,8 +69,10 @@ install: depend
 .PHONY: docker
 docker: depend
 	cd $(DEST)
-	KUBESTACK_VERSION?=$(shell ./$(OUTPUT)/kubestack -v)
+	cp _output/kubestack deployment/kubestack
-	docker build -t stackube/kubestack:v$(KUBESTACK_VERSION) ./deployment/kubestack/
+	sudo docker build -t stackube/kubestack:v$(KUBESTACK_VERSION) ./deployment/kubestack/
+	cp _output/stackube-controller deployment/stackube-controller
+	sudo docker build -t stackube/stackube-controller:v$(STACKUBE_VERSION) ./deployment/stackube-controller/
 
 .PHONY: test
 test: test-unit

deployment/kubestack/10-kubestack.conf

@@ -3,4 +3,5 @@
    "name": "net",
    "type": "kubestack",
    "kubestack-config": "/etc/kubestack.conf"
+   "kubernetes-config": "/etc/kubernetes/admin.conf"
 }

deployment/kubestack/Dockerfile

@@ -12,14 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-FROM busybox:1.26.2
+FROM alpine:3.6
 
 MAINTAINER stackube team
 
-ADD _output/kubestack /opt/cni/bin/kubestack
+RUN apk --no-cache add bash
-ADD deployment/kubestack/install-cni.sh /install-cni.sh
+ADD kubestack /opt/cni/bin/kubestack
-ADD deployment/kubestack/10-kubestack.conf /etc/cni/net.d/10-kubestack.conf
+ADD install-cni.sh /install-cni.sh
-ADD deployment/kubestack/kubestack.conf.default /kubestack.conf.tmp
+ADD 10-kubestack.conf /etc/cni/net.d/10-kubestack.conf
+ADD kubestack.conf.default /kubestack.conf.tmp
 
 ENV PATH=$PATH:/opt/cni/bin
 VOLUME /opt/cni

deployment/kubestack/install-cni.sh

@@ -45,10 +45,10 @@ TMP_CONF='/kubestack.conf.tmp'
 # Check environment variables before any real actions.
 for i in 'AUTH_URL' 'USERNAME' 'PASSWORD' 'TENANT_NAME' 'REGION' 'EXT_NET_ID' 'PLUGIN_NAME' 'INTEGRATION_BRIDGE';do
 	if [ "${!i}" ];then
-	 echo "environment variable $i = ${!i}"
+		echo "environment variable $i = ${!i}"
 	else
-	 echo "environment variable $i has not been setted or is empty,exit..."
+		echo "environment variable $i is empty, exit..."
-	 exit
+		exit
 	fi
 done
 

deployment/kubestack/kubestack.yaml

@@ -1,115 +0,0 @@
-# This ConfigMap is used to configure a self-hosted kubestack installation.
-kind: ConfigMap
-apiVersion: v1
-metadata:
-  name: kubestack-config
-  namespace: kube-system
-data:
-  auth-url: "<Your-openstack-authentication-endpoint>"
-  username: "admin"
-  password: "password"
-  tenant-name: "admin"
-  region: "RegionOne"
-  ext-net-id: "<Your-external-network-id>"
-  plugin-name: "ovs"
-  integration-bridge: "br-int"
-
----
-
-# This manifest installs kubestack CNI plugins and network config
-# on each master and worker node in a Kubernetes cluster.
-kind: DaemonSet
-apiVersion: extensions/v1beta1
-metadata:
-  name: kubestack
-  namespace: kube-system
-  labels:
-    k8s-app: kubestack
-spec:
-  selector:
-    matchLabels:
-      k8s-app: kubestack
-  template:
-    metadata:
-      labels:
-        k8s-app: kubestack
-      annotations:
-        scheduler.alpha.kubernetes.io/critical-pod: ''
-        scheduler.alpha.kubernetes.io/tolerations: |
-          [{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
-           {"key":"CriticalAddonsOnly", "operator":"Exists"}]
-    spec:
-      hostNetwork: true
-      containers:
-        # This container installs the kubestack CNI binaries
-        # and CNI network config file on each node.
-        - name: install-cni
-          image: stackube/kubestack:v0.1
-          command: ["/install-cni.sh"]
-          env:
-            # The endpoint of openstack authentication.
-            - name: AUTH_URL
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: auth-url
-            # The username for openstack authentication.
-            - name: USERNAME
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: username
-            # The password for openstack authentication.
-            - name: PASSWORD
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: password
-            # The tenant name for openstack authentication.
-            - name: TENANT_NAME
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: tenant-name
-            # The region for openstack authentication.
-            - name: REGION
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: region
-            # The id of openstack external network.
-            - name: EXT_NET_ID
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: ext-net-id
-            # The name of openstack neutron plugin.
-            - name: PLUGIN_NAME
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: plugin-name
-            # The name of openstack neutron integration bridge.
-            - name: INTEGRATION_BRIDGE
-              valueFrom:
-                configMapKeyRef:
-                  name: kubestack-config
-                  key: integration-bridge
-          volumeMounts:
-            - mountPath: /host/opt/cni/bin
-              name: cni-bin-dir
-            - mountPath: /host/etc/cni/net.d
-              name: cni-net-dir
-            - mountPath: /host/etc
-              name: kubestack-config-dir
-      volumes:
-        # Used to install CNI.
-        - name: cni-bin-dir
-          hostPath:
-            path: /opt/cni/bin
-        - name: cni-net-dir
-          hostPath:
-            path: /etc/cni/net.d
-        - name: kubestack-config-dir
-          hostPath:
-            path: /etc

deployment/stackube-controller/Dockerfile

@@ -0,0 +1,33 @@
+# Copyright (c) 2017 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM alpine:3.6
+
+MAINTAINER stackube team
+
+RUN apk --no-cache add bash
+
+# Download and install glibc in one layer
+RUN apk --no-cache add wget ca-certificates libgcc && \
+    wget -q -O /etc/apk/keys/sgerrand.rsa.pub https://raw.githubusercontent.com/sgerrand/alpine-pkg-glibc/master/sgerrand.rsa.pub && \
+    wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-2.23-r3.apk && \
+    wget https://github.com/sgerrand/alpine-pkg-glibc/releases/download/2.23-r3/glibc-bin-2.23-r3.apk && \
+    apk add glibc-2.23-r3.apk glibc-bin-2.23-r3.apk && \
+    /usr/glibc-compat/sbin/ldconfig /lib /usr/glibc/usr/lib && \
+    apk del wget && \
+    rm -f glibc-2.23-r3.apk glibc-bin-2.23-r3.apk
+
+ADD stackube-controller /stackube-controller
+ADD start.sh /start.sh
+ADD stackube.conf.default /stackube.conf.tmp

deployment/stackube-controller/stackube.conf.default

@@ -0,0 +1,7 @@
+[Global]
+auth-url = _AUTH_URL_
+username = _USERNAME_
+password = _PASSWORD_
+tenant-name = _TENANT_NAME_
+region = _REGION_
+ext-net-id = _EXT_NET_ID_

deployment/stackube-controller/start.sh

@@ -0,0 +1,53 @@
+#!/bin/bash
+# Copyright (c) 2017 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Ensure all variables are defined.
+set -u
+
+TMP_CONF='/stackube.conf.tmp'
+# Check environment variables before any real actions.
+for i in 'AUTH_URL' 'USERNAME' 'PASSWORD' 'TENANT_NAME' 'REGION' 'EXT_NET_ID';do
+	if [ "${!i}" ];then
+		echo "environment variable $i = ${!i}"
+	else
+		echo "environment variable $i is empty, exit..."
+		exit
+	fi
+done
+
+# Insert parameters.
+sed -i s~_AUTH_URL_~${AUTH_URL:-}~g $TMP_CONF
+sed -i s/_USERNAME_/${USERNAME:-}/g $TMP_CONF
+sed -i s/_PASSWORD_/${PASSWORD:-}/g $TMP_CONF
+sed -i s/_TENANT_NAME_/${TENANT_NAME:-}/g $TMP_CONF
+sed -i s/_REGION_/${REGION:-}/g $TMP_CONF
+sed -i s/_EXT_NET_ID_/${EXT_NET_ID:-}/g $TMP_CONF
+
+# Move the temporary stackube config into place.
+STACKUBE_CONFIG_PATH='/etc/stackube.conf'
+mv $TMP_CONF $STACKUBE_CONFIG_PATH
+echo "Wrote stackube config: $(cat ${STACKUBE_CONFIG_PATH})"
+
+if [ -z $USER_CIDR ];then
+	echo "environment variable USER_CIDR is empty,use default value \"10.244.0.0/16\""
+	USER_CIDR='10.244.0.0/16'
+fi
+
+if [ -z $USER_GATEWAY ];then
+	echo "environment variable USER_GATEWAY is empty,use default value \"10.244.0.1\""
+	USER_GATEWAY='10.244.0.1'
+fi
+
+./stackube-controller --kubeconfig="" --user-cidr=${USER_CIDR} --user-gateway=${USER_GATEWAY}

deployment/stackube.yaml

@@ -0,0 +1,302 @@
+# Copyright (c) 2017 OpenStack Foundation.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This ConfigMap is used to configure stackube-controller and kubestack.
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: stackube-config
+  namespace: kube-system
+data:
+  auth-url: "<Your-openstack-authentication-endpoint>"
+  username: "admin"
+  password: "password"
+  tenant-name: "admin"
+  region: "RegionOne"
+  ext-net-id: "<Your-external-network-id>"
+  plugin-name: "ovs"
+  integration-bridge: "br-int"
+  user-cidr: "10.244.0.0/16"
+  user-gateway: "10.244.0.1"
+  kubernetes-host: "<Your-external-kubernetes-host>"
+  kubernetes-port: "<Your-external-kubernetes-port>"
+
+---
+
+# This manifest installs kubestack CNI plugins and network config
+# on each master and worker node in a Kubernetes cluster.
+kind: DaemonSet
+apiVersion: extensions/v1beta1
+metadata:
+  name: kubestack
+  namespace: kube-system
+  labels:
+    k8s-app: kubestack
+spec:
+  selector:
+    matchLabels:
+      k8s-app: kubestack
+  template:
+    metadata:
+      labels:
+        k8s-app: kubestack
+      annotations:
+        scheduler.alpha.kubernetes.io/critical-pod: ''
+        scheduler.alpha.kubernetes.io/tolerations: |
+          [{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
+           {"key":"CriticalAddonsOnly", "operator":"Exists"}]
+    spec:
+      hostNetwork: true
+      containers:
+        # This container installs the kubestack CNI binaries
+        # and CNI network config file on each node.
+        - name: install-cni
+          image: stackube/kubestack:v0.1
+          command: ["/install-cni.sh"]
+          env:
+            # The endpoint of openstack authentication.
+            - name: AUTH_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: auth-url
+            # The username for openstack authentication.
+            - name: USERNAME
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: username
+            # The password for openstack authentication.
+            - name: PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: password
+            # The tenant name for openstack authentication.
+            - name: TENANT_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: tenant-name
+            # The region for openstack authentication.
+            - name: REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: region
+            # The id of openstack external network.
+            - name: EXT_NET_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: ext-net-id
+            # The name of openstack neutron plugin.
+            - name: PLUGIN_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: plugin-name
+            # The name of openstack neutron integration bridge.
+            - name: INTEGRATION_BRIDGE
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: integration-bridge
+          volumeMounts:
+            - mountPath: /host/opt/cni/bin
+              name: cni-bin-dir
+            - mountPath: /host/etc/cni/net.d
+              name: cni-net-dir
+            - mountPath: /host/etc
+              name: kubestack-config-dir
+      volumes:
+        # Used to install CNI.
+        - name: cni-bin-dir
+          hostPath:
+            path: /opt/cni/bin
+        - name: cni-net-dir
+          hostPath:
+            path: /etc/cni/net.d
+        - name: kubestack-config-dir
+          hostPath:
+            path: /etc
+
+---
+
+# This manifest deploys the stackube-controller on Kubernetes.
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  name: stackube-controller
+  namespace: kube-system
+  labels:
+    k8s-app: stackube-controller
+  annotations:
+    scheduler.alpha.kubernetes.io/critical-pod: ''
+    scheduler.alpha.kubernetes.io/tolerations: |
+      [{"key": "dedicated", "value": "master", "effect": "NoSchedule" },
+       {"key":"CriticalAddonsOnly", "operator":"Exists"}]
+spec:
+  # The stackube-controller can only have a single active instance.
+  replicas: 1
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      name: stackube-controller
+      namespace: kube-system
+      labels:
+        k8s-app: stackube-controller
+    spec:
+      # The stackube controller run in the host network namespace for the moment
+      hostNetwork: true
+      serviceAccountName: stackube-controller
+      containers:
+        - name: stackube-controller
+          image: stackube/stackube-controller:v0.1
+          command: ["/start.sh"]
+          env:
+            # The endpoint of openstack authentication.
+            - name: AUTH_URL
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: auth-url
+            # The username for openstack authentication.
+            - name: USERNAME
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: username
+            # The password for openstack authentication.
+            - name: PASSWORD
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: password
+            # The tenant name for openstack authentication.
+            - name: TENANT_NAME
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: tenant-name
+            # The region for openstack authentication.
+            - name: REGION
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: region
+            # The id of openstack external network.
+            - name: EXT_NET_ID
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: ext-net-id
+            # The network cidr of user pod.
+            - name: USER_CIDR
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: user-cidr
+            # The network gateway of user pod.
+            - name: USER_GATEWAY
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: user-gateway
+            # The kubernetes external service host.
+            - name: KUBERNETES_SERVICE_HOST
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: kubernetes-host
+            # The kubernetes external service port.
+            - name: KUBERNETES_SERVICE_PORT
+              valueFrom:
+                configMapKeyRef:
+                  name: stackube-config
+                  key: kubernetes-port
+          volumeMounts:
+            - mountPath: /etc/ssl/certs
+              name: certs
+            - mountPath: /etc/pki
+              name: pki
+      volumes:
+        # Used to verify the keystone server.
+        - name: certs
+          hostPath:
+            path: /etc/ssl/certs
+        - name: pki
+          hostPath:
+            path: /etc/pki
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: stackube-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: stackube-controller
+subjects:
+- kind: ServiceAccount
+  name: stackube-controller
+  namespace: kube-system
+
+---
+
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRole
+metadata:
+  name: stackube-controller
+rules:
+- apiGroups:
+  - "*"
+  resources:
+  - namespaces
+  - services
+  - services/status
+  - endpoints
+  verbs:
+  - "*"
+- apiGroups:
+  - apiextensions.k8s.io
+  resources:
+  - customresourcedefinitions
+  verbs:
+  - create
+- apiGroups:
+  - rbac.authorization.k8s.io
+  resources:
+  - clusterroles
+  - clusterrolebindings
+  verbs:
+  - create
+- apiGroups:
+  - stackube.kubernetes.io
+  resources:
+  - tenants
+  - networks
+  verbs:
+  - "*"
+
+---
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: stackube-controller
+  namespace: kube-system