From 1a183467e79712e9aef1f8966aea1a6fa825564b Mon Sep 17 00:00:00 2001 From: Tyler Britten Date: Tue, 5 Jul 2016 13:45:51 -0400 Subject: [PATCH] fixes refrence to wrong file 1) Updated file references to remove depricated terrform option 2) fixed the ADVERTISE_IP var to use the OpenStack private net IP instead of floating ip. proxy on compute nodes could not reach via FIP 3) added missing sed line for HYPERKUBE_VERSION on compute nodes Changed to allow all TCP ports from whitelisted IP instead of just 22 updated image reference and added overrides for kubectl and hyperkube versions Remove Outdated Add-ons Update kubelet service definition to use the newer kubelet-wrapper in CoreOS. Updated image references to use CoreOS Specifc ones found on quay.io Fixed Quay-based hyperkube versioning typo typo- missing LR Fixed wrong service account key Updated README as well md formatting fix Change-Id: I4faaf00319c332d15748f17ebda7d9b8306d7716 --- terraform/kubernetes-coreos/README.md | 40 ++++---- .../kubernetes-coreos/_securitygroups.tf | 4 +- .../files/addons/kube-dns-rc.yaml | 98 ------------------- .../files/addons/kube-dns-svc.yaml | 20 ---- .../files/addons/kube-ui-rc.yaml | 40 -------- .../files/addons/kube-ui-svc.yaml | 15 --- .../files/compute/kube-kubelet.service | 3 +- .../files/compute/kube-proxy.yaml | 2 +- .../files/controller/kube-apiserver.yaml | 6 +- .../controller/kube-controller-manager.yaml | 4 +- .../files/controller/kube-kubelet.service | 3 +- .../files/controller/kube-proxy.yaml | 2 +- .../files/controller/kube-scheduler.yaml | 2 +- terraform/kubernetes-coreos/kubernetes.tf | 13 ++- terraform/kubernetes-coreos/terraform.tfvars | 5 +- 15 files changed, 48 insertions(+), 209 deletions(-) delete mode 100644 terraform/kubernetes-coreos/files/addons/kube-dns-rc.yaml delete mode 100644 terraform/kubernetes-coreos/files/addons/kube-dns-svc.yaml delete mode 100644 terraform/kubernetes-coreos/files/addons/kube-ui-rc.yaml delete mode 100644 terraform/kubernetes-coreos/files/addons/kube-ui-svc.yaml diff --git a/terraform/kubernetes-coreos/README.md b/terraform/kubernetes-coreos/README.md index 8cf212a..6976172 100644 --- a/terraform/kubernetes-coreos/README.md +++ b/terraform/kubernetes-coreos/README.md @@ -13,7 +13,7 @@ Will install a single controller node and two compute nodes by default, can incr ## Prep - [Install Terraform](https://www.terraform.io/intro/getting-started/install.html) -- Upload a CoreOS image to glance. +- Upload a CoreOS image to glance. [Instructions Here](https://coreos.com/os/docs/latest/booting-on-openstack.html) ## Terraform @@ -36,6 +36,9 @@ Ensure that you have your Openstack credentials loaded into environment variable $ source ~/.stackrc ``` +Edit the terraform.tfvars file to put the name of your CoreOS image, OpenStack network names, etc. You'll also set the Kubernetes versions there. For the hyperkube version, you need to use the tags [here](https://quay.io/repository/coreos/hyperkube?tab=tags). + + ### Provision the Kubernetes Cluster If you wish to re-use previously generated SSL key/certs for CA and admin, simply add `-var "generate_ssl=0" \`. @@ -68,7 +71,7 @@ $ terraform apply \ -var "whitelist_network=${MY_IP}/32" ... ... -Apply complete! Resources: 12 added, 0 changed, 0 destroyed. +Apply complete! Resources: 16 added, 0 changed, 0 destroyed. The state of your infrastructure has been saved to the path below. This state is required to modify and destroy your @@ -89,8 +92,6 @@ Outputs: ``` $ ssh -A core@xx.xx.xx.xx -$ kubectl config use-context kubernetes -switched to context "kubernetes". $ kubectl config view apiVersion: v1 @@ -112,9 +113,10 @@ users: user: token: kubernetes -$ kubectl get nodes -NAME LABELS STATUS AGE -10.230.7.23 kubernetes.io/hostname=10.230.7.23 Ready 5m +$ kubectl get nodes +NAME STATUS AGE +192.168.3.197 Ready 1m +192.168.3.198 Ready 11s ``` @@ -149,18 +151,22 @@ $ kubectl delete svc my-nginx service "my-nginx" deleted ``` -### Install some addons +### Install The Dashboard Addon ``` -$ kubectl create -f /etc/kubernetes/addons/kube-ui-rc.yaml \ - --namespace=kube-system -$ kubectl create -f /etc/kubernetes/addons/kube-ui-svc.yaml \ - --namespace=kube-system -$ kubectl create -f /etc/kubernetes/addons/kube-dns-rc.yaml \ - --namespace=kube-system -$ kubectl create -f /etc/kubernetes/addons/kube-dns-svc.yaml \ - --namespace=kube-system +$ kubectl create -f https://rawgit.com/kubernetes/dashboard/master/src/deploy/kubernetes-dashboard.yaml + +deployment "kubernetes-dashboard" created +You have exposed your service on an external port on all nodes in your +cluster. If you want to expose this service to the external internet, you may +need to set up firewall rules for the service port(s) (tcp:32584) to serve traffic. + +See http://releases.k8s.io/release-1.2/docs/user-guide/services-firewalls.md for more details. + ``` +You can now access the dashboard from your whitelisted IP at: + ```http://:``` +The service port is supplied when you create the dashboard. In the example here, it was 32584. ### Destroy the cluster @@ -183,5 +189,5 @@ Do you really want to destroy? openstack_compute_secgroup_v2.kubernetes_controller: Destruction complete openstack_compute_secgroup_v2.kubernetes_internal: Destruction complete -Apply complete! Resources: 0 added, 0 changed, 12 destroyed. +Apply complete! Resources: 0 added, 0 changed, 16 destroyed. ``` diff --git a/terraform/kubernetes-coreos/_securitygroups.tf b/terraform/kubernetes-coreos/_securitygroups.tf index fcfee4c..aff9cb0 100644 --- a/terraform/kubernetes-coreos/_securitygroups.tf +++ b/terraform/kubernetes-coreos/_securitygroups.tf @@ -4,8 +4,8 @@ resource "openstack_compute_secgroup_v2" "kubernetes_controller" { description = "kubernetes Controller Security Group" rule { ip_protocol = "tcp" - from_port = "443" - to_port = "443" + from_port = "1" + to_port = "65535" cidr = "${var.whitelist_network}" } rule { diff --git a/terraform/kubernetes-coreos/files/addons/kube-dns-rc.yaml b/terraform/kubernetes-coreos/files/addons/kube-dns-rc.yaml deleted file mode 100644 index 44dc35d..0000000 --- a/terraform/kubernetes-coreos/files/addons/kube-dns-rc.yaml +++ /dev/null @@ -1,98 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-dns-v9 - namespace: kube-system - labels: - k8s-app: kube-dns - version: v9 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-dns - version: v9 - template: - metadata: - labels: - k8s-app: kube-dns - version: v9 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: etcd - image: gcr.io/google_containers/etcd:2.0.9 - resources: - limits: - cpu: 100m - memory: 50Mi - command: - - /usr/local/bin/etcd - - -data-dir - - /var/etcd/data - - -listen-client-urls - - http://127.0.0.1:2379,http://127.0.0.1:4001 - - -advertise-client-urls - - http://127.0.0.1:2379,http://127.0.0.1:4001 - - -initial-cluster-token - - skydns-etcd - volumeMounts: - - name: etcd-storage - mountPath: /var/etcd/data - - name: kube2sky - image: gcr.io/google_containers/kube2sky:1.11 - resources: - limits: - cpu: 100m - memory: 50Mi - args: - # command = "/kube2sky" - - -domain=cluster.local - - name: skydns - image: gcr.io/google_containers/skydns:2015-03-11-001 - resources: - limits: - cpu: 100m - memory: 50Mi - args: - # command = "/skydns" - - -machines=http://localhost:4001 - - -addr=0.0.0.0:53 - - -domain=cluster.local. - ports: - - containerPort: 53 - name: dns - protocol: UDP - - containerPort: 53 - name: dns-tcp - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 30 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /healthz - port: 8080 - scheme: HTTP - initialDelaySeconds: 1 - timeoutSeconds: 5 - - name: healthz - image: gcr.io/google_containers/exechealthz:1.0 - resources: - limits: - cpu: 10m - memory: 20Mi - args: - - -cmd=nslookup kubernetes.default.svc.cluster.local localhost >/dev/null - - -port=8080 - ports: - - containerPort: 8080 - protocol: TCP - volumes: - - name: etcd-storage - emptyDir: {} - dnsPolicy: Default diff --git a/terraform/kubernetes-coreos/files/addons/kube-dns-svc.yaml b/terraform/kubernetes-coreos/files/addons/kube-dns-svc.yaml deleted file mode 100644 index d9e45ae..0000000 --- a/terraform/kubernetes-coreos/files/addons/kube-dns-svc.yaml +++ /dev/null @@ -1,20 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-dns - namespace: kube-system - labels: - k8s-app: kube-dns - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeDNS" -spec: - selector: - k8s-app: kube-dns - clusterIP: CLUSTER_DNS - ports: - - name: dns - port: 53 - protocol: UDP - - name: dns-tcp - port: 53 - protocol: TCP diff --git a/terraform/kubernetes-coreos/files/addons/kube-ui-rc.yaml b/terraform/kubernetes-coreos/files/addons/kube-ui-rc.yaml deleted file mode 100644 index 87277fb..0000000 --- a/terraform/kubernetes-coreos/files/addons/kube-ui-rc.yaml +++ /dev/null @@ -1,40 +0,0 @@ -apiVersion: v1 -kind: ReplicationController -metadata: - name: kube-ui-v4 - namespace: kube-system - labels: - k8s-app: kube-ui - version: v4 - kubernetes.io/cluster-service: "true" -spec: - replicas: 1 - selector: - k8s-app: kube-ui - version: v4 - template: - metadata: - labels: - k8s-app: kube-ui - version: v4 - kubernetes.io/cluster-service: "true" - spec: - containers: - - name: kube-ui - image: gcr.io/google_containers/kube-ui:v4 - resources: - # keep request = limit to keep this container in guaranteed class - limits: - cpu: 100m - memory: 50Mi - requests: - cpu: 100m - memory: 50Mi - ports: - - containerPort: 8080 - livenessProbe: - httpGet: - path: / - port: 8080 - initialDelaySeconds: 30 - timeoutSeconds: 5 diff --git a/terraform/kubernetes-coreos/files/addons/kube-ui-svc.yaml b/terraform/kubernetes-coreos/files/addons/kube-ui-svc.yaml deleted file mode 100644 index cf960c8..0000000 --- a/terraform/kubernetes-coreos/files/addons/kube-ui-svc.yaml +++ /dev/null @@ -1,15 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: kube-ui - namespace: kube-system - labels: - k8s-app: kube-ui - kubernetes.io/cluster-service: "true" - kubernetes.io/name: "KubeUI" -spec: - selector: - k8s-app: kube-ui - ports: - - port: 80 - targetPort: 8080 diff --git a/terraform/kubernetes-coreos/files/compute/kube-kubelet.service b/terraform/kubernetes-coreos/files/compute/kube-kubelet.service index e60d3c9..d28da2d 100644 --- a/terraform/kubernetes-coreos/files/compute/kube-kubelet.service +++ b/terraform/kubernetes-coreos/files/compute/kube-kubelet.service @@ -4,7 +4,8 @@ Requires=flanneld.service After=flanneld.service [Service] -ExecStart=/usr/bin/kubelet \ +Environment=KUBELET_VERSION=HYPERKUBE_VERSION +ExecStart=/usr/lib/coreos/kubelet-wrapper \ --api_servers=https://CONTROLLER_HOST \ --register-node=true \ --allow-privileged=true \ diff --git a/terraform/kubernetes-coreos/files/compute/kube-proxy.yaml b/terraform/kubernetes-coreos/files/compute/kube-proxy.yaml index 31af3ab..5ac537c 100644 --- a/terraform/kubernetes-coreos/files/compute/kube-proxy.yaml +++ b/terraform/kubernetes-coreos/files/compute/kube-proxy.yaml @@ -7,7 +7,7 @@ spec: hostNetwork: true containers: - name: kube-proxy - image: gcr.io/google_containers/hyperkube:HYPERKUBE_VERSION + image: quay.io/coreos/hyperkube:HYPERKUBE_VERSION command: - /hyperkube - proxy diff --git a/terraform/kubernetes-coreos/files/controller/kube-apiserver.yaml b/terraform/kubernetes-coreos/files/controller/kube-apiserver.yaml index 1394955..671f881 100644 --- a/terraform/kubernetes-coreos/files/controller/kube-apiserver.yaml +++ b/terraform/kubernetes-coreos/files/controller/kube-apiserver.yaml @@ -7,7 +7,7 @@ spec: hostNetwork: true containers: - name: kube-apiserver - image: gcr.io/google_containers/hyperkube:HYPERKUBE_VERSION + image: quay.io/coreos/hyperkube:HYPERKUBE_VERSION command: - /hyperkube - apiserver @@ -17,11 +17,11 @@ spec: - --service-cluster-ip-range=PORTAL_NET - --secure-port=443 - --advertise-address=ADVERTISE_IP - #- --admission-control=NamespaceLifecycle,NamespaceExists,LimitRanger,SecurityContextDeny,ServiceAccount,ResourceQuota + - --admission-control=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota - --tls-cert-file=/etc/kubernetes/ssl/controller.pem - --tls-private-key-file=/etc/kubernetes/ssl/controller-key.pem - --client-ca-file=/etc/kubernetes/ssl/ca.pem - - --service-account-key-file=/etc/kubernetes/ssl/admin-key.pem + - --service-account-key-file=/etc/kubernetes/ssl/controller-key.pem ports: - containerPort: 443 hostPort: 443 diff --git a/terraform/kubernetes-coreos/files/controller/kube-controller-manager.yaml b/terraform/kubernetes-coreos/files/controller/kube-controller-manager.yaml index cd10678..29e5780 100644 --- a/terraform/kubernetes-coreos/files/controller/kube-controller-manager.yaml +++ b/terraform/kubernetes-coreos/files/controller/kube-controller-manager.yaml @@ -7,12 +7,12 @@ spec: hostNetwork: true containers: - name: kube-controller-manager - image: gcr.io/google_containers/hyperkube:HYPERKUBE_VERSION + image: quay.io/coreos/hyperkube:HYPERKUBE_VERSION command: - /hyperkube - controller-manager - --master=http://127.0.0.1:8080 - - --service-account-private-key-file=/etc/kubernetes/ssl/controller.pem + - --service-account-private-key-file=/etc/kubernetes/ssl/controller-key.pem - --root-ca-file=/etc/kubernetes/ssl/ca.pem livenessProbe: httpGet: diff --git a/terraform/kubernetes-coreos/files/controller/kube-kubelet.service b/terraform/kubernetes-coreos/files/controller/kube-kubelet.service index 8ae194d..c0a586e 100644 --- a/terraform/kubernetes-coreos/files/controller/kube-kubelet.service +++ b/terraform/kubernetes-coreos/files/controller/kube-kubelet.service @@ -4,7 +4,8 @@ Requires=flanneld.service After=flanneld.service [Service] -ExecStart=/usr/bin/kubelet \ +Environment=KUBELET_VERSION=HYPERKUBE_VERSION +ExecStart=/usr/lib/coreos/kubelet-wrapper \ --api_servers=http://127.0.0.1:8080 \ --register-node=false \ --allow-privileged=true \ diff --git a/terraform/kubernetes-coreos/files/controller/kube-proxy.yaml b/terraform/kubernetes-coreos/files/controller/kube-proxy.yaml index 569a1bd..4ef5cd1 100644 --- a/terraform/kubernetes-coreos/files/controller/kube-proxy.yaml +++ b/terraform/kubernetes-coreos/files/controller/kube-proxy.yaml @@ -7,7 +7,7 @@ spec: hostNetwork: true containers: - name: kube-proxy - image: gcr.io/google_containers/hyperkube:HYPERKUBE_VERSION + image: quay.io/coreos/hyperkube:HYPERKUBE_VERSION command: - /hyperkube - proxy diff --git a/terraform/kubernetes-coreos/files/controller/kube-scheduler.yaml b/terraform/kubernetes-coreos/files/controller/kube-scheduler.yaml index c7752bc..81af5f7 100644 --- a/terraform/kubernetes-coreos/files/controller/kube-scheduler.yaml +++ b/terraform/kubernetes-coreos/files/controller/kube-scheduler.yaml @@ -7,7 +7,7 @@ spec: hostNetwork: true containers: - name: kube-scheduler - image: gcr.io/google_containers/hyperkube:HYPERKUBE_VERSION + image: quay.io/coreos/hyperkube:HYPERKUBE_VERSION command: - /hyperkube - scheduler diff --git a/terraform/kubernetes-coreos/kubernetes.tf b/terraform/kubernetes-coreos/kubernetes.tf index 84a20ae..61037f0 100644 --- a/terraform/kubernetes-coreos/kubernetes.tf +++ b/terraform/kubernetes-coreos/kubernetes.tf @@ -12,14 +12,14 @@ resource "null_resource" "generate_ssl" { } resource "template_file" "discovery_url" { - template = "templates/discovery_url" + template = "${file("templates/discovery_url")}" depends_on = [ "null_resource.discovery_url_template" ] } resource "template_file" "controller_cloud_init" { - template = "templates/cloud-init" + template = "${file("templates/cloud-init")}" vars { flannel_network = "${var.flannel_network}" flannel_backend = "${var.flannel_backend}" @@ -30,8 +30,8 @@ resource "template_file" "controller_cloud_init" { } resource "template_file" "compute_cloud_init" { - template = "templates/cloud-init" - vars { + template = "${file("templates/cloud-init")}" + vars { flannel_network = "${var.flannel_network}" flannel_backend = "${var.flannel_backend}" etcd_servers = "${join(",", "${formatlist("http://%s:2379", openstack_compute_instance_v2.controller.*.network.0.fixed_ip_v4)}")}" @@ -99,7 +99,7 @@ resource "openstack_compute_instance_v2" "controller" { "sudo mv ca.pem /etc/kubernetes/ssl", "sudo chown root:core /etc/kubernetes/ssl/*; sudo chmod 0640 /etc/kubernetes/ssl/*-key.pem", "sed -i 's/MY_IP/${self.network.0.fixed_ip_v4}/' /tmp/stage/*/*", - "sed -i 's/ADVERTISE_IP/${element(openstack_networking_floatingip_v2.controller.*.address, count.index)}/' /tmp/stage/*/*", + "sed -i 's/ADVERTISE_IP/${self.network.0.fixed_ip_v4}/' /tmp/stage/*/*", "sed -i 's|PORTAL_NET|${var.portal_net}|' /tmp/stage/*/*", "sed -i 's|CLUSTER_DNS|${cidrhost(var.portal_net, 200)}|' /tmp/stage/*/*", "sed -i 's|HYPERKUBE_VERSION|${var.hyperkube_version}|' /tmp/stage/*/*", @@ -169,6 +169,7 @@ resource "openstack_compute_instance_v2" "compute" { "sed -i 's/CONTROLLER_HOST/${openstack_compute_instance_v2.controller.0.network.0.fixed_ip_v4}/' /tmp/stage/*/*", "sed -i 's|PORTAL_NET|${var.portal_net}|' /tmp/stage/*/*", "sed -i 's|CLUSTER_DNS|${cidrhost(var.portal_net, 200)}|' /tmp/stage/*/*", + "sed -i 's|HYPERKUBE_VERSION|${var.hyperkube_version}|' /tmp/stage/*/*", "sudo mkdir -p /etc/kubernetes/manifests", "sudo mv /tmp/stage/compute/*.yaml /etc/kubernetes/manifests/", "sudo mv /tmp/stage/compute/*.service /etc/systemd/system/", @@ -200,6 +201,8 @@ resource "null_resource" "controller" { " --client-certificate=/etc/kubernetes/ssl/admin.pem", "/opt/bin/kubectl config set-context ${var.kubernetes_user} --cluster=${var.cluster_name} --user=${var.kubernetes_user}", "/opt/bin/kubectl config set-context kubernetes --cluster=${var.cluster_name} --user=${var.kubernetes_user}", + "/opt/bin/kubectl config set current-context kubernetes", + "/opt/bin/kubectl create namespace kube-system", ] connection { user = "core" diff --git a/terraform/kubernetes-coreos/terraform.tfvars b/terraform/kubernetes-coreos/terraform.tfvars index f384def..86041ba 100644 --- a/terraform/kubernetes-coreos/terraform.tfvars +++ b/terraform/kubernetes-coreos/terraform.tfvars @@ -1,9 +1,10 @@ flannel_backend = "vxlan" flannel_network = "10.10.0.0/16" -kubernetes_image = "coreos-alpha-884-0-0" +kubernetes_image = "coreos-stable-1010-6-0" portal_net = "10.200.0.0/16" cluster_name = "kubestack-testing" - +kubectl_version = "v1.2.4" +hyperkube_version = "v1.2.4_coreos.1" public_key_path = "~/.ssh/id_rsa.pub" network_name = "internal" floatingip_pool = "external"