diff --git a/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml b/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml new file mode 100755 index 0000000..5350b76 --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/apps/tasks/main.yml @@ -0,0 +1,42 @@ +--- +- name: Setup few variables for coreos target + set_fact: + master_ip: "{{ groups['cmasters'][0] }}" + public_ip: "{{ ansible_host }}" + when: app_env.target_os == "coreos" + +- name: Setup few variables for ubuntu target + set_fact: + master_ip: "{{ groups['umasters'][0] }}" + public_ip: "{{ ansible_host }}" + when: app_env.target_os == "ubuntu" + +- name: Setup private IP variable for master node + set_fact: + private_ip: "{{ hostvars[master_ip].inter_ip }}" + +- name: Setup first node IP when stand alone + set_fact: + first_node_ip: "{{ master_ip }}" + when: app_env.app_setting.own_cluster == True + +- name: Setup first node IP when join others + set_fact: + first_node_ip: "{{ app_env.app_setting.public_node }}" + when: app_env.app_setting.own_cluster == False + +- name: Upload app configuration files + template: + src: "roles/apps/templates/{{ item }}.j2" + dest: "/etc/kubernetes/{{ item }}.yaml" + mode: 0644 + with_items: + - cockroachdb-pod + +- name: Create the app pod + command: >- + /opt/bin/kubectl --server="{{ private_ip }}:8080" create + -f "/etc/kubernetes/{{ item }}.yaml" + with_items: + - cockroachdb-pod + diff --git a/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 b/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 new file mode 100755 index 0000000..66869b6 --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/apps/templates/cockroachdb-pod.j2 @@ -0,0 +1,45 @@ + +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-{{ inter_name }} + labels: + app: cockroachdb +spec: + nodeName: {{ inter_name }} + restartPolicy: Always + containers: + - name: cockroachdb + image: cockroachdb/cockroach + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + hostPort: 26257 + name: grpc + - containerPort: 8080 + hostPort: 8090 + name: http + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + CRARGS=("start" "--logtostderr" "--insecure" "--http-host" "0.0.0.0" ) + CRARGS+=("--advertise-host" "{{ public_ip }}" ) + CRARGS+=("--locality" "cloud={{ env }}") + CRARGS+=("--join" "{{ first_node_ip }}") + exec /cockroach/cockroach ${CRARGS[*]} + - name: loadgen + image: cockroachdb/interop-demo:1.1 + imagePullPolicy: IfNotPresent + command: + - /kv + - --read-percent=50 + - --max-rate=500 + terminationGracePeriodSeconds: 30 + volumes: + - name: datadir + hostPath: + path: /storage/cockroachdb diff --git a/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml index ef2457a..b8710b4 100755 --- a/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/post/tasks/apply.yml @@ -12,7 +12,7 @@ with_items: - dnscontroller - dashboard - - cockroachdb + - cockroachdb-init-pod - name: Label the master node command: >- @@ -26,5 +26,9 @@ with_items: - dnscontroller - dashboard - - cockroachdb +- name: Setup first cockroachdb node + command: >- + /opt/bin/kubectl --server={{ private_ip }}:8080 create + -f /etc/kubernetes/cockroachdb-init-pod.yaml + when: app_env.app_setting.own_cluster == True diff --git a/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 b/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 new file mode 100755 index 0000000..5e755fe --- /dev/null +++ b/workloads/ansible/shade/k8s/roles/post/templates/cockroachdb-init-pod.j2 @@ -0,0 +1,44 @@ +apiVersion: v1 +kind: Pod +metadata: + name: cockroachdb-{{ inter_name }} + labels: + app: cockroachdb +spec: + nodeName: {{ inter_name }} + restartPolicy: Always + containers: + - name: cockroachdb + image: cockroachdb/cockroach + imagePullPolicy: IfNotPresent + ports: + - containerPort: 26257 + hostPort: 26257 + name: grpc + - containerPort: 8080 + hostPort: 8090 + name: http + volumeMounts: + - name: datadir + mountPath: /cockroach/cockroach-data + command: + - "/bin/bash" + - "-ecx" + - | + CRARGS=("start" "--logtostderr" "--insecure" "--http-host") + CRARGS+=("0.0.0.0" "--advertise-host" "{{ public_ip }}") + CRARGS+=("--locality" "cloud={{ env }}") + exec /cockroach/cockroach ${CRARGS[*]} + - name: loadgen + image: cockroachdb/interop-demo:1.1 + imagePullPolicy: IfNotPresent + command: + - /kv + - --splits=100 + - --read-percent=50 + - --max-rate=500 + terminationGracePeriodSeconds: 30 + volumes: + - name: datadir + hostPath: + path: /storage/cockroachdb diff --git a/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml index 7b26e29..35f8142 100755 --- a/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/prepare/tasks/apply.yml @@ -63,11 +63,12 @@ - { p_min: 53, p_max: 53, dir: ingress, protocol: udp } - { p_min: 53, p_max: 53, dir: egress, protocol: udp } - { p_min: 8080, p_max: 8080, dir: ingress, protocol: tcp } + - { p_min: 8090, p_max: 8090, dir: ingress, protocol: tcp } - { p_min: 8285, p_max: 8285, dir: ingress, protocol: udp } - { p_min: 2379, p_max: 2380, dir: ingress, protocol: tcp } - { p_min: 2379, p_max: 2380, dir: egress, protocol: tcp } - { p_min: 10250, p_max: 10250, dir: ingress, protocol: tcp } - - { p_min: 30000, p_max: 32767, dir: ingress, protocol: tcp } + - { p_min: 20000, p_max: 32767, dir: ingress, protocol: tcp } - { p_min: -1, p_max: -1, dir: ingress, protocol: icmp } - { p_min: -1, p_max: -1, dir: egress, protocol: icmp } diff --git a/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml b/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml index e4d539e..c9a03de 100755 --- a/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml +++ b/workloads/ansible/shade/k8s/roles/provision/tasks/apply.yml @@ -25,6 +25,7 @@ flavor: "{{ app_env.flavor_name }}" network: "{{ app_env.private_net_name }}" floating_ip_pools: "{{ app_env.public_net_name | default(omit) }}" + floating_ips: "{{ app_env.app_setting.ips[inventory_hostname] }}" reuse_ips: False userdata: "{{ lookup('template', tp_path) }}" config_drive: "{{ app_env.config_drive | default('no') }}" diff --git a/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml b/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml index 079e371..9577835 100755 --- a/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml +++ b/workloads/ansible/shade/k8s/roles/provision/tasks/destroy.yml @@ -1,5 +1,9 @@ --- +- name: Setup release floating IP flag + set_fact: + ip_flag: "{{ app_env.app_setting.ips[inventory_hostname] == '' }}" + - name: Destroy the OpenStack VM os_server: state: "absent" @@ -9,7 +13,7 @@ validate_certs: "{{ app_env.validate_certs }}" name: "{{ inventory_hostname }}" image: "{{ app_env.image_name }}" - delete_fip: True + delete_fip: "{{ ip_flag }}" key_name: "k8s" timeout: 200 network: "{{ app_env.private_net_name }}" diff --git a/workloads/ansible/shade/k8s/site.yml b/workloads/ansible/shade/k8s/site.yml index 2436b2f..df30317 100755 --- a/workloads/ansible/shade/k8s/site.yml +++ b/workloads/ansible/shade/k8s/site.yml @@ -110,6 +110,19 @@ environment: "{{ proxy_env }}" tags: "post" +- name: Start up applications + hosts: cworkers, uworkers + gather_facts: true + user: "{{ app_env.ssh_user }}" + become: true + become_user: root + vars_files: + - "vars/{{ env }}.yml" + roles: + - apps + environment: "{{ proxy_env }}" + tags: "apps" + - name: Inform the installer hosts: cloud connection: local diff --git a/workloads/ansible/shade/k8s/vars/coreos.yml b/workloads/ansible/shade/k8s/vars/coreos.yml index 386da5e..f14377d 100755 --- a/workloads/ansible/shade/k8s/vars/coreos.yml +++ b/workloads/ansible/shade/k8s/vars/coreos.yml @@ -40,7 +40,25 @@ app_env: { dns_service_ip: "172.16.0.4", dashboard_service_ip: "172.16.0.5", + app_setting: { + public_node: "", + own_cluster: True, + # The following section shows how to pre allocate floating IPs for each + # server. If you wish not to pre allocate floating IPs or your cloud + # does not support floating IPs, leave them empty. The stack_size + # above should dictate how many worker nodes should be. For example, if + # your stack_size is 10, you will need to add worker-1 to worker-9. + ips: { + master: [], + worker-1: [], + worker-2: [] + } + }, + # The following section shows an example when use a local repo. + # If you have exported some container images such as images that being used + # by this workload, you can place the url point to tar.gz file for + # cimages_repo cimages_repo: "http://10.0.10.12/cimages.tar.gz", flannel_repo: "http://10.0.10.12/flannel-v0.7.0-linux-amd64.tar.gz", k8s_repo: "http://10.0.10.12/v1.5.4/" diff --git a/workloads/ansible/shade/k8s/vars/ubuntu.yml b/workloads/ansible/shade/k8s/vars/ubuntu.yml index 16a3398..4cf2d1c 100755 --- a/workloads/ansible/shade/k8s/vars/ubuntu.yml +++ b/workloads/ansible/shade/k8s/vars/ubuntu.yml @@ -40,7 +40,25 @@ app_env: { dns_service_ip: "172.16.0.4", dashboard_service_ip: "172.16.0.5", + app_setting: { + public_node: "", + own_cluster: True, + # The following section shows how to pre allocate floating IPs for each + # server. If you wish not to pre allocate floating IPs or your cloud + # does not support floating IPs, leave them empty. The stack_size + # above should dictate how many worker nodes should be. For example, if + # your stack_size is 10, you will need to add worker-1 to worker-9. + ips: { + master: [], + worker-1: [], + worker-2: [] + } + }, + # The following section shows an example when use a remote repo. + # If you have exported some container images such as images that being used + # by this workload, you can place the url point to tar.gz file for + # cimages_repo cimages_repo: "", flannel_repo: "https://github.com/coreos/flannel/releases/download/v0.7.0/flannel-v0.7.0-linux-amd64.tar.gz", k8s_repo: "https://storage.googleapis.com/kubernetes-release/release/v1.5.3/bin/linux/amd64/"