Browse Source

Add procedures/scripts as starting point of multi-DC deployment

Change-Id: If455358e9164aa51b99ed3a06dd49ab2cf03d69b
changes/55/109155/7
Changbin Liu 4 years ago
parent
commit
8233e63e95

+ 10
- 0
bin/delete_docker_instances.sh View File

@@ -0,0 +1,10 @@
1
+#!/bin/bash
2
+
3
+CONTROLLERS=$@
4
+
5
+docker.io rm -f `docker.io ps -a | grep changbl | awk '{print $1}'`
6
+
7
+for i in `ovs-vsctl show | grep Port | grep pl | awk '{print $2}' | cut -d '"' -f 2`
8
+do
9
+    ovs-vsctl del-port obr2 $i
10
+done

bin/pre_install_chefserver.sh → bin/install_chefserver_deps.sh View File

@@ -14,7 +14,7 @@ sudo apt-get install -y opscode-keyring # permanent upgradeable keyring
14 14
 sudo apt-get install -y debconf-utils
15 15
 sudo apt-get -y upgrade
16 16
 
17
-sudo apt-get install -y  libgnumail-java ruby-addressable libextlib-ruby jsvc \
17
+sudo apt-get install -y libgnumail-java ruby-addressable libextlib-ruby jsvc \
18 18
   libdb5.1-java-gcj erlang-eunit libjaxp1.3-java libcommons-pool-java \
19 19
   libdb-je-java ruby-mixlib-config gcj-4.6-jre-lib libjson-ruby1.8 \
20 20
   libbcel-java erlang-crypto libgeronimo-jta-1.1-spec-java libecj-java \

bin/pre_install_libvirt.sh → bin/install_libvirt.sh View File

@@ -47,20 +47,3 @@ cd libvirt
47 47
 		   # --with-xen=yes to the command
48 48
 make -j
49 49
 sudo make install
50
-
51
-# enable libvirt for VM live migration
52
-sudo sed -i /etc/libvirt/libvirtd.conf \
53
-    -e 's/#listen_tls = 0/listen_tls = 0/g' \
54
-    -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
55
-    -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
56
-sudo sed -i /etc/init/libvirt-bin.conf \
57
-    -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
58
-sudo sed -i /etc/default/libvirt-bin \
59
-    -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
60
-
61
-# restart libvirt
62
-sudo service libvirt-bin restart
63
-
64
-# Remove the default network created by libvirt
65
-sudo virsh net-destroy default
66
-sudo virsh net-undefine default

bin/pre_install_ovs.sh → bin/install_openvswitch.sh View File


+ 58
- 0
bin/install_zookeeper.sh View File

@@ -0,0 +1,58 @@
1
+#!/bin/bash
2
+
3
+## Install ZooKeeper and OpenJDK 7
4
+
5
+VERSION=3.4.6
6
+
7
+sudo apt-get -y install openjdk-7-jdk
8
+
9
+sudo mkdir -p /opt/zookeeper
10
+sudo chmod 00777 /opt/zookeeper
11
+sudo chown root:root /opt/zookeeper
12
+
13
+cd /opt/zookeeper
14
+wget http://mirrors.gigenet.com/apache/zookeeper/zookeeper-${VERSION}/zookeeper-${VERSION}.tar.gz
15
+tar xzvf zookeeper-${VERSION}.tar.gz
16
+mv zookeeper-${VERSION} zookeeper
17
+cd zookeeper
18
+
19
+sudo mkdir -p /mnt/zookeeper
20
+sudo chmod 00777 /mnt/zookeeper
21
+sudo chown root:root /mnt/zookeeper
22
+
23
+echo "1" | tee /mnt/zookeeper/myid
24
+
25
+echo "# The number of milliseconds of each tick
26
+tickTime=2000
27
+# The number of ticks that the initial
28
+# synchronization phase can take
29
+initLimit=5
30
+# The number of ticks that can pass between
31
+# sending a request and getting an acknowledgement
32
+syncLimit=2
33
+# the directory where the snapshot is stored.
34
+# do not use /tmp for storage, /tmp here is just
35
+# example sakes.
36
+dataDir=/mnt/zookeeper
37
+# the port at which the clients will connect
38
+clientPort=2181
39
+server.1=<zk_host_1>:2888:3888
40
+server.2=<zk_host_2>:2888:3888
41
+server.3=<zk_host_3>:2888:3888
42
+# the directory where the log is stored
43
+# dataLogDir=/mnt/zookeeper
44
+#
45
+# Be sure to read the maintenance section of the
46
+# administrator guide before turning on autopurge.
47
+#
48
+# http://zookeeper.apache.org/doc/current/zookeeperAdmin.html#sc_maintenance
49
+#
50
+# The number of snapshots to retain in dataDir
51
+#autopurge.snapRetainCount=3
52
+# Purge task interval in hours
53
+# Set to 0 to disable auto purge feature
54
+#autopurge.purgeInterval=1
55
+" | tee /opt/zookeeper/zookeeper/conf/zoo.cfg
56
+
57
+# Start the service
58
+# /opt/zookeeper/zookeeper/bin/zkServer.sh start

+ 81
- 0
bin/launch_docker_instance.py View File

@@ -0,0 +1,81 @@
1
+#!/usr/bin/env python
2
+
3
+import sys
4
+import os
5
+
6
+BRIDGE = "obr2"
7
+# BRIDGE = "br2"
8
+GATEWAY = "10.2.2.10"
9
+NET_PREFIX = "10.2"
10
+MAC_ADDR_PREFIX = "52:54:00:2d:"
11
+IMAGE = "changbl/u1404-3"
12
+
13
+
14
+def int_to_hex(n):
15
+    if not 1 <= n <= 254:
16
+        raise KeyError('Wrong value: number not within [1, 254]')
17
+    s = hex(n)[2:]
18
+    if len(s) == 2:
19
+        return s
20
+    else:
21
+        return '0' + s
22
+
23
+
24
+def launch(subnet, begin, end):
25
+    for i in range(begin, end):
26
+        name = "%s.%s" % (subnet, i)
27
+        mac_addr = (MAC_ADDR_PREFIX + "%s:%s" % (
28
+            int_to_hex(subnet), int_to_hex(i)))
29
+
30
+#         cmd = """/root/pipework %s -i eth0 $(\
31
+# docker.io run --privileged=true -n=false --name=%s -d %s /usr/sbin/sshd -D\
32
+# ) %s.%s.%s/16@%s %s""" % (
33
+#     BRIDGE, name, IMAGE, NET_PREFIX, subnet, i, GATEWAY, mac_addr)
34
+
35
+        cmd = """/root/pipework %s -i eth0 $(\
36
+docker.io run -n=false --privileged=true --name=%s -d %s /usr/sbin/sshd -D\
37
+) dhcp %s""" % (BRIDGE, name, IMAGE, mac_addr)
38
+
39
+        print cmd
40
+        os.system(cmd)
41
+
42
+"""
43
+docker.io run --privileged=true -d -n=false \
44
+    -lxc-conf="lxc.network.type = veth" \
45
+    -lxc-conf="lxc.network.link = br2" \
46
+    -lxc-conf="lxc.network.flags = up" \
47
+    -lxc-conf="lxc.network.name = eth0" \
48
+    -lxc-conf="lxc.network.ipv4 = 10.2.101.3/16" \
49
+    -lxc-conf="lxc.network.hwaddr=52:54:00:2d:65:03" \
50
+    -lxc-conf="lxc.network.ipv4.gateway = 10.2.1.10" \
51
+    --name=101.3 changbl/u1401-1 /usr/sbin/sshd -D
52
+
53
+ID=$(docker.io run -n=false --name=test4 -d changbl/u1404-2 /usr/sbin/sshd -D)
54
+
55
+./pipework obr2 -i eth0 $ID dhcp 52:54:00:2d:c9:04
56
+
57
+./pipework obr2 -i eth0 $ID 10.2.201.8/16@10.2.2.10 52:54:00:2d:c9:08
58
+
59
+docker.io inspect da1fbd5421f7 | grep ID
60
+
61
+lxc-attach -n \
62
+da1fbd5421f75ef1a640019d4659489ee53faf4135f4e6feeb8872580f74549a -- /bin/bash
63
+"""
64
+#         cmd = """docker.io run --privileged=true -d -n=false \
65
+# -lxc-conf="lxc.network.type = veth" \
66
+# -lxc-conf="lxc.network.link = %s" \
67
+# -lxc-conf="lxc.network.flags = up" \
68
+# -lxc-conf="lxc.network.name = eth0" \
69
+# -lxc-conf="lxc.network.ipv4 = %s.%s.%s/16" \
70
+# -lxc-conf="lxc.network.hwaddr=%s" \
71
+# -lxc-conf="lxc.network.ipv4.gateway = %s" \
72
+# --name=%s \
73
+# %s /usr/sbin/sshd -D""" % (
74
+#     BRIDGE, NET_PREFIX, subnet, i, mac_addr, GATEWAY, name, IMAGE)
75
+#
76
+
77
+if __name__ == "__main__":
78
+    subnet = int(sys.argv[1])
79
+    begin = int(sys.argv[2])
80
+    end = int(sys.argv[3])
81
+    launch(subnet, begin, end)

+ 94
- 0
bin/launch_libvirt_instance.py View File

@@ -0,0 +1,94 @@
1
+#!/usr/bin/env python
2
+
3
+import sys
4
+import os
5
+import uuid
6
+
7
+XML_TEMPLATE = """<domain type='kvm'>
8
+  <name>%s</name>
9
+  <uuid>%s</uuid>
10
+  <memory unit='KiB'>262144</memory>
11
+  <vcpu placement='static'>1</vcpu>
12
+  <os>
13
+    <type arch='x86_64' machine='pc-i440fx-trusty'>hvm</type>
14
+    <boot dev='hd'/>
15
+  </os>
16
+  <features>
17
+    <acpi/>
18
+  </features>
19
+  <clock offset='utc'/>
20
+  <on_poweroff>destroy</on_poweroff>
21
+  <on_reboot>restart</on_reboot>
22
+  <on_crash>destroy</on_crash>
23
+  <devices>
24
+    <emulator>/usr/bin/kvm</emulator>
25
+    <disk type='file' device='disk'>
26
+      <driver name='qemu' type='qcow2'/>
27
+      <source file='/var/lib/libvirt/images/%s/image.qcow2'/>
28
+      <target dev='vda' bus='virtio'/>
29
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x05' function='0x0'/>
30
+    </disk>
31
+    <controller type='usb' index='0'>
32
+    <address type='pci' domain='0x0000' bus='0x00' slot='0x01' function='0x2'/>
33
+    </controller>
34
+    <controller type='pci' index='0' model='pci-root'/>
35
+    <interface type='bridge'>
36
+      <mac address='52:54:00:2d:%s:%s'/>
37
+      <source bridge='obr2'/>
38
+      <virtualport type='openvswitch'/>
39
+      <model type='virtio'/>
40
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x03' function='0x0'/>
41
+    </interface>
42
+    <input type='mouse' bus='ps2'/>
43
+    <input type='keyboard' bus='ps2'/>
44
+    <graphics type='vnc' port='-1' autoport='yes' listen='0.0.0.0'>
45
+      <listen type='address' address='0.0.0.0'/>
46
+    </graphics>
47
+    <video>
48
+      <model type='cirrus' vram='9216' heads='1'/>
49
+      <address type='pci' domain='0x0000' bus='0x00' slot='0x02' function='0x0'/>
50
+    </video>
51
+    <memballoon model='virtio'>
52
+    <address type='pci' domain='0x0000' bus='0x00' slot='0x04' function='0x0'/>
53
+    </memballoon>
54
+  </devices>
55
+</domain>
56
+"""
57
+
58
+
59
+def int_to_hex(n):
60
+    if not 1 <= n <= 254:
61
+        raise KeyError('Wrong value: number not within [1, 254]')
62
+    s = hex(n)[2:]
63
+    if len(s) == 2:
64
+        return s
65
+    else:
66
+        return '0' + s
67
+
68
+
69
+def gen_libvirt_xml(subnet, begin, end):
70
+    for i in range(begin, end):
71
+        name = "%s.%s" % (subnet, i)
72
+        xml = XML_TEMPLATE % (name, str(uuid.uuid4()), name,
73
+                              int_to_hex(subnet), int_to_hex(i))
74
+        print xml
75
+        fout = open('/tmp/%s.%s.xml' % (subnet, i), 'w')
76
+        fout.write(xml)
77
+        fout.close()
78
+        cmd = "mkdir -p /var/lib/libvirt/images/%s.%s/" % (subnet, i)
79
+        print cmd
80
+        os.system(cmd)
81
+        cmd = ("qemu-img create -b /var/lib/libvirt/images/base.qcow2 "
82
+               "-f qcow2 /var/lib/libvirt/images/%s.%s/image.qcow2" %
83
+               (subnet, i))
84
+        print cmd
85
+        os.system(cmd)
86
+        cmd = "virsh define /tmp/%s.%s.xml" % (subnet, i)
87
+        print cmd
88
+        os.system(cmd)
89
+
90
+if __name__ == "__main__":
91
+    subnet = int(sys.argv[1])
92
+    begin = int(sys.argv[2])
93
+    end = int(sys.argv[3])
94
+    gen_libvirt_xml(subnet, begin, end)

+ 126
- 0
bin/procedures View File

@@ -0,0 +1,126 @@
1
+(*) Physical machine
2
+
3
+# check CPU auto scaling
4
+grep -E '^model name|^cpu MHz' /proc/cpuinfo
5
+
6
+# disable CPU auto scaling if enabled
7
+for CPUFREQ in /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor; do 
8
+    [ -f $CPUFREQ ] || continue
9
+    echo -n performance > $CPUFREQ
10
+done
11
+
12
+sudo sed -i \
13
+    -e 's/echo -n ondemand > $CPUFREQ/echo -n performance > $CPUFREQ/' \
14
+    /etc/init.d/ondemand; 
15
+
16
+(*) OS image 
17
+
18
+# switch_kernel.sh (deprecated)
19
+
20
+sudo apt-get -y update && sudo apt-get -y upgrade
21
+
22
+(*) Basic software and configuration
23
+
24
+sudo apt-get -y install \
25
+    emacs \
26
+    htop \
27
+    iftop \
28
+    iotop \
29
+    sysstat \
30
+    arping \
31
+    cpu-checker \
32
+    build-essential \
33
+    git \
34
+    python-dev
35
+
36
+# enabel IP forwarding
37
+sudo sed -i 's/#net.ipv4.ip_forward=1/net.ipv4.ip_forward=1/' \
38
+    /etc/sysctl.conf && sudo sysctl net.ipv4.ip_forward=1
39
+
40
+# enable fast ssh login
41
+sudo sed -i.orig \
42
+    -e 's/#GSSAPIAuthentication no/GSSAPIAuthentication no\nUseDNS no/' \
43
+    /etc/ssh/sshd_config && sudo service ssh restart
44
+
45
+# shorten failsafe sleep time
46
+sudo sed -i \
47
+    -e 's/sleep 20/sleep 1/' \
48
+    -e 's/sleep 40/sleep 1/' \
49
+    -e 's/sleep 59/sleep 1/' \
50
+    /etc/init/failsafe.conf
51
+
52
+(*) Chef server (following steps can be done either manually, or via chef-client)
53
+
54
+# install_chefserver_deps.sh (skip-able)
55
+install_chefserver.sh
56
+configure_knife.sh
57
+setup_chef_repo.sh
58
+
59
+(*) Open vSwitch
60
+
61
+# install_openvswitch.sh (deprecated)
62
+
63
+sudo apt-get -y install openvswitch-switch
64
+
65
+(*) libvirt, kvm
66
+
67
+# install_libvirt.sh (deprecated)
68
+
69
+sudo apt-get -y install libvirt-bin kvm qemu
70
+
71
+sudo virsh net-destroy default
72
+sudo virsh net-undefine default
73
+
74
+# enable libvirt for VM live migration
75
+sudo sed -i.orig /etc/libvirt/libvirtd.conf \
76
+    -e 's/#listen_tls = 0/listen_tls = 0/g' \
77
+    -e 's/#listen_tcp = 1/listen_tcp = 1/g' \
78
+    -e 's/#auth_tcp = "sasl"/auth_tcp = "none"/g'
79
+sudo sed -i.orig /etc/init/libvirt-bin.conf \
80
+    -e 's/env libvirtd_opts="-d"/env libvirtd_opts="-d -l"/g'
81
+sudo sed -i.orig /etc/default/libvirt-bin \
82
+    -e 's/libvirtd_opts="-d"/libvirtd_opts="-d -l"/g'
83
+sudo service libvirt-bin restart
84
+
85
+# /etc/nova/nova.conf
86
+# live_migration_flag=VIR_MIGRATE_UNDEFINE_SOURCE,VIR_MIGRATE_PEER2PEER,VIR_MIGRATE_LIVE
87
+# libvirt_cpu_mode=custom
88
+# libvirt_cpu_model=kvm64
89
+
90
+(*) ZooKeeper
91
+
92
+install_zookeeper.sh
93
+
94
+(*) docker
95
+
96
+sudo apt-get -y install docker.io lxc udhcpc arping
97
+
98
+# enable lxc as backend driver
99
+sudo sed -i     /etc/default/docker.io \
100
+    -e 's/#DOCKER_OPTS="-dns 8.8.8.8 -dns 8.8.4.4"/DOCKER_OPTS="-e lxc"/'
101
+sudo service docker.io restart
102
+
103
+# download and setup pipework
104
+wget https://raw.githubusercontent.com/jpetazzo/pipework/master/pipework -P /root/
105
+sed -i.orig -e 's/docker/docker.io/' /root/pipework
106
+chmod +x /root/pipework
107
+
108
+(*) VXLAN tunnels
109
+
110
+ovs-vsctl add-br obr1; ovs-vsctl add-br obr2; ovs-vsctl add-br obr3; ovs-vsctl set Bridge obr1 stp_enable=true; ovs-vsctl set Bridge obr2 stp_enable=true; ovs-vsctl set Bridge obr3 stp_enable=true; ovs-vsctl add-port obr1 eth1p; ovs-vsctl add-port obr2 eth2p; ovs-vsctl add-port obr3 eth3p
111
+
112
+prefix_ip=10.255; postfix_ip_1=100; postfix_ip_2=64; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} obr${i}_${postfix_ip_1}_${postfix_ip_2} -- set interface obr${i}_${postfix_ip_1}_${postfix_ip_2} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
113
+
114
+DC=2; prefix_ip=135.207; postfix_ip_1=223; postfix_ip_2=152; for i in 1 2 3; do ovs-vsctl --may-exist add-port obr${i} gateway${i}_${DC} -- set interface gateway${i}_${DC} type=vxlan options:remote_ip=${prefix_ip}.${postfix_ip_1}.${postfix_ip_2} options:key=${i}; done
115
+
116
+(*) Ryu WL2 controller
117
+
118
+sudo pip install kazoo oslo.config netaddr eventlet webob routes
119
+
120
+wget https://github.com/att/ryu.git
121
+
122
+(*) Launch guest VMs (or use OpenStack)
123
+
124
+launch_libvirt_instance.py
125
+
126
+launch_docker_instance.py

bin/pre_switch_kernel.sh → bin/switch_kernel.sh View File

@@ -1,7 +1,6 @@
1 1
 #!/bin/bash
2 2
 
3
-## Switch kernel from virtual to generic, for vanilla Ubuntu 12.04
4
-## image launched instance.
3
+## Switch kernel from virtual to generic to use nested virtualization
5 4
 
6 5
 # routine
7 6
 sudo apt-get -y update

+ 28
- 21
setup.py View File

@@ -20,26 +20,33 @@ setup(
20 20
         "python-novaclient>=2.13.0",
21 21
         "IPython>=0.13.2",
22 22
     ],
23
-    data_files=[('bin', ['bin/configure_knife.sh',
24
-                         'bin/install_chefserver.sh',
25
-                         'bin/pre_install_chefserver.sh',
26
-                         'bin/pre_install_ovs.sh',
27
-                         'bin/pre_switch_kernel.sh',
28
-                         'bin/setup_chef_repo.sh',
29
-                         'bin/userdata.sh.template',
30
-                         ]),
31
-                ('inception/webui/templates/inception',
32
-                 ['inception/webui/templates/inception/detail.html',
33
-                 'inception/webui/templates/inception/_detail_log.html',
34
-                 'inception/webui/templates/inception/_detail_overview.html',
35
-                 'inception/webui/templates/inception/_flavors_and_quotas.html',
36
-                 'inception/webui/templates/inception/index.html',
37
-                 'inception/webui/templates/inception/_launch_customize_help.html',
38
-                 'inception/webui/templates/inception/_launch_details_help.html',
39
-                 'inception/webui/templates/inception/_launch_network_help.html',
40
-                 'inception/webui/templates/inception/_launch_volumes_help.html',
41
-                 'inception/webui/templates/inception/_update_networks.html',
42
-                 ]),
43
-               ],
23
+    data_files=[
24
+        ('bin', [
25
+            # 'configure_knife.sh',
26
+            # 'delete_docker_instances.sh',
27
+            # 'install_chefserver_deps.sh',
28
+            # 'install_chefserver.sh',
29
+            # 'install_libvirt.sh',
30
+            # 'install_openvswitch.sh',
31
+            # 'install_zookeeper.sh',
32
+            # 'launch_docker_instance.py',
33
+            # 'launch_libvirt_instance.py',
34
+            # 'setup_chef_repo.sh',
35
+            # 'switch_kernel.sh',
36
+            # 'userdata.sh.template',
37
+        ]),
38
+        ('inception/webui/templates/inception', [
39
+            # 'detail.html',
40
+            # '_detail_log.html',
41
+            # '_detail_overview.html',
42
+            # '_flavors_and_quotas.html',
43
+            # 'index.html',
44
+            # '_launch_customize_help.html',
45
+            # '_launch_details_help.html',
46
+            # '_launch_network_help.html',
47
+            # '_launch_volumes_help.html',
48
+            # '_update_networks.html',
49
+        ]),
50
+    ],
44 51
     scripts=['bin/orchestrator'],
45 52
 )

Loading…
Cancel
Save