Browse Source

Restructure the vagrant scripts and fix a dependency error

The vagrant scripts buried so deep. Move it up a bit.
The python-kafka client 1.0 release breaks a lot of things plus
it peeks the CPU as soon as it starts. Have to make sure we use
the python-kafka version less than 1.0.0

Change-Id: Ib271dde51dcb0c5705dffbd277ba70f25cd4403b
Tong Li 3 years ago
parent
commit
771e4b5a28

+ 1
- 1
.gitignore View File

@@ -23,7 +23,7 @@ sdist/
23 23
 var/
24 24
 covhtml/
25 25
 cover/
26
-tools/vagrant/ubuntu/leapbin
26
+vagrant/leapbin
27 27
 *.egg-info/
28 28
 .installed.cfg
29 29
 *.egg

+ 6
- 4
kiloeyes/common/kafka_conn.py View File

@@ -159,10 +159,12 @@ class KafkaConnection(object):
159 159
             if not self._consumer:
160 160
                 self._init_consumer()
161 161
 
162
-            for msg in self._consumer:
163
-                if msg.message:
164
-                    LOG.debug(msg.message.value)
165
-                    yield msg
162
+            if self._consumer:
163
+                for msg in self._consumer:
164
+                    if msg.message:
165
+                        LOG.debug(msg.message.value)
166
+                        yield msg
167
+            time.sleep(self.wait_time)
166 168
         except common.OffsetOutOfRangeError:
167 169
             self._consumer.seek(0, 0)
168 170
             LOG.error('Seems consumer has been down for a long time.')

+ 1
- 1
requirements.txt View File

@@ -6,7 +6,7 @@
6 6
 falcon==0.1.9
7 7
 gunicorn==19.1.0
8 8
 iso8601>=0.1.9
9
-kafka-python>=0.8.1
9
+kafka-python<1.0.0,>=0.9.5
10 10
 oslo.config>=1.11.0
11 11
 oslo.i18n>=1.5.0
12 12
 oslo.log>=1.0.0

+ 0
- 16
tools/vagrant/ubuntu/onvm/scripts/install-java.sh View File

@@ -1,16 +0,0 @@
1
-#!/usr/bin/env bash
2
-# $1 sys_password
3
-# $2 public ip eth0
4
-
5
-java_installed=$(which java)
6
-if [ -z $java_installed ] && [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then
7
-  mkdir -p /opt/jdk
8
-  tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk
9
-  mv /opt/jdk/* /opt/jdk/jdk1.8
10
-  update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100
11
-  update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100
12
-  echo 'Java install is now complete!'
13
-else
14
-  echo 'Java has been either installed or not binary package found!'
15
-  echo 'Download java and place it in tools/vagrant/ubuntu/leapbin directory.'
16
-fi

+ 0
- 19
tools/vagrant/ubuntu/onvm/scripts/install-kiloeyes.sh View File

@@ -1,19 +0,0 @@
1
-#!/usr/bin/env bash
2
-# $1 sys_password
3
-# $2 public ip eth0
4
-
5
-source /onvm/scripts/ini-config
6
-eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
7
-
8
-wget https://bootstrap.pypa.io/ez_setup.py -O - | python
9
-apt-get update
10
-apt-get -qqy install git python-dev python-pip python-pastedeploy
11
-
12
-cd /opt/kiloeyes
13
-pip install -r requirements.txt
14
-python setup.py install
15
-
16
-# Config the kiloeyes
17
-
18
-echo 'Kiloeyes install is now complete!'
19
-

tools/vagrant/ubuntu/README.md → vagrant/README.md View File


tools/vagrant/ubuntu/Vagrantfile → vagrant/Vagrantfile View File


tools/vagrant/ubuntu/onvm/conf/ids.conf.yml → vagrant/onvm/conf/ids.conf.yml View File


tools/vagrant/ubuntu/onvm/conf/nodes.conf.yml → vagrant/onvm/conf/nodes.conf.yml View File

@@ -45,5 +45,5 @@ synchfolders:
45 45
     source: leapbin
46 46
     target: /leapbin
47 47
   kiloeyes:
48
-    source: ./../../../../kiloeyes
48
+    source: ./../../kiloeyes
49 49
     target: /opt/kiloeyes

tools/vagrant/ubuntu/onvm/scripts/ini-config → vagrant/onvm/scripts/ini-config View File


tools/vagrant/ubuntu/onvm/scripts/install-agent.sh → vagrant/onvm/scripts/install-agent.sh View File


tools/vagrant/ubuntu/onvm/scripts/install-devstack.sh → vagrant/onvm/scripts/install-devstack.sh View File


tools/vagrant/ubuntu/onvm/scripts/install-elastic.sh → vagrant/onvm/scripts/install-elastic.sh View File


+ 19
- 0
vagrant/onvm/scripts/install-java.sh View File

@@ -0,0 +1,19 @@
1
+#!/usr/bin/env bash
2
+# $1 sys_password
3
+# $2 public ip eth0
4
+
5
+java_installed=$(which java)
6
+if [ -z $java_installed ]; then
7
+  if [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then
8
+    mkdir -p /opt/jdk
9
+    tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk
10
+    mv /opt/jdk/* /opt/jdk/jdk1.8
11
+    update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100
12
+    update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100
13
+    echo 'Java install is now complete!'
14
+  else
15
+    echo 'Download java and place it in /vagrant/leapbin directory.'
16
+  fi
17
+else
18
+  echo 'Java has been installed!'
19
+fi

tools/vagrant/ubuntu/onvm/scripts/install-kafka.sh → vagrant/onvm/scripts/install-kafka.sh View File

@@ -5,10 +5,17 @@
5 5
 source /onvm/scripts/ini-config
6 6
 eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
7 7
 
8
+# Java is required, install java first
9
+source /onvm/scripts/install-java.sh
10
+
8 11
 if [ -f /leapbin/kafka_*-0.9.0.0.tgz ]; then
9 12
   tar -zxf /leapbin/kafka_*-0.9.0.0.tgz -C /opt
10 13
   mv /opt/kafka_*-0.9.0.0 /opt/kafka_0.9.0.0
11 14
 
15
+  echo '# Changes made by kiloeyes installation...' >> /opt/kafka_0.9.0.0/config/server.properties
16
+  echo 'auto.create.topics.enable=true' >> /opt/kafka_0.9.0.0/config/server.properties
17
+  echo "host.name=$2" >> /opt/kafka_0.9.0.0/config/server.properties
18
+
12 19
   /opt/kafka_0.9.0.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_0.9.0.0/config/zookeeper.properties
13 20
   sleep 2
14 21
 

+ 74
- 0
vagrant/onvm/scripts/install-kiloeyes.sh View File

@@ -0,0 +1,74 @@
1
+#!/usr/bin/env bash
2
+# $1 sys_password
3
+# $2 public ip eth0
4
+
5
+source /onvm/scripts/ini-config
6
+eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
7
+
8
+wget https://bootstrap.pypa.io/ez_setup.py -O - | python
9
+apt-get update
10
+apt-get -qqy install git python-dev python-pip python-pastedeploy
11
+
12
+cd /opt/kiloeyes
13
+pip install -r requirements.txt
14
+python setup.py install
15
+
16
+echo 'Finding IP addresses...'
17
+eval node_ip=\$leap_${leap_logical2physical_kafka}_eth0; node_ip=`echo $node_ip`
18
+kafka_ip=$node_ip
19
+eval node_ip=\$leap_${leap_logical2physical_elastic}_eth0; node_ip=`echo $node_ip`
20
+elastic_ip=$node_ip
21
+
22
+
23
+k_log_dir='/var/log/kiloeyes'
24
+k_pid_dir='/var/run/kiloeyes'
25
+mkdir -p $k_log_dir $k_pid_dir
26
+
27
+# Config the kiloeyes
28
+echo 'Config /etc/kiloeyes/kiloeyes.conf file...'
29
+iniset /etc/kiloeyes/kiloeyes.conf DEFAULT log_dir $k_log_dir
30
+iniset /etc/kiloeyes/kiloeyes.conf kafka_opts uri $kafka_ip:9092
31
+iniset /etc/kiloeyes/kiloeyes.conf es_conn uri http://$elastic_ip:9200
32
+
33
+echo 'Config /etc/kiloeyes/metrics-persister.conf...'
34
+iniset /etc/kiloeyes/metrics-persister.conf DEFAULT log_dir $k_log_dir
35
+iniset /etc/kiloeyes/metrics-persister.conf kafka_opts uri $kafka_ip:9092
36
+iniset /etc/kiloeyes/metrics-persister.conf es_conn uri http://$elastic_ip:9200
37
+
38
+echo 'Config /etc/kiloeyes/alarms-persister.conf...'
39
+iniset /etc/kiloeyes/alarms-persister.conf DEFAULT log_dir $k_log_dir
40
+iniset /etc/kiloeyes/alarms-persister.conf kafka_opts uri $kafka_ip:9092
41
+iniset /etc/kiloeyes/alarms-persister.conf es_conn uri http://$elastic_ip:9200
42
+
43
+echo 'Config /etc/kiloeyes/kiloeyes-notification-engine.conf...'
44
+iniset /etc/kiloeyes/kiloeyes-notification-engine.conf DEFAULT log_dir $k_log_dir
45
+iniset /etc/kiloeyes/kiloeyes-notification-engine.conf kafka_opts uri $kafka_ip:9092
46
+iniset /etc/kiloeyes/kiloeyes-notification-engine.conf es_conn uri http://$elastic_ip:9200
47
+
48
+echo 'Config /etc/kiloeyes/kiloeyes-threshold-engine.conf...'
49
+iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf DEFAULT log_dir $k_log_dir
50
+iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf kafka_opts uri $kafka_ip:9092
51
+iniset /etc/kiloeyes/kiloeyes-threshold-engine.conf es_conn uri http://$elastic_ip:9200
52
+
53
+echo 'Start all kiloeyes services...'
54
+
55
+gunicorn -k eventlet --worker-connections=20 --backlog=10 --paste /etc/kiloeyes/kiloeyes.ini -D
56
+
57
+start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
58
+  --pidfile $k_pid_dir/metrics-persister.pid --make-pidfile --background \
59
+  -- --config-file /etc/kiloeyes/metrics-persister.conf >> /dev/null 2>&1
60
+
61
+start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
62
+  --pidfile $k_pid_dir/alarms-persister.pid --make-pidfile --background \
63
+  -- --config-file /etc/kiloeyes/alarms-persister.conf >> /dev/null 2>&1
64
+
65
+start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
66
+  --pidfile $k_pid_dir/kiloeyes-notification-engine.pid --make-pidfile --background \
67
+  -- --config-file /etc/kiloeyes/kiloeyes-notification-engine.conf >> /dev/null 2>&1
68
+
69
+start-stop-daemon --start --quiet --chuid root --exec /usr/local/bin/kiloeyes-service \
70
+  --pidfile $k_pid_dir/kiloeyes-threshold-engine.pid --make-pidfile --background \
71
+  -- --config-file /etc/kiloeyes/kiloeyes-threshold-engine.conf >> /dev/null 2>&1
72
+
73
+echo 'Kiloeyes install is now complete!'
74
+

tools/vagrant/rebootvm.sh → vagrant/rebootvm.sh View File


tools/vagrant/stvm.sh → vagrant/stvm.sh View File


Loading…
Cancel
Save