Changed vagrant file so that installation can be easier

Change-Id: I986c293d9f0a63cbc63c52ce68b1d821b681f48c
This commit is contained in:
Tong Li 2016-03-29 23:36:34 -04:00
parent b540bbb550
commit 1eeb1de33a
19 changed files with 556 additions and 647 deletions

4
.gitignore vendored
View File

@ -9,6 +9,7 @@ __pycache__/
.Python
.testrepository/
.venv/
.vagrant/
env/
bin/
build/
@ -22,10 +23,11 @@ sdist/
var/
covhtml/
cover/
tools/vagrant/ubuntu/leapbin
*.egg-info/
.installed.cfg
*.egg
.DS_Store
# Installer logs
pip-log.txt
pip-delete-this-directory.txt

23
ChangeLog Normal file → Executable file
View File

@ -1,18 +1,7 @@
CHANGES
=======
kiloeyes (1.0)
* ES now returns timestamp as milliseconds vs seconds
* enable bulk message post on persister
* Added more instructions
* bulk insert can not be done implicitly
* fix the partitions data type error
* Updated the installation instructions
* added more instruction on how to create an all-in-one kiloeyes
* unit test passed for py27
* Add Vagrant sample file to ease development environment bootstrap
* Make the server more flexible with configuration files
* remove old openstack incubator project reference
* remove old oslo.config and use new oslo_config
* Make minor modifications in the README
* seeding the project
* Added .gitreview
* Initial project setup
Choose framework of wsgiref, pasteDeploy, falcon.
The server will be wsgiref server like any other OpenStack server
Use PasteDeploy to allow WSGI pipelines
Use Falcon framework to implement ReSTful API services

View File

@ -1,330 +0,0 @@
#!/usr/bin/env python
# Copyright (c) 2013, Nebula, Inc.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# Colorizer Code is borrowed from Twisted:
# Copyright (c) 2001-2010 Twisted Matrix Laboratories.
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""Display a subunit stream through a colorized unittest test runner."""
import heapq
import sys
import unittest
import subunit
import testtools
class _AnsiColorizer(object):
"""A colorizer is an object that loosely wraps around a stream.
That allows callers to write text to the stream in a particular color.
Colorizer classes must implement C{supported()} and C{write(text, color)}.
"""
_colors = dict(black=30, red=31, green=32, yellow=33,
blue=34, magenta=35, cyan=36, white=37)
def __init__(self, stream):
self.stream = stream
@staticmethod
def supported(stream=sys.stdout):
"""Method that checks if the current terminal supports coloring.
Returns True or False.
"""
if not stream.isatty():
return False # auto color only on TTYs
try:
import curses
except ImportError:
return False
else:
try:
try:
return curses.tigetnum("colors") > 2
except curses.error:
curses.setupterm()
return curses.tigetnum("colors") > 2
except Exception:
# guess false in case of error
return False
def write(self, text, color):
"""Write the given text to the stream in the given color.
@param text: Text to be written to the stream.
@param color: A string label for a color. e.g. 'red', 'white'.
"""
color = self._colors[color]
self.stream.write('\x1b[%s;1m%s\x1b[0m' % (color, text))
class _Win32Colorizer(object):
"""See _AnsiColorizer docstring."""
def __init__(self, stream):
import win32console
red, green, blue, bold = (win32console.FOREGROUND_RED,
win32console.FOREGROUND_GREEN,
win32console.FOREGROUND_BLUE,
win32console.FOREGROUND_INTENSITY)
self.stream = stream
self.screenBuffer = win32console.GetStdHandle(
win32console.STD_OUT_HANDLE)
self._colors = {
'normal': red | green | blue,
'red': red | bold,
'green': green | bold,
'blue': blue | bold,
'yellow': red | green | bold,
'magenta': red | blue | bold,
'cyan': green | blue | bold,
'white': red | green | blue | bold
}
@staticmethod
def supported(stream=sys.stdout):
try:
import win32console
screenBuffer = win32console.GetStdHandle(
win32console.STD_OUT_HANDLE)
except ImportError:
return False
import pywintypes
try:
screenBuffer.SetConsoleTextAttribute(
win32console.FOREGROUND_RED |
win32console.FOREGROUND_GREEN |
win32console.FOREGROUND_BLUE)
except pywintypes.error:
return False
else:
return True
def write(self, text, color):
color = self._colors[color]
self.screenBuffer.SetConsoleTextAttribute(color)
self.stream.write(text)
self.screenBuffer.SetConsoleTextAttribute(self._colors['normal'])
class _NullColorizer(object):
"""See _AnsiColorizer docstring."""
def __init__(self, stream):
self.stream = stream
@staticmethod
def supported(stream=sys.stdout):
return True
def write(self, text, color):
self.stream.write(text)
def get_elapsed_time_color(elapsed_time):
if elapsed_time > 1.0:
return 'red'
elif elapsed_time > 0.25:
return 'yellow'
else:
return 'green'
class SubunitTestResult(testtools.TestResult):
def __init__(self, stream, descriptions, verbosity):
super(SubunitTestResult, self).__init__()
self.stream = stream
self.showAll = verbosity > 1
self.num_slow_tests = 10
self.slow_tests = [] # this is a fixed-sized heap
self.colorizer = None
# NOTE(vish): reset stdout for the terminal check
stdout = sys.stdout
sys.stdout = sys.__stdout__
for colorizer in [_Win32Colorizer, _AnsiColorizer, _NullColorizer]:
if colorizer.supported():
self.colorizer = colorizer(self.stream)
break
sys.stdout = stdout
self.start_time = None
self.last_time = {}
self.results = {}
self.last_written = None
def _writeElapsedTime(self, elapsed):
color = get_elapsed_time_color(elapsed)
self.colorizer.write(" %.2f" % elapsed, color)
def _addResult(self, test, *args):
try:
name = test.id()
except AttributeError:
name = 'Unknown.unknown'
test_class, test_name = name.rsplit('.', 1)
elapsed = (self._now() - self.start_time).total_seconds()
item = (elapsed, test_class, test_name)
if len(self.slow_tests) >= self.num_slow_tests:
heapq.heappushpop(self.slow_tests, item)
else:
heapq.heappush(self.slow_tests, item)
self.results.setdefault(test_class, [])
self.results[test_class].append((test_name, elapsed) + args)
self.last_time[test_class] = self._now()
self.writeTests()
def _writeResult(self, test_name, elapsed, long_result, color,
short_result, success):
if self.showAll:
self.stream.write(' %s' % str(test_name).ljust(66))
self.colorizer.write(long_result, color)
if success:
self._writeElapsedTime(elapsed)
self.stream.writeln()
else:
self.colorizer.write(short_result, color)
def addSuccess(self, test):
super(SubunitTestResult, self).addSuccess(test)
self._addResult(test, 'OK', 'green', '.', True)
def addFailure(self, test, err):
if test.id() == 'process-returncode':
return
super(SubunitTestResult, self).addFailure(test, err)
self._addResult(test, 'FAIL', 'red', 'F', False)
def addError(self, test, err):
super(SubunitTestResult, self).addFailure(test, err)
self._addResult(test, 'ERROR', 'red', 'E', False)
def addSkip(self, test, reason=None, details=None):
super(SubunitTestResult, self).addSkip(test, reason, details)
self._addResult(test, 'SKIP', 'blue', 'S', True)
def startTest(self, test):
self.start_time = self._now()
super(SubunitTestResult, self).startTest(test)
def writeTestCase(self, cls):
if not self.results.get(cls):
return
if cls != self.last_written:
self.colorizer.write(cls, 'white')
self.stream.writeln()
for result in self.results[cls]:
self._writeResult(*result)
del self.results[cls]
self.stream.flush()
self.last_written = cls
def writeTests(self):
time = self.last_time.get(self.last_written, self._now())
if not self.last_written or (self._now() - time).total_seconds() > 2.0:
diff = 3.0
while diff > 2.0:
classes = self.results.keys()
oldest = min(classes, key=lambda x: self.last_time[x])
diff = (self._now() - self.last_time[oldest]).total_seconds()
self.writeTestCase(oldest)
else:
self.writeTestCase(self.last_written)
def done(self):
self.stopTestRun()
def stopTestRun(self):
for cls in list(self.results.iterkeys()):
self.writeTestCase(cls)
self.stream.writeln()
self.writeSlowTests()
def writeSlowTests(self):
# Pare out 'fast' tests
slow_tests = [item for item in self.slow_tests
if get_elapsed_time_color(item[0]) != 'green']
if slow_tests:
slow_total_time = sum(item[0] for item in slow_tests)
slow = ("Slowest %i tests took %.2f secs:"
% (len(slow_tests), slow_total_time))
self.colorizer.write(slow, 'yellow')
self.stream.writeln()
last_cls = None
# sort by name
for elapsed, cls, name in sorted(slow_tests,
key=lambda x: x[1] + x[2]):
if cls != last_cls:
self.colorizer.write(cls, 'white')
self.stream.writeln()
last_cls = cls
self.stream.write(' %s' % str(name).ljust(68))
self._writeElapsedTime(elapsed)
self.stream.writeln()
def printErrors(self):
if self.showAll:
self.stream.writeln()
self.printErrorList('ERROR', self.errors)
self.printErrorList('FAIL', self.failures)
def printErrorList(self, flavor, errors):
for test, err in errors:
self.colorizer.write("=" * 70, 'red')
self.stream.writeln()
self.colorizer.write(flavor, 'red')
self.stream.writeln(": %s" % test.id())
self.colorizer.write("-" * 70, 'red')
self.stream.writeln()
self.stream.writeln("%s" % err)
test = subunit.ProtocolTestCase(sys.stdin, passthrough=None)
if sys.version_info[0:2] <= (2, 6):
runner = unittest.TextTestRunner(verbosity=2)
else:
runner = unittest.TextTestRunner(
verbosity=2, resultclass=SubunitTestResult)
if runner.run(test).wasSuccessful():
exit_code = 0
else:
exit_code = 1
sys.exit(exit_code)

View File

@ -1,73 +0,0 @@
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#
# Copyright 2010 OpenStack Foundation
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Installation script for Glance's development virtualenv
"""
from __future__ import print_function
import os
import sys
import install_venv_common as install_venv # noqa
def print_help():
help = """
Glance development environment setup is complete.
Glance development uses virtualenv to track and manage Python dependencies
while in development and testing.
To activate the Glance virtualenv for the extent of your current shell session
you can run:
$ source .venv/bin/activate
Or, if you prefer, you can run commands in the virtualenv on a case by case
basis by running:
$ tools/with_venv.sh <your command>
Also, make test will automatically use the virtualenv.
"""
print(help)
def main(argv):
root = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
venv = os.path.join(root, '.venv')
pip_requires = os.path.join(root, 'requirements.txt')
test_requires = os.path.join(root, 'test-requirements.txt')
py_version = "python%s.%s" % (sys.version_info[0], sys.version_info[1])
project = 'Kiloeyes'
install = install_venv.InstallVenv(root, venv, pip_requires, test_requires,
py_version, project)
options = install.parse_args(argv)
install.check_python_version()
install.check_dependencies()
install.create_virtualenv(no_site_packages=options.no_site_packages)
install.install_dependencies()
install.run_command([os.path.join(venv, 'bin/python'),
'setup.py', 'develop'])
print_help()
if __name__ == '__main__':
main(sys.argv)

View File

@ -1,172 +0,0 @@
# Copyright 2013 OpenStack Foundation
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Provides methods needed by installation script for OpenStack development
virtual environments.
Since this script is used to bootstrap a virtualenv from the system's Python
environment, it should be kept strictly compatible with Python 2.6.
Synced in from openstack-common
"""
from __future__ import print_function
import optparse
import os
import subprocess
import sys
class InstallVenv(object):
def __init__(self, root, venv, requirements,
test_requirements, py_version,
project):
self.root = root
self.venv = venv
self.requirements = requirements
self.test_requirements = test_requirements
self.py_version = py_version
self.project = project
def die(self, message, *args):
print(message % args, file=sys.stderr)
sys.exit(1)
def check_python_version(self):
if sys.version_info < (2, 6):
self.die("Need Python Version >= 2.6")
def run_command_with_code(self, cmd, redirect_output=True,
check_exit_code=True):
"""Runs a command in an out-of-process shell.
Returns the output of that command. Working directory is self.root.
"""
if redirect_output:
stdout = subprocess.PIPE
else:
stdout = None
proc = subprocess.Popen(cmd, cwd=self.root, stdout=stdout)
output = proc.communicate()[0]
if check_exit_code and proc.returncode != 0:
self.die('Command "%s" failed.\n%s', ' '.join(cmd), output)
return (output, proc.returncode)
def run_command(self, cmd, redirect_output=True, check_exit_code=True):
return self.run_command_with_code(cmd, redirect_output,
check_exit_code)[0]
def get_distro(self):
if (os.path.exists('/etc/fedora-release') or
os.path.exists('/etc/redhat-release')):
return Fedora(
self.root, self.venv, self.requirements,
self.test_requirements, self.py_version, self.project)
else:
return Distro(
self.root, self.venv, self.requirements,
self.test_requirements, self.py_version, self.project)
def check_dependencies(self):
self.get_distro().install_virtualenv()
def create_virtualenv(self, no_site_packages=True):
"""Creates the virtual environment and installs PIP.
Creates the virtual environment and installs PIP only into the
virtual environment.
"""
if not os.path.isdir(self.venv):
print('Creating venv...', end=' ')
if no_site_packages:
self.run_command(['virtualenv', '-q', '--no-site-packages',
self.venv])
else:
self.run_command(['virtualenv', '-q', self.venv])
print('done.')
else:
print("venv already exists...")
pass
def pip_install(self, *args):
self.run_command(['tools/with_venv.sh',
'pip', 'install', '--upgrade'] + list(args),
redirect_output=False)
def install_dependencies(self):
print('Installing dependencies with pip (this can take a while)...')
# First things first, make sure our venv has the latest pip and
# setuptools and pbr
self.pip_install('pip>=1.4')
self.pip_install('setuptools')
self.pip_install('pbr')
self.pip_install('-r', self.requirements, '-r', self.test_requirements)
def parse_args(self, argv):
"""Parses command-line arguments."""
parser = optparse.OptionParser()
parser.add_option('-n', '--no-site-packages',
action='store_true',
help="Do not inherit packages from global Python "
"install.")
return parser.parse_args(argv[1:])[0]
class Distro(InstallVenv):
def check_cmd(self, cmd):
return bool(self.run_command(['which', cmd],
check_exit_code=False).strip())
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if self.check_cmd('easy_install'):
print('Installing virtualenv via easy_install...', end=' ')
if self.run_command(['easy_install', 'virtualenv']):
print('Succeeded')
return
else:
print('Failed')
self.die('ERROR: virtualenv not found.\n\n%s development'
' requires virtualenv, please install it using your'
' favorite package management tool' % self.project)
class Fedora(Distro):
"""This covers all Fedora-based distributions.
Includes: Fedora, RHEL, CentOS, Scientific Linux
"""
def check_pkg(self, pkg):
return self.run_command_with_code(['rpm', '-q', pkg],
check_exit_code=False)[1] == 0
def install_virtualenv(self):
if self.check_cmd('virtualenv'):
return
if not self.check_pkg('python-virtualenv'):
self.die("Please install 'python-virtualenv'.")
super(Fedora, self).install_virtualenv()

7
tools/vagrant/rebootvm.sh Executable file
View File

@ -0,0 +1,7 @@
VBoxManage snapshot h2-compute01 restore "Snapshot 3"
VBoxManage snapshot h2-nova restore "Snapshot 3"
VBoxManage snapshot h2-controller restore "Snapshot 3"
vboxmanage startvm h2-compute01 --type headless
vboxmanage startvm h2-nova --type headless
vboxmanage startvm h2-controller --type headless

3
tools/vagrant/stvm.sh Executable file
View File

@ -0,0 +1,3 @@
vboxmanage controlvm h2-compute01 acpipowerbutton
vboxmanage controlvm h2-nova acpipowerbutton
vboxmanage controlvm h2-controller acpipowerbutton

View File

@ -2,49 +2,59 @@
# vi: set ft=ruby :
require 'yaml'
config_file=File.expand_path(File.join(File.dirname(__FILE__), 'ubuntu-virtualbox.yml'))
settings=YAML.load_file(config_file)
nodes = YAML.load_file("onvm/conf/nodes.conf.yml")
ids = YAML.load_file("onvm/conf/ids.conf.yml")
VM_MEM = settings['vm_memory']
KAFKA_URL = settings['kafka_url']
# ElasticSearch URL
ES_URL = settings['elasticsearch_url']
Vagrant.configure("2") do |config|
config.vm.box = "tknerr/managed-server-dummy"
config.ssh.username = ids['username']
config.ssh.password = ids['password']
# All Vagrant configuration is done below. The "2" in Vagrant.configure
# configures the configuration version (we support older styles for
# backwards compatibility). Please don't change it unless you know what
# you're doing.
Vagrant.configure(2) do |config|
# The most common configuration options are documented and commented below.
# For a complete reference, please see the online documentation at
# https://docs.vagrantup.com.
config.vm.synced_folder ".", "/vagrant", disabled: true
config.vm.synced_folder "onvm", "/onvm", disabled: false, create: true
# Every Vagrant development environment requires a box. You can search for
# boxes at https://atlas.hashicorp.com/search.
config.vm.box = "ubuntu/trusty64"
config.vm.provider "virtualbox" do |vb|
# # Display the VirtualBox GUI when booting the machine
# vb.gui = true
#
# # Customize the amount of memory on the VM:
vb.memory = VM_MEM
config.vm.network "private_network", ip: "192.168.0.10"
end
config.vm.provision "shell", inline: <<-SHELL
echo "deb http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | sudo tee /etc/apt/sources.list.d/webupd8team-java.list
echo "deb-src http://ppa.launchpad.net/webupd8team/java/ubuntu trusty main" | sudo tee -a /etc/apt/sources.list.d/webupd8team-java.list
sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys EEA14886
sudo apt-get update
sudo apt-get install -y git build-essential python-pip python-dev vim
sudo apt-get install -y virtualenvwrapper
cd /vagrant && sudo git clone https://github.com/openstack/kiloeyes.git
echo debconf shared/accepted-oracle-license-v1-1 select true | sudo debconf-set-selections
echo debconf shared/accepted-oracle-license-v1-1 seen true | sudo debconf-set-selections
sudo apt-get install -y oracle-java8-installer
cd /vagrant && wget #{KAFKA_URL} -O - | tar -zxvf - -C /vagrant
wget #{ES_URL} -O /tmp/es.deb && sudo dpkg -i /tmp/es.deb
echo source /usr/share/virtualenvwrapper/virtualenvwrapper_lazy.sh | tee -a ~/.profile
SHELL
sync_cfg = nodes['synchfolders']
lnodes = nodes['ctlnodes']
if lnodes
lnodes.each do |key|
config.vm.define "#{key}" do |node|
nodekey = nodes['logical2physical'][key]
if sync_cfg[key]
node.vm.synced_folder sync_cfg[key]['source'],
sync_cfg[key]['target'], disabled: false, create: true
end
node.vm.provider :managed do |managed|
managed.server = nodes[nodekey]['eth0']
end
node.vm.provision "#{key}-install", type: "shell" do |s|
s.path = "onvm/scripts/install-" + key + ".sh"
s.args = ids['sys_password'] + ' ' + nodes[nodekey]['eth0']
end
end
end
end
# agent node setup
lnodes = nodes['agentes']
if lnodes
lnodes.each do |key|
config.vm.define "#{key}" do |node|
node.vm.provider :managed do |managed|
managed.server = nodes[key]['eth0']
end
node.vm.provision "#{key}-install", type: "shell" do |s|
s.path = "onvm/scripts/install-agent.sh"
s.args = ids['sys_password'] + " " + nodes[key]['eth0']
end
end
end
end
end

View File

@ -0,0 +1,7 @@
---
username: "root"
password: "time4fun"
sys_password: "secret"

View File

@ -0,0 +1,49 @@
---
repo:
host_name: repo.leap.dev
eth0: 192.168.1.88
eth1: 192.168.1.88
controller:
host_name: controller.leap.dev
eth0: 192.168.1.90
devstack:
host_name: devstack.leap.dev
eth0: 192.168.1.93
agent01:
host_name: agent01.leap.dev
eth0: 192.168.1.88
logical2physical:
kiloeyes: controller
elastic: controller
kafka: controller
devstack: devstack
# Define how many logical nodes and the sequence of the installation
ctlnodes:
- devstack
- elastic
- kafka
- kiloeyes
agentes:
- agent01
uselocalrepo: yes
aptopt: --force-yes
# The nodes should be a list of logical name
# The folder should be a local directory start from the project root
synchfolders:
elastic:
source: leapbin
target: /leapbin
kiloeyes:
source: ./../../../../kiloeyes
target: /opt/kiloeyes

View File

@ -0,0 +1,330 @@
#!/bin/bash
#
# **inc/ini-config** - Configuration/INI functions
#
# Support for manipulating INI-style configuration files
#
# These functions have no external dependencies and no side-effects
# Save trace setting
INC_CONF_TRACE=$(set +o | grep xtrace)
set +o xtrace
# Config Functions
# ================
# Append a new option in an ini file without replacing the old value
# iniadd [-sudo] config-file section option value1 value2 value3 ...
function iniadd {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="-sudo "
shift
fi
local file=$1
local section=$2
local option=$3
shift 3
local values="$(iniget_multiline $file $section $option) $@"
iniset_multiline $sudo $file $section $option $values
$xtrace
}
# Comment an option in an INI file
# inicomment [-sudo] config-file section option
function inicomment {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
$sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ s|^\($option[ \t]*=.*$\)|#\1|" "$file"
$xtrace
}
# Get an option from an INI file
# iniget config-file section option
function iniget {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local file=$1
local section=$2
local option=$3
local line
line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file")
echo ${line#*=}
$xtrace
}
# Get a multiple line option from an INI file
# iniget_multiline config-file section option
function iniget_multiline {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local file=$1
local section=$2
local option=$3
local values
values=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { s/^$option[ \t]*=[ \t]*//gp; }" "$file")
echo ${values}
$xtrace
}
# Determinate is the given option present in the INI file
# ini_has_option config-file section option
function ini_has_option {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local file=$1
local section=$2
local option=$3
local line
line=$(sed -ne "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ p; }" "$file")
$xtrace
[ -n "$line" ]
}
# Add another config line for a multi-line option.
# It's normally called after iniset of the same option and assumes
# that the section already exists.
#
# Note that iniset_multiline requires all the 'lines' to be supplied
# in the argument list. Doing that will cause incorrect configuration
# if spaces are used in the config values.
#
# iniadd_literal [-sudo] config-file section option value
function iniadd_literal {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
local value=$4
if [[ -z $section || -z $option ]]; then
$xtrace
return
fi
# Add it
$sudo sed -i -e "/^\[$section\]/ a\\
$option = $value
" "$file"
$xtrace
}
# Remove an option from an INI file
# inidelete [-sudo] config-file section option
function inidelete {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
if [[ -z $section || -z $option ]]; then
$xtrace
return
fi
# Remove old values
$sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ d; }" "$file"
$xtrace
}
# Set an option in an INI file
# iniset [-sudo] config-file section option value
# - if the file does not exist, it is created
function iniset {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
local value=$4
if [[ -z $section || -z $option ]]; then
$xtrace
return
fi
if ! grep -q "^\[$section\]" "$file" 2>/dev/null; then
# Add section at the end
echo -e "\n[$section]" | $sudo tee --append "$file" > /dev/null
fi
if ! ini_has_option "$file" "$section" "$option"; then
# Add it
$sudo sed -i -e "/^\[$section\]/ a\\
$option = $value
" "$file"
else
local sep
sep=$(echo -ne "\x01")
# Replace it
$sudo sed -i -e '/^\['${section}'\]/,/^\[.*\]/ s'${sep}'^\('${option}'[ \t]*=[ \t]*\).*$'${sep}'\1'"${value}"${sep} "$file"
fi
$xtrace
}
# Set an option in a conf file
# confset [-sudo] config-file option value
# - if the file does not exist, it is created
function confset {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local option=$2
local value=$3
if ! grep -q "^$option" "$file" 2>/dev/null; then
echo -e "\n$option = $value\n" | $sudo tee --append "$file" > /dev/null
else
$sudo sed -i -e 's/^'$option'.*$/'$option' = '$value'/g' $file
fi
$xtrace
}
# Set a multiple line option in an INI file
# iniset_multiline [-sudo] config-file section option value1 value2 value3 ...
function iniset_multiline {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
shift 3
local values
for v in $@; do
# The later sed command inserts each new value in the line next to
# the section identifier, which causes the values to be inserted in
# the reverse order. Do a reverse here to keep the original order.
values="$v ${values}"
done
if ! grep -q "^\[$section\]" "$file"; then
# Add section at the end
echo -e "\n[$section]" | $sudo tee --append "$file" > /dev/null
else
# Remove old values
$sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ { /^$option[ \t]*=/ d; }" "$file"
fi
# Add new ones
for v in $values; do
$sudo sed -i -e "/^\[$section\]/ a\\
$option = $v
" "$file"
done
$xtrace
}
# Uncomment an option in an INI file
# iniuncomment config-file section option
function iniuncomment {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local section=$2
local option=$3
$sudo sed -i -e "/^\[$section\]/,/^\[.*\]/ s|[^ \t]*#[ \t]*\($option[ \t]*=.*$\)|\1|" "$file"
$xtrace
}
## Remove all comments in a file
function iniremcomment {
local xtrace
xtrace=$(set +o | grep xtrace)
set +o xtrace
local sudo=""
if [ $1 == "-sudo" ]; then
sudo="sudo "
shift
fi
local file=$1
local tpfile="$(mktemp)"
$sudo grep -v "^\#" $1 | sed '/^$/N;/^\n$/D' > $tpfile
$sudo cp $tpfile $1
$sudo rm -rf $tmpfile
$xtrace
}
function parse_yaml {
local prefix=$2
local s='[[:space:]]*' w='[a-zA-Z0-9_]*' fs=$(echo @|tr @ '\034')
sed -ne "s|^\($s\):|\1|" \
-e "s|^\($s\)\($w\)$s:$s[\"']\(.*\)[\"']$s\$|\1$fs\2$fs\3|p" \
-e "s|^\($s\)\($w\)$s:$s\(.*\)$s\$|\1$fs\2$fs\3|p" $1 |
awk -F$fs '{
indent = length($1)/2;
vname[indent] = $2;
for (i in vname) {if (i > indent) {delete vname[i]}}
if (length($3) > 0) {
vn=""; for (i=0; i<indent; i++) {vn=(vn)(vname[i])("_")}
printf("%s%s%s=\"%s\"\n", "'$prefix'",vn, $2, $3);
}
}'
}
# Restore xtrace
$INC_CONF_TRACE
# Local variables:
# mode: shell-script
# End:

View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
#apt-get update
echo 'Agent install is now complete!'

View File

@ -0,0 +1,12 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
#apt-get update
echo 'Devstack install is now complete!'

View File

@ -0,0 +1,20 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
# Java is required, install java first
source /onvm/scripts/install-java.sh
if [ -f /leapbin/elasticsearch-2.*.deb ];then
dpkg -i /leapbin/elasticsearch-2.*.deb
echo -e "network.host: $2" >> /etc/elasticsearch/elasticsearch.yml
update-rc.d elasticsearch defaults
service elasticsearch restart
echo 'Elastic install is now complete!'
else
echo 'Elasticsearch binary was not found!'
echo 'Download elasticsearch 2.2.x.deb and place it in tools/vagrant/ubuntu/leapbin directory.'
fi

View File

@ -0,0 +1,16 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
java_installed=$(which java)
if [ -z $java_installed ] && [ -f /leapbin/jdk-8*-linux-x64.tar.gz ]; then
mkdir -p /opt/jdk
tar -zxf /leapbin/jdk-*-linux-x64.tar.gz -C /opt/jdk
mv /opt/jdk/* /opt/jdk/jdk1.8
update-alternatives --install /usr/bin/java java /opt/jdk/jdk1.8/bin/java 100
update-alternatives --install /usr/bin/javac javac /opt/jdk/jdk1.8/bin/javac 100
echo 'Java install is now complete!'
else
echo 'Java has been either installed or not binary package found!'
echo 'Download java and place it in tools/vagrant/ubuntu/leapbin directory.'
fi

View File

@ -0,0 +1,21 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
if [ -f /leapbin/kafka_*-0.9.0.0.tgz ]; then
tar -zxf /leapbin/kafka_*-0.9.0.0.tgz -C /opt
mv /opt/kafka_*-0.9.0.0 /opt/kafka_0.9.0.0
/opt/kafka_0.9.0.0/bin/zookeeper-server-start.sh -daemon /opt/kafka_0.9.0.0/config/zookeeper.properties
sleep 2
/opt/kafka_0.9.0.0/bin/kafka-server-start.sh -daemon /opt/kafka_0.9.0.0/config/server.properties
echo 'Kafka install is now complete!'
else
echo 'Kafka binary was not found!'
fi

View File

@ -0,0 +1,19 @@
#!/usr/bin/env bash
# $1 sys_password
# $2 public ip eth0
source /onvm/scripts/ini-config
eval $(parse_yaml '/onvm/conf/nodes.conf.yml' 'leap_')
wget https://bootstrap.pypa.io/ez_setup.py -O - | python
apt-get update
apt-get -qqy install git python-dev python-pip python-pastedeploy
cd /opt/kiloeyes
pip install -r requirements.txt
python setup.py install
# Config the kiloeyes
echo 'Kiloeyes install is now complete!'

View File

@ -1,6 +0,0 @@
---
vm_memory: 8192
kafka_url: http://apache.stu.edu.tw/kafka/0.9.0.0/kafka_2.11-0.9.0.0.tgz
elasticsearch_url: https://download.elasticsearch.org/elasticsearch/release/org/elasticsearch/distribution/deb/elasticsearch/2.1.0/elasticsearch-2.1.0.deb

View File

@ -1,7 +0,0 @@
#!/bin/bash
TOOLS_PATH=${TOOLS_PATH:-$(dirname $0)}
VENV_PATH=${VENV_PATH:-${TOOLS_PATH}}
VENV_DIR=${VENV_NAME:-/../.venv}
TOOLS=${TOOLS_PATH}
VENV=${VENV:-${VENV_PATH}/${VENV_DIR}}
source ${VENV}/bin/activate && "$@"