From 79d91f5be2edc088ac668d9b75c85ec09523d320 Mon Sep 17 00:00:00 2001
From: Przemyslaw Kaminski <pkaminski@mirantis.com>
Date: Thu, 6 Aug 2015 10:57:48 +0200
Subject: [PATCH] Solar bootstarpping playbooks & packer.io guide

---
 .gitignore                             |  4 ++
 Vagrantfile                            | 25 +++++-------
 bootstrap/README.md                    | 20 +++++++++
 bootstrap/build-image.yml              |  8 ++++
 bootstrap/packer.yml                   | 12 ++++++
 bootstrap/playbooks/base.yml           | 56 ++++++++++++++++++++++++++
 bootstrap/playbooks/build-main.yml     | 10 +++++
 bootstrap/playbooks/celery.yml         | 20 +++++++++
 bootstrap/playbooks/cinder.yml         | 11 +++++
 bootstrap/playbooks/custom-configs.yml |  7 ++++
 bootstrap/playbooks/docker.yml         |  7 ++++
 bootstrap/playbooks/files/hiera.yaml   | 15 +++++++
 bootstrap/playbooks/puppet.yml         | 24 +++++++++++
 bootstrap/playbooks/solar.yml          |  7 ++++
 bootstrap/solar-master.json            | 28 +++++++++++++
 15 files changed, 238 insertions(+), 16 deletions(-)
 create mode 100644 bootstrap/README.md
 create mode 100644 bootstrap/build-image.yml
 create mode 100644 bootstrap/packer.yml
 create mode 100644 bootstrap/playbooks/base.yml
 create mode 100644 bootstrap/playbooks/build-main.yml
 create mode 100644 bootstrap/playbooks/celery.yml
 create mode 100644 bootstrap/playbooks/cinder.yml
 create mode 100644 bootstrap/playbooks/custom-configs.yml
 create mode 100644 bootstrap/playbooks/docker.yml
 create mode 100644 bootstrap/playbooks/files/hiera.yaml
 create mode 100644 bootstrap/playbooks/puppet.yml
 create mode 100644 bootstrap/playbooks/solar.yml
 create mode 100644 bootstrap/solar-master.json

diff --git a/.gitignore b/.gitignore
index 18550fc1..1d9f0c69 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,7 @@ celery*.log
 *.dot
 *.png
 resources_compiled.py
+
+# bootstrap
+bootstrap/trusty64
+bootstrap/solar-master.box
diff --git a/Vagrantfile b/Vagrantfile
index 539766f9..bac27f72 100644
--- a/Vagrantfile
+++ b/Vagrantfile
@@ -5,38 +5,31 @@
 VAGRANTFILE_API_VERSION = "2"
 SLAVES_COUNT = 2
 
-init_script = <<SCRIPT
-apt-get update
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml
+solar_script = <<SCRIPT
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/solar.yml
 SCRIPT
 
 slave_script = <<SCRIPT
-apt-get update
-apt-get upgrade
-apt-get dist-upgrade
-apt-get -y install python-pip python-dev
-pip install ansible
-ansible-playbook -i "localhost," -c local /vagrant/main.yml /vagrant/docker.yml /vagrant/slave.yml /vagrant/slave_cinder.yml
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/custom-configs.yml -e master_ip=10.0.0.2
 SCRIPT
 
 master_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags slave
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags slave
 SCRIPT
 
 slave_celery = <<SCRIPT
-ansible-playbook -i "localhost," -c local /vagrant/celery.yml --skip-tags master
+ansible-playbook -i "localhost," -c local /vagrant/bootstrap/playbooks/celery.yml --skip-tags master
 SCRIPT
 
 Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
 
   #config.vm.box = "deb/jessie-amd64"
   #config.vm.box = "rustyrobot/deb-jessie-amd64"
-  config.vm.box = "ubuntu/trusty64"
+  #config.vm.box = "ubuntu/trusty64"
+  config.vm.box = "solar-master.box"
 
   config.vm.define "solar-dev", primary: true do |config|
-    config.vm.provision "shell", inline: init_script, privileged: true
+    config.vm.provision "shell", inline: solar_script, privileged: true
     config.vm.provision "shell", inline: master_celery, privileged: true
     config.vm.provision "file", source: "~/.vagrant.d/insecure_private_key", destination: "/vagrant/tmp/keys/ssh_private"
     config.vm.provision "file", source: "ansible.cfg", destination: "/home/vagrant/.ansible.cfg"
@@ -53,8 +46,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     index = i + 1
     ip_index = i + 3
     config.vm.define "solar-dev#{index}" do |config|
-      config.vm.provision "shell", inline: init_script, privileged: true
       config.vm.provision "shell", inline: slave_script, privileged: true
+      config.vm.provision "shell", inline: solar_script, privileged: true
       config.vm.provision "shell", inline: slave_celery, privileged: true
       config.vm.network "private_network", ip: "10.0.0.#{ip_index}"
       config.vm.host_name = "solar-dev#{index}"
diff --git a/bootstrap/README.md b/bootstrap/README.md
new file mode 100644
index 00000000..834df1e0
--- /dev/null
+++ b/bootstrap/README.md
@@ -0,0 +1,20 @@
+# Solar image building
+
+In `bootstrap/trusty64` directory there are `box.ovf` and `box-disk1.vmdk`
+files from the `trusty64` Vagrant box (normally found in
+ `~/.vagrant.d/boxes/trusty64/0/virtualbox`).
+
+To build, install Packer (https://www.packer.io/):
+```
+cd bootstrap
+packer build solar-master.json
+cp solar-master.box ../
+cd ..
+vagrant up
+```
+
+If Vagrant throws error about `vboxsf` try this:
+```
+vagrant plugin install vagrant-vbguest
+```
+(see https://github.com/shiguredo/packer-templates/issues/16).
\ No newline at end of file
diff --git a/bootstrap/build-image.yml b/bootstrap/build-image.yml
new file mode 100644
index 00000000..9dd5e139
--- /dev/null
+++ b/bootstrap/build-image.yml
@@ -0,0 +1,8 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+     #- shell: vagrant init ubuntu/trusty64
+
+     - shell: /usr/local/bin/packer build solar.json -var 'is_master=true'
diff --git a/bootstrap/packer.yml b/bootstrap/packer.yml
new file mode 100644
index 00000000..e5c81d4f
--- /dev/null
+++ b/bootstrap/packer.yml
@@ -0,0 +1,12 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - shell: wget 'https://dl.bintray.com/mitchellh/packer/packer_0.8.2_linux_amd64.zip' -O /tmp/packer-0.8.2.zip
+      args:
+        creates: /tmp/packer-0.8.2.zip
+    - unarchive:
+       src: /tmp/packer-0.8.2.zip
+       dest: /usr/local/bin
+       copy: no
diff --git a/bootstrap/playbooks/base.yml b/bootstrap/playbooks/base.yml
new file mode 100644
index 00000000..2debca36
--- /dev/null
+++ b/bootstrap/playbooks/base.yml
@@ -0,0 +1,56 @@
+---
+
+- shell: apt-get update
+
+- name: Base packages
+  apt: name={{ item }} state=present
+  with_items:
+  - git
+  - python-mock
+  - python-keystoneclient
+  - python-mysqldb
+  - python-setuptools
+  - ruby-dev
+  - unzip
+  # Redis
+  - redis-server
+  # Graph drawing
+  - python-pygraphviz
+  # Other tools
+  - htop
+  - jq
+  - tmux
+  - vim
+  # Dev
+  - ipython
+  - python-pudb
+  # Required by packer
+  - build-essential
+
+# PIP
+#- apt: name=python-pip state=absent
+#- shell: easy_install pip
+#- shell: pip install -U pip
+#- shell: pip install -U setuptools
+- shell: pip install httpie
+- shell: pip install docker-py==1.1.0
+
+# Redis
+- shell: pip install redis
+- lineinfile: dest=/etc/redis/redis.conf regexp='^bind ' line='bind 0.0.0.0'
+- service: name=redis-server state=restarted
+
+# Ubuntu OpenStack packages
+#- apt: name=ubuntu-cloud-keyring state=present
+#- shell: echo "deb http://ubuntu-cloud.archive.canonical.com/ubuntu trusty-updates/kilo main" > /etc/apt/sources.list.d/cloudarchive-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-2015-06-25-194717 trusty-updates main" > /etc/apt/sources.list.d/fuel-kilo.list
+#- shell: echo "deb http://osci-mirror-poz.infra.mirantis.net/pkgs/ubuntu-latest trusty main" > /etc/apt/sources.list.d/fuel-kilo.list
+# cloudarchive key
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 5EDB1B62EC4926EA
+# some other keys
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 9D6D8F6BC857C906
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 7638D0442B90D010
+# mirantis poznan
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 40976EAF437D05B5
+#- shell: apt-key adv --recv-key --keyserver keyserver.ubuntu.com 3B4FE6ACC0B21F32
+#- shell: apt-get update
diff --git a/bootstrap/playbooks/build-main.yml b/bootstrap/playbooks/build-main.yml
new file mode 100644
index 00000000..f71615a1
--- /dev/null
+++ b/bootstrap/playbooks/build-main.yml
@@ -0,0 +1,10 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - include: base.yml
+    - include: puppet.yml
+    - include: docker.yml
+    #- include: celery.yml tags=['master'] celery_dir=/var/run/celery
+    - include: cinder.yml
diff --git a/bootstrap/playbooks/celery.yml b/bootstrap/playbooks/celery.yml
new file mode 100644
index 00000000..5bbf7651
--- /dev/null
+++ b/bootstrap/playbooks/celery.yml
@@ -0,0 +1,20 @@
+---
+
+- hosts: all
+  sudo: yes
+  vars:
+    celery_dir: /var/run/celery
+  tasks:
+    - shell: mkdir -p {{ celery_dir }}
+    - shell: pip install celery
+    - shell: hostname
+      register: hostname
+    - shell: celery multi kill 2
+             chdir={{ celery_dir }}
+      tags: [stop]
+    - shell: celery multi start 2 -A solar.orchestration.runner -Q:1 scheduler,system_log -Q:2 celery,{{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [master]
+    - shell: celery multi start 1 -A solar.orchestration.runner -Q:1  {{ hostname.stdout }}
+             chdir={{ celery_dir }}
+      tags: [slave]
diff --git a/bootstrap/playbooks/cinder.yml b/bootstrap/playbooks/cinder.yml
new file mode 100644
index 00000000..8d95d2f3
--- /dev/null
+++ b/bootstrap/playbooks/cinder.yml
@@ -0,0 +1,11 @@
+---
+
+- apt: name=lvm2 state=present
+- command: sudo truncate -s 10G /root/cinder.img creates=/root/cinder.img
+- shell: sudo losetup -a|grep cinder
+  register: loop_created
+  ignore_errors: True
+- command: sudo losetup /dev/loop0 /root/cinder.img
+  when: loop_created|failed
+- lvg: vg=cinder-volumes pvs=/dev/loop0
+  when: loop_created|failed
diff --git a/bootstrap/playbooks/custom-configs.yml b/bootstrap/playbooks/custom-configs.yml
new file mode 100644
index 00000000..76636a56
--- /dev/null
+++ b/bootstrap/playbooks/custom-configs.yml
@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    - lineinfile: line='slaveof {{ master_ip }} 6379' dest=/etc/redis/redis.conf
+    - service: name=redis-server state=restarted
diff --git a/bootstrap/playbooks/docker.yml b/bootstrap/playbooks/docker.yml
new file mode 100644
index 00000000..e855cc53
--- /dev/null
+++ b/bootstrap/playbooks/docker.yml
@@ -0,0 +1,7 @@
+---
+
+- shell: docker --version
+  ignore_errors: true
+  register: docker_version
+- shell: curl -sSL https://get.docker.com/ | sudo sh
+  when: docker_version | failed
diff --git a/bootstrap/playbooks/files/hiera.yaml b/bootstrap/playbooks/files/hiera.yaml
new file mode 100644
index 00000000..ca27ee57
--- /dev/null
+++ b/bootstrap/playbooks/files/hiera.yaml
@@ -0,0 +1,15 @@
+:backends:
+  - redis
+  #- yaml
+  #- json
+:yaml:
+  :datadir: /etc/puppet/hieradata
+:json:
+  :datadir: /etc/puppet/hieradata
+:redis:
+  :port: 6379
+  :db: 0
+  :host: localhost
+  :deserialize: :json
+:hierarchy:
+  - resource
diff --git a/bootstrap/playbooks/puppet.yml b/bootstrap/playbooks/puppet.yml
new file mode 100644
index 00000000..91542c57
--- /dev/null
+++ b/bootstrap/playbooks/puppet.yml
@@ -0,0 +1,24 @@
+---
+
+# Puppet
+- shell: wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /root/puppetlabs-release-trusty.deb
+- shell: dpkg -i /root/puppetlabs-release-trusty.deb
+- shell: apt-get update
+
+- apt: name=puppet state=present
+- git: repo=https://github.com/CGenie/hiera-redis dest=/root/hiera-redis
+- shell: gem build hiera-redis.gemspec && gem install hiera-redis-3.0.0.gem chdir=/root/hiera-redis
+- template: src=files/hiera.yaml dest=/etc/puppet/hiera.yaml
+- file: path=/etc/puppet/hieradata state=directory
+# Make paths puppet 4 compatible
+- file: path=/etc/puppetlabs/code/ state=directory
+- file: src=/etc/puppet/hiera.yaml dest=/etc/puppetlabs/code/hiera.yaml state=link
+- file: path=/var/tmp/puppet/modules state=directory owner=puppet
+- file: path=/var/tmp/puppet/Puppetfile state=touch owner=puppet
+- file: path=/etc/puppet/modules state=absent
+- file: path=/etc/puppetlabs/code/modules state=absent
+- file: path=/etc/puppetlabs/code/hieradata state=absent
+- file: src=/var/tmp/puppet/modules dest=/etc/puppet/modules state=link
+- file: src=/var/tmp/puppet/modules dest=/etc/puppetlabs/code/modules state=link
+- file: src=/etc/puppet/hieradata dest=/etc/puppetlabs/code/hieradata state=link
+- shell: gem install librarian-puppet --no-ri --no-rdoc
diff --git a/bootstrap/playbooks/solar.yml b/bootstrap/playbooks/solar.yml
new file mode 100644
index 00000000..e4c3ee6b
--- /dev/null
+++ b/bootstrap/playbooks/solar.yml
@@ -0,0 +1,7 @@
+---
+
+- hosts: all
+  sudo: yes
+  tasks:
+    # Setup development env for solar
+    - shell: python setup.py develop chdir=/vagrant/solar
diff --git a/bootstrap/solar-master.json b/bootstrap/solar-master.json
new file mode 100644
index 00000000..b799801e
--- /dev/null
+++ b/bootstrap/solar-master.json
@@ -0,0 +1,28 @@
+{
+  "builders": [{
+    "type": "virtualbox-ovf",
+    "source_path": "trusty64/box.ovf",
+    "ssh_username": "vagrant",
+    "ssh_password": "vagrant",
+    "shutdown_command": "echo 'vagrant' | sudo -S shutdown -P now"
+  }],
+  "provisioners": [
+    {
+      "type": "shell",
+      "inline": [
+        "sudo apt-get remove -f python-pip",
+        "sudo apt-get install -y python-setuptools",
+        "sudo easy_install pip",
+        "sudo pip install -U pip",
+        "sudo pip install ansible"
+      ]
+    }, {
+      "type": "ansible-local",
+      "playbook_dir": "playbooks",
+      "playbook_file": "playbooks/build-main.yml"
+    }],
+  "post-processors": [{
+    "type": "vagrant",
+    "output": "solar-master.box"
+  }]
+}
\ No newline at end of file