Cleanup files and improve Vagrantfile

* integrate bootstrap.sh into the Vagrantfile
* remove deprecated patches
* remove unused files
* remove support for local Squid installation
* fix several syntax issues identified by Rubocop

Change-Id: I45d7aada7a9c5d1b7affcc3e3d246bebecd02fc4

Vagrantfile

@@ -47,23 +47,18 @@ Vagrant.configure(2) do |config|
   config.vm.provision 'ansible' do |ansible|
     ansible.playbook = 'playbook.yaml'
     ansible.extra_vars = {
-      install_proxy: CONFIG['proxy']['install'],
       storage_backend: CONFIG['storage_backend']
     }
   end
 
-  if CONFIG['proxy']['use'] && Vagrant.has_plugin?('vagrant-proxyconf')
+  if Vagrant.has_plugin?('vagrant-proxyconf') &&
-    if CONFIG['proxy']['install']
+     CONFIG['proxy'] != 'http://proxy.dummy.site:3128'
-      config.proxy.http  = "http://#{CONFIG['address']['controller']}:3128/"
+    config.proxy.enabled = true
-    else
+    config.proxy.http  = CONFIG['proxy']
-      config.proxy.http  = CONFIG['proxy']['address']
-    end
     config.proxy.no_proxy = 'localhost,127.0.0.1'
   end
 
-  if Vagrant.has_plugin?("vagrant-vbguest")
+  config.vbguest.auto_update = false if Vagrant.has_plugin?('vagrant-vbguest')
-    config.vbguest.auto_update = false
-  end
 
   CONFIG['address']['compute'].each_with_index do |address, index|
     name = "compute#{index + 1}"
@@ -113,8 +108,6 @@ Vagrant.configure(2) do |config|
     end
     add_block_device(node, 1, CONFIG['resources']['storage'])
     add_block_device(node, 2, CONFIG['resources']['storage'])
-    if CONFIG['proxy']['install'] && CONFIG['proxy']['use']
+    node.vm.provision 'shell', path: 'files/initialize.sh', privileged: false
-      node.proxy.enabled = false
-    end
   end
 end

ansible/controller.yaml

@@ -18,19 +18,9 @@
 - template: src=files/openrc.j2 dest=/home/vagrant/openrc owner=vagrant group=vagrant mode=0644
 - lineinfile: dest=/home/vagrant/.bashrc line='source $HOME/openrc'
 - copy: src=files/setup.sh dest=/home/vagrant/scripts/setup.sh owner=vagrant group=vagrant mode=0755
-- copy: src=files/initialize.sh dest=/home/vagrant/scripts/initialize.sh mode=0755 owner=vagrant group=vagrant
 - filesystem: fstype=xfs dev=/dev/sdb force=yes
 - filesystem: fstype=xfs dev=/dev/sdc force=yes
 - template: src=files/packstack.answers.j2 dest=/home/vagrant/packstack.answers owner=vagrant group=vagrant
-- yum: name=patch state=present
-  when: not development
-- copy: src=patches dest=/home/vagrant owner=vagrant group=vagrant mode=0644
-  when: not development
-- patch: patchfile={{ item }} basedir=/ strip=1
-  when: not development
-  with_items:
-  - /home/vagrant/patches/mongodb.pp
-  - /home/vagrant/patches/nova_compute_libvirt.pp
 - copy: src=files/gitconfig dest=/home/vagrant/.gitconfig owner=vagrant group=vagrant mode=0644
 - file: src=/home/vagrant/.ssh/id_packstack dest=/home/vagrant/.ssh/id_rsa owner=vagrant group=vagrant state=link
 - file: src=/home/vagrant/.ssh/id_packstack.pub dest=/home/vagrant/.ssh/id_rsa.pub owner=vagrant group=vagrant state=link

ansible/initialize.yaml

@@ -5,7 +5,6 @@
 - copy: src=files/id_rsa.pub dest=/home/vagrant/.ssh/id_packstack.pub mode=0600 owner=vagrant group=vagrant
 - copy: src=files/selinux dest=/etc/selinux/config mode=0644 owner=root group=root
 - file: path=/home/vagrant/scripts state=directory owner=vagrant group=vagrant mode=0755
-- copy: src=files/disable_network_manager.sh dest=/home/vagrant/scripts/disable_network_manager.sh mode=0755 owner=vagrant group=vagrant
 - shell: cat /home/vagrant/.ssh/id_packstack.pub >> /home/vagrant/.ssh/authorized_keys
 - file: path=/home/vagrant/.ssh/authorized_keys owner=vagrant group=vagrant
 - copy: src=files/motd dest=/etc/motd owner=root group=root mode=0644

ansible/proxy.yaml

@@ -1,4 +0,0 @@
----
-- yum: name=squid state=present
-- copy: src=files/squid.conf dest=/etc/squid/squid.conf
-- service: name=squid state=started enabled=yes

config.yaml.sample

@@ -9,10 +9,7 @@ network_agent: openvswitch
 network_type: gre
 netmask_internal: 255.255.0.0
 secret: password
-proxy:
+proxy: http://proxy.dummy.site:3128
-  address: 'http://proxy.company.site:3128'
-  install: false
-  use: false
 storage_backend: nfs
 address:
   controller: 10.100.50.10

doc/source/configuration.rst

@@ -139,30 +139,12 @@ configures a HTTP proxy to be used by ``yum``.
     $ vagrant plugin install vagrant-proxyconf
 
 When the plugin is installed caching/proxying is not enabled by default. To
-enable caching/proxying set ``use`` to ``true``.
+enable caching/proxying ``proxy`` has to point to an existing HTTP proxy
-
+server.
-``address`` has to point to an existing HTTP proxy server (e.g.
-``http://proxy.company.site:3128``).
 
 ::
 
-  proxy:
+  proxy: http://proxy.company.site:3128
-    use: true
-    install: false
-    address: 'http://proxy.company.site:3128'
-
-To install the HTTP proxy `Squid <http://www.squid-cache.org/>`__ on the
-controller node set ``install`` to ``true``.
-
-To use the local proxy set ``use`` to ``true``.  ``address`` has not to
-be set when installing Squid on the controller node. ``address`` will be
-overwritten when installing Squid as local HTTP proxy.
-
-::
-
-  proxy:
-    install: true
-    use: true
 
 Components
 ----------

doc/source/initialization.rst

@@ -1,18 +1,13 @@
 Initialization
 ==============
 
-First run the ``bootstrap.sh`` script to prepare all required nodes.
-
 ::
 
-    $ ./scripts/bootstrap.sh
+    $ vagrant up
 
-- A logfile for each node will be created in the directory ``log``.
+Afterwards run the following command on the controller node as the
-- It will take a long time (approximately 30 minutes, depends on your
+unprivileged user ``vagrant`` (``vagrant ssh controller``)
-  local environment) to boostrap all required nodes.
+to deploy OpenStack with Packstack.
-
-Afterwards run the following command on the controller node
-(``vagrant ssh controller``) to deploy OpenStack with Packstack.
 
 ::
 
@@ -29,7 +24,7 @@ Run ``packstack`` with ``--debug`` to enable debug logging.
 -  The generated manifests are available at:
    ``/var/tmp/packstack/YYMMDD-HHMMSS-abcdef/manifests``
 
-Optionally you can run the ``setup.sh`` script after the successful
+Finally (optional) you can run the ``setup.sh`` script after the successful
 deployment to add cloud images etc. pp.
 
 ::

doc/source/requirements.rst

@@ -13,6 +13,16 @@ Ensure to use at least version ``1.7.2`` of Vagrant.
     $ vagrant --version
     Vagrant 1.7.2
 
+Vagrant plugins
+~~~~~~~~~~~~~~~
+
+If you want to use an external HTTP proxy server for caching/proxying install
+the Vagrant plugin ``vagrant-proxyconf``.
+
+::
+
+    $ vagrant plugin install vagrant-proxyconf
+
 Ansible
 -------
 
@@ -22,13 +32,3 @@ to start this Vagrant environment install Ansible on the Vagrant host.
 ::
 
     $ sudo yum install -y ansible
-
-PyYAML
-------
-
-A helper script (`scripts/get_hosts.py`) requires the Python library
-`PyYAML <https://pypi.python.org/pypi/PyYAML/3.11>`__.
-
-::
-
-    $ sudo yum install -y PyYAML

files/disable_network_manager.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-for f in /etc/sysconfig/network-scripts/ifcfg-enp*; do
-    echo "NM_CONTROLLED=no" | sudo tee -a $f
-done

files/hosts.j2

@@ -3,9 +3,9 @@
 
 # <<< Packstack >>>
 {% for name in ['controller', 'network', 'storage'] %}
-{{ address[name] }} {{ name }}
+{{ address[name] }} {{ name }}.site {{ name }}
 {% endfor %}
 {% for ip in address['compute'] %}
-{{ ip }} compute{{ loop.index }} 
+{{ ip }} compute{{ loop.index }}.site compute{{ loop.index }}
 {% endfor %}
 # >>> Packstack <<<

files/initialize.sh

@@ -4,6 +4,10 @@ for node in $(sed -n '/<<< Packstack >>>/{:a;n;/>>> Packstack <<</b;p;ba}' /etc/
     ssh-keyscan $node >> /home/vagrant/.ssh/known_hosts
 done
 
+for node in $(sed -n '/<<< Packstack >>>/{:a;n;/>>> Packstack <<</b;p;ba}' /etc/hosts | awk '{ print $3 }'); do
+    ssh-keyscan $node >> /home/vagrant/.ssh/known_hosts
+done
+
 chown vagrant:vagrant /home/vagrant/.ssh/known_hosts
 
 for node in $(sed -n '/<<< Packstack >>>/{:a;n;/>>> Packstack <<</b;p;ba}' /etc/hosts | awk '{ print $2 }'); do

files/squid.conf

@@ -1,76 +0,0 @@
-#
-# Recommended minimum configuration:
-#
-
-# Example rule allowing access from your local networks.
-# Adapt to list your (internal) IP networks from where browsing
-# should be allowed
-acl localnet src 10.0.0.0/8	# RFC1918 possible internal network
-acl localnet src 172.16.0.0/12	# RFC1918 possible internal network
-acl localnet src 192.168.0.0/16	# RFC1918 possible internal network
-acl localnet src fc00::/7       # RFC 4193 local private network range
-acl localnet src fe80::/10      # RFC 4291 link-local (directly plugged) machines
-
-acl SSL_ports port 443
-acl Safe_ports port 80		# http
-acl Safe_ports port 21		# ftp
-acl Safe_ports port 443		# https
-acl Safe_ports port 70		# gopher
-acl Safe_ports port 210		# wais
-acl Safe_ports port 1025-65535	# unregistered ports
-acl Safe_ports port 280		# http-mgmt
-acl Safe_ports port 488		# gss-http
-acl Safe_ports port 591		# filemaker
-acl Safe_ports port 777		# multiling http
-acl CONNECT method CONNECT
-
-#
-# Recommended minimum Access Permission configuration:
-#
-# Deny requests to certain unsafe ports
-http_access deny !Safe_ports
-
-# Deny CONNECT to other than secure SSL ports
-http_access deny CONNECT !SSL_ports
-
-# Only allow cachemgr access from localhost
-http_access allow localhost manager
-http_access deny manager
-
-# We strongly recommend the following be uncommented to protect innocent
-# web applications running on the proxy server who think the only
-# one who can access services on "localhost" is a local user
-#http_access deny to_localhost
-
-#
-# INSERT YOUR OWN RULE(S) HERE TO ALLOW ACCESS FROM YOUR CLIENTS
-#
-
-# Example rule allowing access from your local networks.
-# Adapt localnet in the ACL section to list your (internal) IP networks
-# from where browsing should be allowed
-http_access allow localnet
-http_access allow localhost
-
-# And finally deny all other access to this proxy
-http_access deny all
-
-# Squid normally listens to port 3128
-http_port 3128
-
-# Uncomment and adjust the following to add a disk cache directory.
-cache_dir ufs /var/spool/squid 1024 16 256
-
-maximum_object_size 256 MB
-maximum_object_size_in_memory 4 MB
-
-# Leave coredumps in the first cache dir
-coredump_dir /var/spool/squid
-
-#
-# Add any of your own refresh_pattern entries above these.
-#
-refresh_pattern ^ftp:		1440	20%	10080
-refresh_pattern ^gopher:	1440	0%	1440
-refresh_pattern -i (/cgi-bin/|\?) 0	0%	0
-refresh_pattern .		0	20%	4320

library/patch

@@ -1,111 +0,0 @@
-#!/usr/bin/python -tt
-
-# (c) 2012, Luis Alberto Perez Lazaro <luisperlazaro@gmail.com>
-#
-# This module is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This software is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this software.  If not, see <http://www.gnu.org/licenses/>.
-#
-
-DOCUMENTATION = '''
----
-module: patch
-author: Luis Alberto Perez Lazaro
-version_added: 0.9
-short_description: apply patch files 
-description:
-    - Apply patch files using the GNU patch tool. Before using this module make sure the patch tool is installed.
-options:
-    patchfile:
-        required: true
-        description:
-            - A patch file as accepted by the gnu patch tool
-    strip:
-        required: true
-        aliases: [ p ]
-        description:
-            - Number that indicates the smallest prefix containing leading slashes that 
-          will be stripped from each file name found in the patch file. For more information
-          see the strip parameter of the gnu patch tool.
-    basedir:
-        required: true
-        description:
-            - base directory in which the patch file will be applied
-examples:
-    - code: "patch: patchfile=/tmp/critical.patch strip=1 basedir=/usr/share/pyshared/paramiko"
-      description: Example git checkout from Ansible Playbooks
-'''
-
-def _run(args):
-    cmd = subprocess.Popen(args, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    out, err = cmd.communicate()
-    rc = cmd.returncode
-    return (rc, out, err)
-
-def _is_already_applied(patch_file, strip):
-    reverse_cmd = "patch -s -R -N -p%s --dry-run < %s" % (strip, patch_file)
-    (rc, _, _) = _run(reverse_cmd)
-    return rc == 0
-
-def _apply_patch(module, patch_file, strip):
-    patch_cmd = "patch -s -N -t -r - -p%s < %s" % (strip, patch_file)
-    (rc, out, err) = _run(patch_cmd)
-    if rc != 0:
-        msg = out if not err else err
-        module.fail_json(msg=msg)
-
-def _get_params(module):
-    patchfile = os.path.expanduser(module.params['patchfile'])
-    strip = module.params['strip']
-    basedir = module.params['basedir']
-
-    if basedir:
-        os.chdir(os.path.expanduser(basedir))
-
-    if not os.path.exists(patchfile):
-        module.fail_json(msg="patchfile %s doesn't exist" % (patchfile))
-    if not os.access(patchfile, os.R_OK):
-        module.fail_json(msg="patchfile %s not readable" % (patchfile))
-
-    if not os.path.exists(basedir):
-        module.fail_json(msg="basedir %s doesn't exist" % (patchfile))
-
-    try:
-        strip = int(strip)
-    except Exception:
-        module.fail_json(msg="p must be a number")
-
-    return patchfile, strip, basedir
-
-# ===========================================
-
-def main():
-    module = AnsibleModule(
-        argument_spec = dict(
-            patchfile=dict(required=True),
-            basedir=dict(),
-            strip=dict(default=0, aliases=['p'])	
-        ),
-    )
-    patchfile, strip, basedir = _get_params(module)
-
-    changed = False
-
-    if not _is_already_applied(patchfile, strip):
-        _apply_patch(module, patchfile, strip)
-        changed = True
-
-    module.exit_json(changed=changed)
-
-# include magic from lib/ansible/module_common.py
-#<<INCLUDE_ANSIBLE_MODULE_COMMON>>
-main()

patches/mongodb.pp

@@ -1,9 +0,0 @@
---- a/usr/lib/python2.7/site-packages/packstack/puppet/templates/mongodb.pp 2014-12-02 16:19:58.274432117 +0000
-+++ b/usr/lib/python2.7/site-packages/packstack/puppet/templates/mongodb.pp 2014-12-02 16:20:15.620432319 +0000
-@@ -3,5 +3,5 @@
- class { 'mongodb::server':
-   smallfiles => true,
-   bind_ip    => [$mongodb_host],
-+  pidfilepath => '/var/run/mongodb/mongod.pid',
- }
--

patches/nova_compute_libvirt.pp

@@ -1,11 +0,0 @@
---- a/usr/lib/python2.7/site-packages/packstack/puppet/templates/nova_compute_libvirt.pp 2014-12-02 16:17:54.839430681 +0000
-+++ b/usr/lib/python2.7/site-packages/packstack/puppet/templates/nova_compute_libvirt.pp 2014-12-02 16:18:03.554430782 +0000
-@@ -3,7 +3,7 @@
- # Ensure Firewall changes happen before libvirt service start
- # preventing a clash with rules being set by libvirt
- 
--if $::is_virtual_packstack == 'true' {
-+if $::is_virtual == 'true' {
-   $libvirt_virt_type = 'qemu'
-   $libvirt_cpu_mode = 'none'
- } else {

playbook.yaml

@@ -9,5 +9,3 @@
     when: inventory_hostname_short == 'network'
   - include: ansible/nfs.yaml
     when: inventory_hostname_short == 'storage' and storage_backend == 'nfs'
-  - include: ansible/proxy.yaml
-    when: inventory_hostname_short == 'controller' and install_proxy

scripts/bootstrap.sh

@@ -1,44 +0,0 @@
-#!/bin/bash
-
-if [[ -z "$1" ]]; then
-    p=1
-else
-    p=$1
-fi
-
-run() {
-    number=$1
-    shift
-    python scripts/get_hosts.py | grep -v controller | xargs -n 1 -P $number \
-        -I BOX sh -c "echo - BOX && (vagrant $* BOX 2>&1 | tee -a log/BOX.log)"
-}
-
-if [[ ! -e config.yaml ]]; then
-    echo "error: configuration file 'config.yaml' does not exist"
-    exit 1
-fi
-
-echo "$(date) cleaning up"
-rm -f log/*
-vagrant destroy --force
-
-echo "$(date) bringing up, provisioning and reloading the controller VM"
-logfile=log/controller.log
-vagrant up controller | tee -a $logfile
-vagrant reload controller | tee -a $logfile
-
-echo "$(date) brining up all VMs"
-run $p up --no-provision
-
-echo "$(date) provisioning all other VMs"
-run $p provision
-
-echo "$(date) reloading all other VMs"
-run $p reload
-
-echo "$(date) initializing the controller node"
-logfile=log/controller.log
-vagrant ssh controller -c '/home/vagrant/scripts/initialize.sh' 2>&1 | tee -a $logfile
-
-echo "$(date) getting status of all VMs"
-vagrant status

scripts/get_hosts.py

@@ -1,25 +0,0 @@
-#!/usr/bin/python
-
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-# implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-import yaml
-
-config = yaml.load(open('config.yaml'))
-
-for name, value in config['address'].items():
-    if name == 'compute':
-        for index, _ in enumerate(value):
-            print("compute%d" % (index + 1))
-    else:
-        print(name)

scripts/reset.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-vagrant destroy --force
-vagrant box remove --force packstack-template
-#vagrant box remove --force b1-systems/centos-packstack