Merge "Implement all-in-one.sh script"

README.md

@@ -6,7 +6,8 @@ puppet-openstack-integration
 1. [Overview - What is Puppet OpenStack Integration?](#overview)
 2. [Description - What does the project do?](#description)
 3. [Development - Guide for contributing](#development)
-4. [Contributors - Those with commits](#contributors)
+4. [All-in-one - How to deploy a cloud with Puppet](#All-In-One)
+5. [Contributors - Those with commits](#contributors)
 
 
 Overview
@@ -27,20 +28,22 @@ OpenStack Infrastructure is deploying two jobs per supported Operating System
 OpenStack services are balanced between two scenarios because OpenStack
 Infastructure Jenkins slaves can not afford the load of running all on the
 same node.
+One manifest (scenario003) is used for people who want to [run a simple All-In-One
+scenario](#All-In-One).
 
-|     -      | scenario001 | scenario002 |
+|     -      | scenario001 | scenario002 | scenario003 |
-|:----------:|:-----------:|:-----------:|
+|:----------:|:-----------:|:-----------:|:-----------:|
-| keystone   |      X      |       X     |
+| keystone   |      X      |       X     |       X     |
-| glance     |      X      |       X     |
+| glance     |      X      |       X     |       X     |
-| nova       |      X      |       X     |
+| nova       |      X      |       X     |       X     |
-| neutron    |      X      |       X     |
+| neutron    |      X      |       X     |       X     |
-| cinder     |      X      |             |
+| cinder     |      X      |             |       X     |
-| ceilometer |      X      |             |
+| ceilometer |      X      |             |             |
-| heat       |             |       X     |
+| heat       |             |       X     |             |
-| swift      |             |       X     |
+| swift      |             |       X     |             |
-| sahara     |             |       X     |
+| sahara     |             |       X     |             |
-| trove      |      X      |             |
+| trove      |      X      |             |             |
-| horizon    |             |       X     |
+| horizon    |             |       X     |       X     |
 
 When the Jenkins slave is created, the *run_tests.sh* script will executed.
 This script will execute *install_modules.sh* that prepare /etc/puppet/modules
@@ -65,6 +68,27 @@ Developer documentation for the entire Puppet OpenStack project:
 * https://wiki.openstack.org/wiki/Puppet
 
 
+All-In-One
+----------
+
+If you're new in Puppet OpenStack and you want to deploy an All-In-One setup of
+an OpenStack Cloud with the Puppet modules, please follow the steps:
+
+```bash
+git clone git://git.openstack.org/openstack/puppet-openstack-integration
+cd puppet-openstack-integration
+./all-in-one.sh
+```
+or
+
+```bash
+curl -sL http://git.openstack.org/cgit/openstack/puppet-openstack-integration/plain/all-in-one.sh | bash
+```
+
+Look at [Description](#description) to see which services it will install
+(scenario003).
+
+
 Contributors
 ------------
 

all-in-one.sh

@@ -0,0 +1,81 @@
+#!/bin/bash
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+#
+# Deploy Puppet OpenStack modules, deploy OpenStack and test the setup.
+# Software requirements:
+# * Ubuntu 14.04 LTS or CentOS7 fresh install
+# * 'git' installed
+#
+# Hardware requirements:
+# * At least 4GB of memory, but 8GB is recommended
+# * At least 10GB of storage
+#
+# Usage:
+# $ git clone git://git.openstack.org/openstack/puppet-openstack-integration
+# $ cd puppet-openstack-integration
+# $ ./all-in-one.sh
+#
+# or
+#
+# $ curl -sL http://git.openstack.org/cgit/openstack/puppet-openstack-integration/plain/all-in-one.sh | bash
+#
+
+set -e
+
+if [ -n "$DEBUG" ]; then
+  set -x
+fi
+
+# Prepare puppet-openstack-integration repository
+rm -rf /tmp/puppet-openstack-integration
+git clone git://git.openstack.org/openstack/puppet-openstack-integration /tmp/puppet-openstack-integration
+cd /tmp/puppet-openstack-integration
+
+SCENARIO=scenario003
+export SCRIPT_DIR=$(cd `dirname $0` && pwd -P)
+source $SCRIPT_DIR/functions
+
+if is_fedora; then
+    sudo yum -y remove facter puppet rdo-release
+    sudo yum -y install libxml2-devel libxslt-devel ruby-devel rubygems
+    sudo yum -y groupinstall "Development Tools"
+    DASHBOARD="dashboard"
+elif uses_debs; then
+    sudo apt-get remove -y --purge facter puppet puppet-common
+    sudo apt-get update
+    sudo apt-get install -y libxml2-dev libxslt-dev zlib1g-dev ruby
+    DASHBOARD="horizon"
+fi
+
+mkdir -p .bundled_gems
+export GEM_HOME=`pwd`/.bundled_gems
+gem install bundler --no-rdoc --no-ri --verbose
+
+set -e
+./run_tests.sh
+RESULT=$?
+set +e
+if [ $RESULT -ne 0 ]; then
+  echo "Deployment failed to finish."
+  exit 1
+fi
+
+cat <<-EOF
+
+OpenStack Dashboard available: http://127.0.0.1/${DASHBOARD}
+To access through Horizon, use the following user/password:
+  admin / a_big_secret
+
+EOF

fixtures/scenario003.pp

@@ -0,0 +1,418 @@
+#
+# Copyright 2015 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+Exec { logoutput => 'on_failure' }
+
+# Common resources
+case $::osfamily {
+  'Debian': {
+    include ::apt
+    class { '::openstack_extras::repo::debian::ubuntu':
+      release         => 'liberty',
+      repo            => 'proposed',
+      package_require => true,
+    }
+    $package_provider = 'apt'
+  }
+  'RedHat': {
+    class { '::openstack_extras::repo::redhat::redhat':
+      manage_rdo => false,
+      repo_hash  => {
+        'openstack-common-testing'  => {
+          'baseurl'  => 'http://cbs.centos.org/repos/cloud7-openstack-common-testing/x86_64/os/',
+          'descr'    => 'openstack-common-testing',
+          'gpgcheck' => 'no',
+        },
+        'openstack-liberty-testing' => {
+          'baseurl'  => 'http://cbs.centos.org/repos/cloud7-openstack-liberty-testing/x86_64/os/',
+          'descr'    => 'openstack-liberty-testing',
+          'gpgcheck' => 'no',
+        },
+        'openstack-liberty-trunk'   => {
+          'baseurl'  => 'http://trunk.rdoproject.org/centos7-liberty/current-passed-ci/',
+          'descr'    => 'openstack-liberty-trunk',
+          'gpgcheck' => 'no',
+        },
+      },
+    }
+    package { 'openstack-selinux': ensure => 'latest' }
+    $package_provider = 'yum'
+  }
+  default: {
+    fail("Unsupported osfamily (${::osfamily})")
+  }
+}
+
+# Deploy MySQL Server
+class { '::mysql::server': }
+
+# Deploy RabbitMQ
+class { '::rabbitmq':
+  delete_guest_user => true,
+  package_provider  => $package_provider,
+}
+rabbitmq_vhost { '/':
+  provider => 'rabbitmqctl',
+  require  => Class['rabbitmq'],
+}
+rabbitmq_user { ['neutron', 'nova', 'cinder', 'glance']:
+  admin    => true,
+  password => 'an_even_bigger_secret',
+  provider => 'rabbitmqctl',
+  require  => Class['rabbitmq'],
+}
+rabbitmq_user_permissions { ['neutron@/', 'nova@/', 'cinder@/', 'glance@/']:
+  configure_permission => '.*',
+  write_permission     => '.*',
+  read_permission      => '.*',
+  provider             => 'rabbitmqctl',
+  require              => Class['rabbitmq'],
+}
+
+# Deploy Keystone
+class { '::keystone::client': }
+class { '::keystone::cron::token_flush': }
+class { '::keystone::db::mysql':
+  password => 'keystone',
+}
+class { '::keystone':
+  verbose             => true,
+  debug               => true,
+  database_connection => 'mysql://keystone:keystone@127.0.0.1/keystone',
+  admin_token         => 'admin_token',
+  enabled             => true,
+  service_name        => 'httpd',
+}
+include ::apache
+class { '::keystone::wsgi::apache':
+  ssl     => false,
+  workers => 2,
+}
+class { '::keystone::roles::admin':
+  email    => 'test@example.tld',
+  password => 'a_big_secret',
+}
+class { '::keystone::endpoint':
+  default_domain => 'admin',
+}
+
+# Deploy Glance
+class { '::glance::db::mysql':
+  password => 'glance',
+}
+include ::glance
+include ::glance::backend::file
+include ::glance::client
+class { '::glance::keystone::auth':
+  password => 'a_big_secret',
+}
+class { '::glance::api':
+  debug               => true,
+  verbose             => true,
+  database_connection => 'mysql://glance:glance@127.0.0.1/glance?charset=utf8',
+  keystone_password   => 'a_big_secret',
+  workers             => 2,
+}
+class { '::glance::registry':
+  debug               => true,
+  verbose             => true,
+  database_connection => 'mysql://glance:glance@127.0.0.1/glance?charset=utf8',
+  keystone_password   => 'a_big_secret',
+  workers             => 2,
+}
+class { '::glance::notify::rabbitmq':
+  rabbit_userid       => 'glance',
+  rabbit_password     => 'an_even_bigger_secret',
+  rabbit_host         => '127.0.0.1',
+  notification_driver => 'messagingv2',
+}
+
+# Deploy Neutron
+class { '::neutron::db::mysql':
+  password => 'neutron',
+}
+class { '::neutron::keystone::auth':
+  password => 'a_big_secret',
+}
+class { '::neutron':
+  rabbit_user           => 'neutron',
+  rabbit_password       => 'an_even_bigger_secret',
+  rabbit_host           => '127.0.0.1',
+  allow_overlapping_ips => true,
+  core_plugin           => 'ml2',
+  service_plugins       => ['router', 'metering'],
+  debug                 => true,
+  verbose               => true,
+}
+class { '::neutron::client': }
+class { '::neutron::server':
+  database_connection => 'mysql://neutron:neutron@127.0.0.1/neutron?charset=utf8',
+  auth_password       => 'a_big_secret',
+  identity_uri        => 'http://127.0.0.1:35357/',
+  sync_db             => true,
+  api_workers         => 4,
+}
+class { '::neutron::plugins::ml2':
+  type_drivers         => ['vxlan'],
+  tenant_network_types => ['vxlan'],
+  mechanism_drivers    => ['openvswitch'],
+}
+class { '::neutron::agents::ml2::ovs':
+  enable_tunneling => true,
+  local_ip         => '127.0.0.1',
+  tunnel_types     => ['vxlan'],
+}
+class { '::neutron::agents::metadata':
+  debug            => true,
+  auth_password    => 'a_big_secret',
+  shared_secret    => 'a_big_secret',
+  metadata_workers => 2,
+}
+class { '::neutron::agents::lbaas':
+  debug => true,
+}
+class { '::neutron::agents::l3':
+  debug => true,
+}
+class { '::neutron::agents::dhcp':
+  debug => true,
+}
+class { '::neutron::agents::metering':
+  debug => true,
+}
+class { '::neutron::server::notifications':
+  nova_admin_password => 'a_big_secret',
+}
+
+# Deploy Nova
+class { '::nova::db::mysql':
+  password => 'nova',
+}
+class { '::nova::keystone::auth':
+  password => 'a_big_secret',
+}
+class { '::nova':
+  database_connection    => 'mysql://nova:nova@127.0.0.1/nova?charset=utf8',
+  rabbit_host            => '127.0.0.1',
+  rabbit_userid          => 'nova',
+  rabbit_password        => 'an_even_bigger_secret',
+  glance_api_servers     => 'localhost:9292',
+  verbose                => true,
+  debug                  => true,
+  notification_driver    => 'messagingv2',
+  notify_on_state_change => 'vm_and_task_state',
+}
+class { '::nova::api':
+  admin_password                       => 'a_big_secret',
+  identity_uri                         => 'http://127.0.0.1:35357/',
+  osapi_v3                             => true,
+  neutron_metadata_proxy_shared_secret => 'a_big_secret',
+  osapi_compute_workers                => 2,
+  ec2_workers                          => 2,
+  metadata_workers                     => 2,
+  default_floating_pool                => 'public',
+}
+class { '::nova::cert': }
+class { '::nova::client': }
+class { '::nova::conductor': }
+class { '::nova::consoleauth': }
+class { '::nova::cron::archive_deleted_rows': }
+class { '::nova::compute':
+  vnc_enabled                 => true,
+  instance_usage_audit        => true,
+  instance_usage_audit_period => 'hour',
+}
+class { '::nova::compute::libvirt':
+  libvirt_virt_type => 'qemu',
+  migration_support => true,
+  vncserver_listen  => '0.0.0.0',
+}
+class { '::nova::scheduler': }
+class { '::nova::vncproxy': }
+class { '::nova::network::neutron':
+  neutron_admin_password => 'a_big_secret',
+  neutron_admin_auth_url => 'http://127.0.0.1:35357/v2.0',
+}
+
+# Deploy Cinder
+class { '::cinder::db::mysql':
+  password => 'cinder',
+}
+class { '::cinder::keystone::auth':
+  password => 'a_big_secret',
+}
+class { '::cinder':
+  database_connection => 'mysql://cinder:cinder@127.0.0.1/cinder?charset=utf8',
+  rabbit_host         => '127.0.0.1',
+  rabbit_userid       => 'cinder',
+  rabbit_password     => 'an_even_bigger_secret',
+  verbose             => true,
+  debug               => true,
+}
+class { '::cinder::api':
+  keystone_password   => 'a_big_secret',
+  identity_uri        => 'http://127.0.0.1:35357/',
+  default_volume_type => 'BACKEND_1',
+  service_workers     => 2,
+}
+class { '::cinder::quota': }
+class { '::cinder::scheduler': }
+class { '::cinder::scheduler::filter': }
+class { '::cinder::volume': }
+class { '::cinder::cron::db_purge': }
+class { '::cinder::glance':
+  glance_api_servers  => 'localhost:9292',
+}
+class { '::cinder::setup_test_volume':
+  size => '15G',
+}
+cinder::backend::iscsi { 'BACKEND_1':
+  iscsi_ip_address => '127.0.0.1',
+}
+class { '::cinder::backends':
+  enabled_backends => ['BACKEND_1'],
+}
+Cinder::Type {
+  os_password    => 'a_big_secret',
+  os_tenant_name => 'services',
+  os_username    => 'cinder',
+  os_auth_url    => 'http://127.0.0.1:5000/v2.0',
+}
+cinder::type { 'BACKEND_1':
+  set_key   => 'volume_backend_name',
+  set_value => 'BACKEND_1',
+  notify    => Service['cinder-volume'],
+  require   => Service['cinder-api'],
+}
+
+# Deploy Horizon
+$vhost_params = { add_listen => false }
+class { '::horizon':
+  secret_key         => 'big_secret',
+  vhost_extra_params => $vhost_params,
+  servername         => $::hostname,
+  allowed_hosts      => $::hostname,
+  # need to disable offline compression due to
+  # https://bugs.launchpad.net/ubuntu/+source/horizon/+bug/1424042
+  compress_offline   => false,
+}
+
+# Configure Tempest and the resources
+$os_auth_options = '--os-username admin --os-password a_big_secret --os-tenant-name openstack --os-auth-url http://127.0.0.1:5000/v2.0'
+
+exec { 'manage_m1.nano_nova_flavor':
+  path     => '/usr/bin:/bin:/usr/sbin:/sbin',
+  provider => shell,
+  command  => "nova ${os_auth_options} flavor-create m1.nano 42 128 0 1",
+  unless   => "nova ${os_auth_options} flavor-list | grep m1.nano",
+}
+Keystone_user_role['admin@openstack'] -> Exec['manage_m1.nano_nova_flavor']
+
+exec { 'manage_m1.micro_nova_flavor':
+  path     => '/usr/bin:/bin:/usr/sbin:/sbin',
+  provider => shell,
+  command  => "nova ${os_auth_options} flavor-create m1.micro 84 128 0 1",
+  unless   => "nova ${os_auth_options} flavor-list | grep m1.micro",
+}
+Keystone_user_role['admin@openstack'] -> Exec['manage_m1.micro_nova_flavor']
+
+neutron_network { 'public':
+  tenant_name     => 'openstack',
+  router_external => true,
+}
+Keystone_user_role['admin@openstack'] -> Neutron_network<||>
+
+neutron_subnet { 'public-subnet':
+  cidr             => '172.24.5.0/24',
+  ip_version       => '4',
+  allocation_pools => ['start=172.24.5.10,end=172.24.5.200'],
+  gateway_ip       => '172.24.5.1',
+  enable_dhcp      => false,
+  network_name     => 'public',
+  tenant_name      => 'openstack',
+}
+
+include ::vswitch::ovs
+vs_bridge { 'br-ex':
+  ensure => present,
+  notify => Exec['create_br-ex_vif'],
+}
+
+# creates br-ex virtual interface to reach floating-ip network
+exec { 'create_br-ex_vif':
+  path        => '/usr/bin:/bin:/usr/sbin:/sbin',
+  provider    => shell,
+  command     => 'ip addr add 172.24.5.1/24 dev br-ex; ip link set br-ex up',
+  refreshonly => true,
+}
+
+glance_image { 'cirros':
+  ensure           => present,
+  container_format => 'bare',
+  disk_format      => 'qcow2',
+  is_public        => 'yes',
+  # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source
+  # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image.
+  # source        => '/home/jenkins/cache/files/cirros-0.3.4-x86_64-disk.img',
+  source           => 'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img',
+}
+glance_image { 'cirros_alt':
+  ensure           => present,
+  container_format => 'bare',
+  disk_format      => 'qcow2',
+  is_public        => 'yes',
+  # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source
+  # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image.
+  # source        => '/home/jenkins/cache/files/cirros-0.3.4-x86_64-disk.img',
+  source           => 'http://download.cirros-cloud.net/0.3.4/cirros-0.3.4-x86_64-disk.img',
+}
+
+class { '::tempest':
+  debug               => true,
+  use_stderr          => false,
+  log_file            => 'tempest.log',
+  git_clone           => true,
+  tempest_clone_path  => '/tmp/tempest',
+  tempest_clone_owner => $::id,
+  lock_path           => '/tmp/tempest',
+  tempest_config_file => '/tmp/tempest/etc/tempest.conf',
+  configure_images    => true,
+  configure_networks  => true,
+  identity_uri        => 'http://127.0.0.1:5000/v2.0',
+  identity_uri_v3     => 'http://127.0.0.1:5000/v3',
+  admin_username      => 'admin',
+  admin_tenant_name   => 'openstack',
+  admin_password      => 'a_big_secret',
+  admin_domain_name   => 'Default',
+  auth_version        => 'v3',
+  image_name          => 'cirros',
+  image_name_alt      => 'cirros_alt',
+  cinder_available    => true,
+  glance_available    => true,
+  horizon_available   => true,
+  nova_available      => true,
+  neutron_available   => true,
+  public_network_name => 'public',
+  flavor_ref          => '42',
+  flavor_ref_alt      => '84',
+  image_ssh_user      => 'cirros',
+  image_alt_ssh_user  => 'cirros',
+  img_file            => 'cirros-0.3.4-x86_64-disk.img',
+  # TODO(emilien) optimization by 1/ using Hiera to configure Glance image source
+  # and 2/ if running in the gate, use /home/jenkins/cache/files/ cirros image.
+  # img_dir           => '/home/jenkins/cache/files',
+  img_dir             => '/tmp/tempest',
+}

functions

@@ -68,3 +68,18 @@ install_modules() {
     install_all
   fi
 }
+
+is_fedora() {
+    if [ -f /etc/os-release ]; then
+        source /etc/os-release
+        test "$ID" = "fedora" -o "$ID" = "centos"
+    else
+        return 1
+    fi
+}
+
+uses_debs() {
+    # check if apt-get is installed, valid for debian based
+    type "apt-get" 2>/dev/null
+}
+

run_tests.sh

@@ -14,6 +14,8 @@
 # under the License.
 
 export SCENARIO=${SCENARIO:-scenario001}
+export SCRIPT_DIR=$(cd `dirname $0` && pwd -P)
+source $SCRIPT_DIR/functions
 
 if [ ! -f fixtures/${SCENARIO}.pp ]; then
     echo "fixtures/${SCENARIO}.pp file does not exist. Please define a valid scenario."
@@ -45,17 +47,6 @@ function run_puppet() {
     return $res
 }
 
-function is_fedora {
-    # note we consider CentOS 7 as fedora for now
-    lsb_release -i 2>/dev/null | grep -iq "fedora" || \
-        lsb_release -i 2>/dev/null | grep -iq "CentOS"
-}
-
-function uses_debs {
-    # check if apt-get is installed, valid for debian based
-    type "apt-get" 2>/dev/null
-}
-
 if uses_debs; then
     wget https://apt.puppetlabs.com/puppetlabs-release-trusty.deb -O /tmp/puppet.deb
     $SUDO dpkg -i /tmp/puppet.deb