openstack
/
openstack-chef
Code Issues Proposed changes

Updated integration attributes and methods 

- Implemented changes consistent with cookstyle
- DRY the environment files
- Added tempest recipes in minimal role; drop build-essential
- Corrected url for chefdk with current scheme
- Removed nova smoke test since this is handled by tempest and tends to fire
  before nova is available in integration

Implements blueprint modern-chef

Change-Id: I5c5371eed12b282a527576822b270cce8b0e76a7
This commit is contained in:
Samuel Cassiba 2017-12-22 17:58:46 -08:00
parent ca3a5df961
commit a317a117c8
6 changed files with 132 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.rubocop_todo.yml
View File

@ -1,12 +1,14 @@
# This configuration was generated by
# `rubocop --auto-gen-config`
# on 2017-08-02 14:33:46 -0400 using RuboCop version 0.47.1.
# on 2017-12-22 17:48:38 -0800 using RuboCop version 0.49.1.
# The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again.
# Offense count: 2
# Configuration parameters: CountComments, ExcludedMethods.
Metrics/BlockLength:
Max: 33
# Offense count: 1
# Configuration parameters: EnforcedStyle, SupportedStyles.
# SupportedStyles: for, each
Style/For:
Exclude:
- 'Rakefile'

27
Berksfile
View File

@ -1,25 +1,24 @@
source 'https://supermarket.chef.io'
%w(
integration-test
orchestration
telemetry
block-storage
common
compute
dashboard
identity
image
network
ops-database
ops-messaging
).each do |cookbook|
integration-test
orchestration
telemetry
block-storage
common
compute
dashboard
identity
image
network
ops-database
ops-messaging
).each do |cookbook|
if Dir.exist?("../cookbook-openstack-#{cookbook}")
cookbook "openstack-#{cookbook}", path: "../cookbook-openstack-#{cookbook}"
else
cookbook "openstack-#{cookbook}", github: "openstack/cookbook-openstack-#{cookbook}"
end
end
cookbook 'openstack-dns', github: 'cloudbau/cookbook-openstack-dns'

168
Rakefile
View File

@ -1,10 +1,10 @@
current_dir = File.dirname(__FILE__)
client_opts = "--force-formatter --no-color -z --config #{current_dir}/.chef/knife.rb"
task default: ["test"]
task default: ['test']
desc "Default gate tests to run"
task :test => [:rubocop, :berks_vendor, :json_check]
desc 'Default gate tests to run'
task test: [:rubocop, :berks_vendor, :json_check]
def run_command(command)
if File.exist?('Gemfile.lock')
@ -15,39 +15,23 @@ def run_command(command)
end
task :destroy_all do
Rake::Task[:destroy_machines].invoke
run_command('rm -rf Gemfile.lock && rm -rf Berksfile.lock && rm -rf cookbooks/')
end
desc "Destroy machines"
task :destroy_machines do
run_command("chef-client #{client_opts} -o 'provisioning::destroy_all'")
end
desc "Vendor your cookbooks/"
desc 'Vendor your cookbooks/'
task :berks_vendor do
run_command('berks vendor cookbooks')
end
desc "Create Chef Key"
desc 'Create Chef Key'
task :create_key do
if not File.exist?('.chef/validator.pem')
unless File.exist?('.chef/validator.pem')
require 'openssl'
File.binwrite('.chef/validator.pem', OpenSSL::PKey::RSA.new(2048).to_pem)
end
end
desc "All-in-One build"
task :allinone => :create_key do
run_command("chef-client #{client_opts} -o 'provisioning::allinone'")
end
desc "Multinode build"
task :multinode => :create_key do
run_command("chef-client #{client_opts} -o 'provisioning::multinode'")
end
desc "Blow everything away"
desc 'Blow everything away'
task clean: [:destroy_all]
# CI tasks
@ -55,10 +39,10 @@ require 'cookstyle'
require 'rubocop/rake_task'
desc 'Run RuboCop'
RuboCop::RakeTask.new do |task|
task.options << "--display-cop-names"
task.options << '--display-cop-names'
end
desc "Validate data bags, environments and roles"
desc 'Validate data bags, environments and roles'
task :json_check do
require 'json'
['data_bags/*', 'environments', 'roles'].each do |sub_dir|
@ -70,7 +54,7 @@ task :json_check do
end
# Helper for running various testing commands
def _run_commands(desc, commands, openstack=true)
def _run_commands(desc, commands, openstack = true)
puts "## Running #{desc}"
commands.each do |command, options|
options.each do |option|
@ -92,121 +76,77 @@ elsif File.exist?('/usr/bin/yum')
end
# Helper for looking at the starting environment
def _run_env_queries # rubocop:disable Metrics/MethodLength
_run_commands('basic common env queries', {
'uname' => ['-a'],
'pwd' => [''],
'env' => ['']},
false
)
def _run_env_queries
_run_commands('basic common env queries', {
'uname' => ['-a'],
'pwd' => [''],
'env' => [''] },
false)
case @platform
when 'ubuntu16'
_run_commands('basic debian env queries', {
'ifconfig' => [''],
'cat' => ['/etc/apt/sources.list']},
false
)
'ifconfig' => [''],
'cat' => ['/etc/apt/sources.list'] },
false)
when 'centos7'
_run_commands('basic rhel env queries', {
'/usr/sbin/ip' => ['addr'],
'cat' => ['/etc/yum.repos.d/*']},
false
)
'/usr/sbin/ip' => ['addr'],
'cat' => ['/etc/yum.repos.d/*'] },
false)
end
end
# Helper for setting up basic query tests
def _run_basic_queries # rubocop:disable Metrics/MethodLength
_run_commands('basic common test queries', {
'sudo netstat' => ['-nlp'],
'nova-manage' => ['version', 'db version'],
'nova' => %w(--version service-list hypervisor-list flavor-list),
'glance-manage' => %w(db_version),
'glance' => %w(--version image-list),
'keystone-manage' => %w(db_version),
'openstack' => ['--version', 'user list', 'endpoint list', 'role list',
'service list', 'project list',
'network agent list', 'extension list --network ',
'network list', 'subnet list'],
'neutron' => %w(port-list quota-list),
'ovs-vsctl' => %w(show) }
)
def _run_basic_queries
_run_commands('basic common test queries',
'sudo netstat' => ['-nlp'],
'nova-manage' => ['version', 'db version'],
'nova' => %w(--version service-list hypervisor-list flavor-list),
'glance-manage' => %w(db_version),
'glance' => %w(--version image-list),
'keystone-manage' => %w(db_version),
'openstack' => ['--version', 'user list', 'endpoint list', 'role list',
'service list', 'project list',
'network agent list', 'extension list --network ',
'network list', 'subnet list'],
'ovs-vsctl' => %w(show))
case @platform
when 'ubuntu16'
_run_commands('basic debian test queries', {
'rabbitmqctl' => %w(cluster_status),
'ip' => ['addr', 'route', '-6 route']}
)
_run_commands('basic debian test queries',
'rabbitmqctl' => %w(cluster_status),
'ip' => ['addr', 'route', '-6 route'])
when 'centos7'
_run_commands('basic rhel test queries', {
'/usr/sbin/rabbitmqctl' => %w(cluster_status),
'/usr/sbin/ip' => ['addr', 'route', '-6 route']}
)
_run_commands('basic rhel test queries',
'/usr/sbin/rabbitmqctl' => %w(cluster_status),
'/usr/sbin/ip' => ['addr', 'route', '-6 route'])
end
end
# Helper for setting up basic nova tests
def _run_nova_tests(pass) # rubocop:disable Metrics/MethodLength
_run_commands('nova server create', {
'openstack' => ['server list', "server create --image cirros --flavor m1.nano test#{pass}"],
'sleep' => ['40'] }
)
_run_commands('nova server cleanup', {
'openstack' => ['server list', "server show test#{pass}", "server delete test#{pass}"],
'sleep' => ['15'] }
)
_run_commands('nova server query', {
'openstack' => ['server list'] }
)
end
# Helper for setting up neutron local network
def _setup_local_network # rubocop:disable Metrics/MethodLength
_run_commands('neutron local network setup', {
'openstack' => ['network create --share local_net', 'subnet create --network local_net --subnet-range 192.168.180.0/24 local_subnet'] }
)
end
# Helper for setting up tempest and upload the default cirros image.
def _setup_tempest(client_opts, log_dir)
sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'recipe[openstack-integration-test::setup]' > #{log_dir}/chef-client-setup-tempest.txt 2>&1 )
end
def _save_logs(prefix, log_dir)
sh %(sleep 25)
%w(nova neutron keystone glance apache2 rabbitmq mysql-default openvswitch mariadb).each do |project|
%w(nova neutron keystone glance apache2 rabbitmq mysql mysql-default openvswitch mariadb).each do |project|
sh %(mkdir -p #{log_dir}/#{prefix}/#{project})
sh %(sudo cp -rL /etc/#{project} #{log_dir}/#{prefix}/#{project}/etc || true)
sh %(sudo cp -rL /var/log/#{project} #{log_dir}/#{prefix}/#{project}/log || true)
end
end
desc "Integration test on Infra"
task :integration => [:create_key, :berks_vendor] do
log_dir = ENV['WORKSPACE']+'/logs'
desc 'Integration test on Infra'
task integration: [:create_key, :berks_vendor] do
log_dir = ENV['WORKSPACE'] + '/logs'
sh %(mkdir #{log_dir})
# This is a workaround for allowing chef-client to run in local mode
sh %(sudo mkdir -p /etc/chef && sudo cp .chef/encrypted_data_bag_secret /etc/chef/openstack_data_bag_secret)
_run_env_queries
# Install mysql2 gem to avoid hitting mirror issues
sh %(wget https://rubygems.org/downloads/mysql2-0.4.5.gem)
sh %(sudo apt-get install -y libmysqlclient-dev)
sh %(chef exec gem install -N ./mysql2-0.4.5.gem)
# Three passes to make sure of cookbooks idempotency
for i in 1..3
# Three passes to ensure idempotency. prefer each to times, even if it
# reads weird
for i in 1..3 do
begin
puts "####### Pass #{i}"
# Kick off chef client in local mode, will converge OpenStack right on the gate job "in place"
sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 )
if i == 1
_setup_tempest(client_opts, log_dir)
_setup_local_network
end
_run_basic_queries
_run_nova_tests(i)
puts "####### Pass #{i}"
# Kick off chef client in local mode, will converge OpenStack right on the gate job "in place"
sh %(sudo chef-client #{client_opts} -E integration -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 )
_run_basic_queries
rescue => e
raise "####### Pass #{i} failed with #{e.message}"
ensure
@ -216,8 +156,4 @@ task :integration => [:create_key, :berks_vendor] do
sh %(sudo chmod -R go+rx #{log_dir}/pass#{i})
end
end
# Run the tempest formal tests, setup with the openstack-integration-test cookbook
Dir.chdir('/opt/tempest') do
sh %(sudo -H /opt/tempest-venv/tempest.sh)
end
end

11
environments/integration-ubuntu16.json → environments/integration.json
View File

@ -1,6 +1,6 @@
{
"name": "integration-ubuntu16",
"description": "Environment used in testing the upstream cookbooks and reference Chef repository with vagrant. To be used with the vagrantfile-allinone vagrantfile. Defines the necessary attributes for a working all-in-one openstack deployment, using neutron for the networking component, and the openvswitch neutron plugin",
"name": "integration",
"description": "Chef environment file for building OpenStack in CI settings. Certain Tempest services are disabled in this scenario",
"cookbook_versions": {
},
"json_class": "Chef::Environment",
@ -8,13 +8,11 @@
"default_attributes": {
"apache": {
"listen": []
},
"apt": {
"compile_time_update" : true
}
},
"override_attributes": {
"openstack": {
"is_release": true,
"apt": {
"update_apt_cache": true
},
@ -41,7 +39,7 @@
"mq": {
"user": "admin"
},
"network":{
"network": {
"conf": {
"DEFAULT": {
"service_plugins": "router"
@ -54,6 +52,7 @@
"compute": {
"conf": {
"libvirt": {
"cpu_type": "none",
"virt_type": "qemu"
}
}

53
minimal.json Normal file
View File

@ -0,0 +1,53 @@
{
"name": "minimal",
"description": "This will deploy a minimal set of services for Openstack Compute to function for integration testing.",
"json_class": "Chef::Role",
"default_attributes": {
},
"override_attributes": {
},
"chef_type": "role",
"run_list": [
"recipe[openstack-common]",
"recipe[openstack-common::logging]",
"recipe[openstack-common::sysctl]",
"recipe[openstack-common::client]",
"recipe[openstack-ops-database::server]",
"recipe[openstack-ops-database::openstack-db]",
"recipe[openstack-ops-messaging::rabbitmq-server]",
"recipe[openstack-identity::server-apache]",
"recipe[openstack-identity::registration]",
"recipe[openstack-identity::openrc]",
"recipe[openstack-image::api]",
"recipe[openstack-image::registry]",
"recipe[openstack-image::identity_registration]",
"recipe[openstack-network::identity_registration]",
"recipe[openstack-network::ml2_core_plugin]",
"recipe[openstack-network::ml2_openvswitch]",
"recipe[openstack-network]",
"recipe[openstack-network::openvswitch]",
"recipe[openstack-network::_bridge_config_example]",
"recipe[openstack-network::plugin_config]",
"recipe[openstack-network::server]",
"recipe[openstack-network::l3_agent]",
"recipe[openstack-network::dhcp_agent]",
"recipe[openstack-network::metadata_agent]",
"recipe[openstack-network::openvswitch_agent]",
"recipe[openstack-compute::nova-setup]",
"recipe[openstack-compute::identity_registration]",
"recipe[openstack-compute::conductor]",
"recipe[openstack-compute::api-os-compute]",
"recipe[openstack-compute::api-metadata]",
"recipe[openstack-compute::placement_api]",
"recipe[openstack-compute::scheduler]",
"recipe[openstack-compute::vncproxy]",
"recipe[openstack-compute::compute]",
"recipe[openstack-compute::identity_registration]",
"recipe[openstack-image::image_upload]",
"recipe[openstack-integration-test::create_network]",
"recipe[openstack-integration-test::setup]",
"recipe[openstack-integration-test::run_tempest]"
],
"env_run_lists": {
}
}

6
playbooks/pre.yaml
View File

@ -7,14 +7,12 @@
- hosts: all
vars:
chefdk: chefdk_1.6.1-1_amd64.deb
chefdkrel: 1.6.1
tasks:
- name: Purge some packages
shell: apt-get -y purge libvirt0 libvirt-dev python-libvirt
become: yes
- name: Fetch chefdk package
get_url:
dest: /tmp/{{ chefdk }}
url: https://packages.chef.io/stable/ubuntu/16.04/{{ chefdk }}
url: https://packages.chef.io/files/stable/chefdk/{{ chefdkrel }}/ubuntu/16.04/{{ chefdk }}
- name: Install chefdk package
shell: dpkg -i /tmp/{{ chefdk }}
become: yes