openstack/openstack-chef-repo
Code Issues Proposed changes

Updated integration attributes and methods

Browse Source 
- Implemented changes consistent with cookstyle
- DRY the environment files
- Added tempest recipes in minimal role; drop build-essential
- Corrected url for chefdk with current scheme
- Removed nova smoke test since this is handled by tempest and tends to fire
  before nova is available in integration

Implements blueprint modern-chef

Change-Id: I5c5371eed12b282a527576822b270cce8b0e76a7
This commit is contained in:
Samuel Cassiba 2017-12-22 17:58:46 -08:00
parent ca3a5df961
commit a317a117c8
6 changed files with 132 additions and 145 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
.rubocop_todo.yml
View File

@ -1,12 +1,14 @@
# This configuration was generated by # This configuration was generated by
# `rubocop --auto-gen-config` # `rubocop --auto-gen-config`
# on 2017-08-02 14:33:46 -0400 using RuboCop version 0.47.1. # on 2017-12-22 17:48:38 -0800 using RuboCop version 0.49.1.
# The point is for the user to remove these configuration records # The point is for the user to remove these configuration records
# one by one as the offenses are removed from the code base. # one by one as the offenses are removed from the code base.
# Note that changes in the inspected code, or installation of new # Note that changes in the inspected code, or installation of new
# versions of RuboCop, may require this file to be generated again. # versions of RuboCop, may require this file to be generated again.
# Offense count: 2 # Offense count: 1
# Configuration parameters: CountComments, ExcludedMethods. # Configuration parameters: EnforcedStyle, SupportedStyles.
Metrics/BlockLength: # SupportedStyles: for, each
Max: 33 Style/For:
Exclude:
- 'Rakefile'

27
Berksfile
View File

@ -1,25 +1,24 @@
source 'https://supermarket.chef.io' source 'https://supermarket.chef.io'
%w( %w(
integration-test integration-test
orchestration orchestration
telemetry telemetry
block-storage block-storage
common common
compute compute
dashboard dashboard
identity identity
image image
network network
ops-database ops-database
ops-messaging ops-messaging
).each do |cookbook| ).each do |cookbook|
if Dir.exist?("../cookbook-openstack-#{cookbook}") if Dir.exist?("../cookbook-openstack-#{cookbook}")
cookbook "openstack-#{cookbook}", path: "../cookbook-openstack-#{cookbook}" cookbook "openstack-#{cookbook}", path: "../cookbook-openstack-#{cookbook}"
else else
cookbook "openstack-#{cookbook}", github: "openstack/cookbook-openstack-#{cookbook}" cookbook "openstack-#{cookbook}", github: "openstack/cookbook-openstack-#{cookbook}"
end end
end end
cookbook 'openstack-dns', github: 'cloudbau/cookbook-openstack-dns' cookbook 'openstack-dns', github: 'cloudbau/cookbook-openstack-dns'

168
Rakefile
View File

@ -1,10 +1,10 @@
current_dir = File.dirname(__FILE__) current_dir = File.dirname(__FILE__)
client_opts = "--force-formatter --no-color -z --config #{current_dir}/.chef/knife.rb" client_opts = "--force-formatter --no-color -z --config #{current_dir}/.chef/knife.rb"
task default: ["test"] task default: ['test']
desc "Default gate tests to run" desc 'Default gate tests to run'
task :test => [:rubocop, :berks_vendor, :json_check] task test: [:rubocop, :berks_vendor, :json_check]
def run_command(command) def run_command(command)
if File.exist?('Gemfile.lock') if File.exist?('Gemfile.lock')
@ -15,39 +15,23 @@ def run_command(command)
end end
task :destroy_all do task :destroy_all do
Rake::Task[:destroy_machines].invoke
run_command('rm -rf Gemfile.lock && rm -rf Berksfile.lock && rm -rf cookbooks/') run_command('rm -rf Gemfile.lock && rm -rf Berksfile.lock && rm -rf cookbooks/')
end end
desc "Destroy machines" desc 'Vendor your cookbooks/'
task :destroy_machines do
run_command("chef-client #{client_opts} -o 'provisioning::destroy_all'")
end
desc "Vendor your cookbooks/"
task :berks_vendor do task :berks_vendor do
run_command('berks vendor cookbooks') run_command('berks vendor cookbooks')
end end
desc "Create Chef Key" desc 'Create Chef Key'
task :create_key do task :create_key do
if not File.exist?('.chef/validator.pem') unless File.exist?('.chef/validator.pem')
require 'openssl' require 'openssl'
File.binwrite('.chef/validator.pem', OpenSSL::PKey::RSA.new(2048).to_pem) File.binwrite('.chef/validator.pem', OpenSSL::PKey::RSA.new(2048).to_pem)
end end
end end
desc "All-in-One build" desc 'Blow everything away'
task :allinone => :create_key do
run_command("chef-client #{client_opts} -o 'provisioning::allinone'")
end
desc "Multinode build"
task :multinode => :create_key do
run_command("chef-client #{client_opts} -o 'provisioning::multinode'")
end
desc "Blow everything away"
task clean: [:destroy_all] task clean: [:destroy_all]
# CI tasks # CI tasks
@ -55,10 +39,10 @@ require 'cookstyle'
require 'rubocop/rake_task' require 'rubocop/rake_task'
desc 'Run RuboCop' desc 'Run RuboCop'
RuboCop::RakeTask.new do |task| RuboCop::RakeTask.new do |task|
task.options << "--display-cop-names" task.options << '--display-cop-names'
end end
desc "Validate data bags, environments and roles" desc 'Validate data bags, environments and roles'
task :json_check do task :json_check do
require 'json' require 'json'
['data_bags/*', 'environments', 'roles'].each do |sub_dir| ['data_bags/*', 'environments', 'roles'].each do |sub_dir|
@ -70,7 +54,7 @@ task :json_check do
end end
# Helper for running various testing commands # Helper for running various testing commands
def _run_commands(desc, commands, openstack=true) def _run_commands(desc, commands, openstack = true)
puts "## Running #{desc}" puts "## Running #{desc}"
commands.each do |command, options| commands.each do |command, options|
options.each do |option| options.each do |option|
@ -92,121 +76,77 @@ elsif File.exist?('/usr/bin/yum')
end end
# Helper for looking at the starting environment # Helper for looking at the starting environment
def _run_env_queries # rubocop:disable Metrics/MethodLength def _run_env_queries
_run_commands('basic common env queries', { _run_commands('basic common env queries', {
'uname' => ['-a'], 'uname' => ['-a'],
'pwd' => [''], 'pwd' => [''],
'env' => ['']}, 'env' => [''] },
false false)
)
case @platform case @platform
when 'ubuntu16' when 'ubuntu16'
_run_commands('basic debian env queries', { _run_commands('basic debian env queries', {
'ifconfig' => [''], 'ifconfig' => [''],
'cat' => ['/etc/apt/sources.list']}, 'cat' => ['/etc/apt/sources.list'] },
false false)
)
when 'centos7' when 'centos7'
_run_commands('basic rhel env queries', { _run_commands('basic rhel env queries', {
'/usr/sbin/ip' => ['addr'], '/usr/sbin/ip' => ['addr'],
'cat' => ['/etc/yum.repos.d/*']}, 'cat' => ['/etc/yum.repos.d/*'] },
false false)
)
end end
end end
# Helper for setting up basic query tests # Helper for setting up basic query tests
def _run_basic_queries # rubocop:disable Metrics/MethodLength def _run_basic_queries
_run_commands('basic common test queries', { _run_commands('basic common test queries',
'sudo netstat' => ['-nlp'], 'sudo netstat' => ['-nlp'],
'nova-manage' => ['version', 'db version'], 'nova-manage' => ['version', 'db version'],
'nova' => %w(--version service-list hypervisor-list flavor-list), 'nova' => %w(--version service-list hypervisor-list flavor-list),
'glance-manage' => %w(db_version), 'glance-manage' => %w(db_version),
'glance' => %w(--version image-list), 'glance' => %w(--version image-list),
'keystone-manage' => %w(db_version), 'keystone-manage' => %w(db_version),
'openstack' => ['--version', 'user list', 'endpoint list', 'role list', 'openstack' => ['--version', 'user list', 'endpoint list', 'role list',
'service list', 'project list', 'service list', 'project list',
'network agent list', 'extension list --network ', 'network agent list', 'extension list --network ',
'network list', 'subnet list'], 'network list', 'subnet list'],
'neutron' => %w(port-list quota-list), 'ovs-vsctl' => %w(show))
'ovs-vsctl' => %w(show) }
)
case @platform case @platform
when 'ubuntu16' when 'ubuntu16'
_run_commands('basic debian test queries', { _run_commands('basic debian test queries',
'rabbitmqctl' => %w(cluster_status), 'rabbitmqctl' => %w(cluster_status),
'ip' => ['addr', 'route', '-6 route']} 'ip' => ['addr', 'route', '-6 route'])
)
when 'centos7' when 'centos7'
_run_commands('basic rhel test queries', { _run_commands('basic rhel test queries',
'/usr/sbin/rabbitmqctl' => %w(cluster_status), '/usr/sbin/rabbitmqctl' => %w(cluster_status),
'/usr/sbin/ip' => ['addr', 'route', '-6 route']} '/usr/sbin/ip' => ['addr', 'route', '-6 route'])
)
end end
end end
# Helper for setting up basic nova tests
def _run_nova_tests(pass) # rubocop:disable Metrics/MethodLength
_run_commands('nova server create', {
'openstack' => ['server list', "server create --image cirros --flavor m1.nano test#{pass}"],
'sleep' => ['40'] }
)
_run_commands('nova server cleanup', {
'openstack' => ['server list', "server show test#{pass}", "server delete test#{pass}"],
'sleep' => ['15'] }
)
_run_commands('nova server query', {
'openstack' => ['server list'] }
)
end
# Helper for setting up neutron local network
def _setup_local_network # rubocop:disable Metrics/MethodLength
_run_commands('neutron local network setup', {
'openstack' => ['network create --share local_net', 'subnet create --network local_net --subnet-range 192.168.180.0/24 local_subnet'] }
)
end
# Helper for setting up tempest and upload the default cirros image.
def _setup_tempest(client_opts, log_dir)
sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'recipe[openstack-integration-test::setup]' > #{log_dir}/chef-client-setup-tempest.txt 2>&1 )
end
def _save_logs(prefix, log_dir) def _save_logs(prefix, log_dir)
sh %(sleep 25) sh %(sleep 25)
%w(nova neutron keystone glance apache2 rabbitmq mysql-default openvswitch mariadb).each do |project| %w(nova neutron keystone glance apache2 rabbitmq mysql mysql-default openvswitch mariadb).each do |project|
sh %(mkdir -p #{log_dir}/#{prefix}/#{project}) sh %(mkdir -p #{log_dir}/#{prefix}/#{project})
sh %(sudo cp -rL /etc/#{project} #{log_dir}/#{prefix}/#{project}/etc || true) sh %(sudo cp -rL /etc/#{project} #{log_dir}/#{prefix}/#{project}/etc || true)
sh %(sudo cp -rL /var/log/#{project} #{log_dir}/#{prefix}/#{project}/log || true) sh %(sudo cp -rL /var/log/#{project} #{log_dir}/#{prefix}/#{project}/log || true)
end end
end end
desc "Integration test on Infra" desc 'Integration test on Infra'
task :integration => [:create_key, :berks_vendor] do task integration: [:create_key, :berks_vendor] do
log_dir = ENV['WORKSPACE']+'/logs' log_dir = ENV['WORKSPACE'] + '/logs'
sh %(mkdir #{log_dir}) sh %(mkdir #{log_dir})
# This is a workaround for allowing chef-client to run in local mode # This is a workaround for allowing chef-client to run in local mode
sh %(sudo mkdir -p /etc/chef && sudo cp .chef/encrypted_data_bag_secret /etc/chef/openstack_data_bag_secret) sh %(sudo mkdir -p /etc/chef && sudo cp .chef/encrypted_data_bag_secret /etc/chef/openstack_data_bag_secret)
_run_env_queries _run_env_queries
# Install mysql2 gem to avoid hitting mirror issues # Three passes to ensure idempotency. prefer each to times, even if it
sh %(wget https://rubygems.org/downloads/mysql2-0.4.5.gem) # reads weird
sh %(sudo apt-get install -y libmysqlclient-dev) for i in 1..3 do
sh %(chef exec gem install -N ./mysql2-0.4.5.gem)
# Three passes to make sure of cookbooks idempotency
for i in 1..3
begin begin
puts "####### Pass #{i}" puts "####### Pass #{i}"
# Kick off chef client in local mode, will converge OpenStack right on the gate job "in place" # Kick off chef client in local mode, will converge OpenStack right on the gate job "in place"
sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 ) sh %(sudo chef-client #{client_opts} -E integration -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 )
if i == 1 _run_basic_queries
_setup_tempest(client_opts, log_dir)
_setup_local_network
end
_run_basic_queries
_run_nova_tests(i)
rescue => e rescue => e
raise "####### Pass #{i} failed with #{e.message}" raise "####### Pass #{i} failed with #{e.message}"
ensure ensure
@ -216,8 +156,4 @@ task :integration => [:create_key, :berks_vendor] do
sh %(sudo chmod -R go+rx #{log_dir}/pass#{i}) sh %(sudo chmod -R go+rx #{log_dir}/pass#{i})
end end
end end
# Run the tempest formal tests, setup with the openstack-integration-test cookbook
Dir.chdir('/opt/tempest') do
sh %(sudo -H /opt/tempest-venv/tempest.sh)
end
end end

11
environments/integration-ubuntu16.json → environments/integration.json
View File

@ -1,6 +1,6 @@
{ {
"name": "integration-ubuntu16", "name": "integration",
"description": "Environment used in testing the upstream cookbooks and reference Chef repository with vagrant. To be used with the vagrantfile-allinone vagrantfile. Defines the necessary attributes for a working all-in-one openstack deployment, using neutron for the networking component, and the openvswitch neutron plugin", "description": "Chef environment file for building OpenStack in CI settings. Certain Tempest services are disabled in this scenario",
"cookbook_versions": { "cookbook_versions": {
}, },
"json_class": "Chef::Environment", "json_class": "Chef::Environment",
@ -8,13 +8,11 @@
"default_attributes": { "default_attributes": {
"apache": { "apache": {
"listen": [] "listen": []
},
"apt": {
"compile_time_update" : true
} }
}, },
"override_attributes": { "override_attributes": {
"openstack": { "openstack": {
"is_release": true,
"apt": { "apt": {
"update_apt_cache": true "update_apt_cache": true
}, },
@ -41,7 +39,7 @@
"mq": { "mq": {
"user": "admin" "user": "admin"
}, },
"network":{ "network": {
"conf": { "conf": {
"DEFAULT": { "DEFAULT": {
"service_plugins": "router" "service_plugins": "router"
@ -54,6 +52,7 @@
"compute": { "compute": {
"conf": { "conf": {
"libvirt": { "libvirt": {
"cpu_type": "none",
"virt_type": "qemu" "virt_type": "qemu"
} }
} }

53
minimal.json Normal file
View File

@ -0,0 +1,53 @@
{
"name": "minimal",
"description": "This will deploy a minimal set of services for Openstack Compute to function for integration testing.",
"json_class": "Chef::Role",
"default_attributes": {
},
"override_attributes": {
},
"chef_type": "role",
"run_list": [
"recipe[openstack-common]",
"recipe[openstack-common::logging]",
"recipe[openstack-common::sysctl]",
"recipe[openstack-common::client]",
"recipe[openstack-ops-database::server]",
"recipe[openstack-ops-database::openstack-db]",
"recipe[openstack-ops-messaging::rabbitmq-server]",
"recipe[openstack-identity::server-apache]",
"recipe[openstack-identity::registration]",
"recipe[openstack-identity::openrc]",
"recipe[openstack-image::api]",
"recipe[openstack-image::registry]",
"recipe[openstack-image::identity_registration]",
"recipe[openstack-network::identity_registration]",
"recipe[openstack-network::ml2_core_plugin]",
"recipe[openstack-network::ml2_openvswitch]",
"recipe[openstack-network]",
"recipe[openstack-network::openvswitch]",
"recipe[openstack-network::_bridge_config_example]",
"recipe[openstack-network::plugin_config]",
"recipe[openstack-network::server]",
"recipe[openstack-network::l3_agent]",
"recipe[openstack-network::dhcp_agent]",
"recipe[openstack-network::metadata_agent]",
"recipe[openstack-network::openvswitch_agent]",
"recipe[openstack-compute::nova-setup]",
"recipe[openstack-compute::identity_registration]",
"recipe[openstack-compute::conductor]",
"recipe[openstack-compute::api-os-compute]",
"recipe[openstack-compute::api-metadata]",
"recipe[openstack-compute::placement_api]",
"recipe[openstack-compute::scheduler]",
"recipe[openstack-compute::vncproxy]",
"recipe[openstack-compute::compute]",
"recipe[openstack-compute::identity_registration]",
"recipe[openstack-image::image_upload]",
"recipe[openstack-integration-test::create_network]",
"recipe[openstack-integration-test::setup]",
"recipe[openstack-integration-test::run_tempest]"
],
"env_run_lists": {
}
}

6
playbooks/pre.yaml
View File

@ -7,14 +7,12 @@
- hosts: all - hosts: all
vars: vars:
chefdk: chefdk_1.6.1-1_amd64.deb chefdk: chefdk_1.6.1-1_amd64.deb
chefdkrel: 1.6.1
tasks: tasks:
- name: Purge some packages
shell: apt-get -y purge libvirt0 libvirt-dev python-libvirt
become: yes
- name: Fetch chefdk package - name: Fetch chefdk package
get_url: get_url:
dest: /tmp/{{ chefdk }} dest: /tmp/{{ chefdk }}
url: https://packages.chef.io/stable/ubuntu/16.04/{{ chefdk }} url: https://packages.chef.io/files/stable/chefdk/{{ chefdkrel }}/ubuntu/16.04/{{ chefdk }}
- name: Install chefdk package - name: Install chefdk package
shell: dpkg -i /tmp/{{ chefdk }} shell: dpkg -i /tmp/{{ chefdk }}
become: yes become: yes