Updated integration attributes and methods

- Implemented changes consistent with cookstyle
- DRY the environment files
- Added tempest recipes in minimal role; drop build-essential
- Corrected url for chefdk with current scheme
- Removed nova smoke test since this is handled by tempest and tends to fire
  before nova is available in integration

Implements blueprint modern-chef

Change-Id: I5c5371eed12b282a527576822b270cce8b0e76a7

.rubocop_todo.yml

@@ -1,12 +1,14 @@
 # This configuration was generated by
 # `rubocop --auto-gen-config`
-# on 2017-08-02 14:33:46 -0400 using RuboCop version 0.47.1.
+# on 2017-12-22 17:48:38 -0800 using RuboCop version 0.49.1.
 # The point is for the user to remove these configuration records
 # one by one as the offenses are removed from the code base.
 # Note that changes in the inspected code, or installation of new
 # versions of RuboCop, may require this file to be generated again.
 
-# Offense count: 2
-# Configuration parameters: CountComments, ExcludedMethods.
-Metrics/BlockLength:
-  Max: 33
+# Offense count: 1
+# Configuration parameters: EnforcedStyle, SupportedStyles.
+# SupportedStyles: for, each
+Style/For:
+  Exclude:
+    - 'Rakefile'

Berksfile

@@ -1,25 +1,24 @@
 source 'https://supermarket.chef.io'
 
 %w(
-    integration-test
-    orchestration
-    telemetry
-    block-storage
-    common
-    compute
-    dashboard
-    identity
-    image
-    network
-    ops-database
-    ops-messaging
-  ).each do |cookbook|
+  integration-test
+  orchestration
+  telemetry
+  block-storage
+  common
+  compute
+  dashboard
+  identity
+  image
+  network
+  ops-database
+  ops-messaging
+).each do |cookbook|
   if Dir.exist?("../cookbook-openstack-#{cookbook}")
     cookbook "openstack-#{cookbook}", path: "../cookbook-openstack-#{cookbook}"
   else
     cookbook "openstack-#{cookbook}", github: "openstack/cookbook-openstack-#{cookbook}"
   end
-
 end
 
 cookbook 'openstack-dns', github: 'cloudbau/cookbook-openstack-dns'

Rakefile

@@ -1,10 +1,10 @@
 current_dir = File.dirname(__FILE__)
 client_opts = "--force-formatter --no-color -z --config #{current_dir}/.chef/knife.rb"
 
-task default: ["test"]
+task default: ['test']
 
-desc "Default gate tests to run"
-task :test => [:rubocop, :berks_vendor, :json_check]
+desc 'Default gate tests to run'
+task test: [:rubocop, :berks_vendor, :json_check]
 
 def run_command(command)
   if File.exist?('Gemfile.lock')
@@ -15,39 +15,23 @@ def run_command(command)
 end
 
 task :destroy_all do
-  Rake::Task[:destroy_machines].invoke
   run_command('rm -rf Gemfile.lock && rm -rf Berksfile.lock && rm -rf cookbooks/')
 end
 
-desc "Destroy machines"
-task :destroy_machines do
-  run_command("chef-client #{client_opts} -o 'provisioning::destroy_all'")
-end
-
-desc "Vendor your cookbooks/"
+desc 'Vendor your cookbooks/'
 task :berks_vendor do
   run_command('berks vendor cookbooks')
 end
 
-desc "Create Chef Key"
+desc 'Create Chef Key'
 task :create_key do
-  if not File.exist?('.chef/validator.pem')
+  unless File.exist?('.chef/validator.pem')
     require 'openssl'
     File.binwrite('.chef/validator.pem', OpenSSL::PKey::RSA.new(2048).to_pem)
   end
 end
 
-desc "All-in-One build"
-task :allinone => :create_key do
-  run_command("chef-client #{client_opts} -o 'provisioning::allinone'")
-end
-
-desc "Multinode build"
-task :multinode => :create_key do
-  run_command("chef-client #{client_opts} -o 'provisioning::multinode'")
-end
-
-desc "Blow everything away"
+desc 'Blow everything away'
 task clean: [:destroy_all]
 
 # CI tasks
@@ -55,10 +39,10 @@ require 'cookstyle'
 require 'rubocop/rake_task'
 desc 'Run RuboCop'
 RuboCop::RakeTask.new do |task|
-  task.options << "--display-cop-names"
+  task.options << '--display-cop-names'
 end
 
-desc "Validate data bags, environments and roles"
+desc 'Validate data bags, environments and roles'
 task :json_check do
   require 'json'
   ['data_bags/*', 'environments', 'roles'].each do |sub_dir|
@@ -70,7 +54,7 @@ task :json_check do
 end
 
 # Helper for running various testing commands
-def _run_commands(desc, commands, openstack=true)
+def _run_commands(desc, commands, openstack = true)
   puts "## Running #{desc}"
   commands.each do |command, options|
     options.each do |option|
@@ -92,121 +76,77 @@ elsif File.exist?('/usr/bin/yum')
 end
 
 # Helper for looking at the starting environment
-def _run_env_queries # rubocop:disable Metrics/MethodLength
-    _run_commands('basic common env queries', {
-      'uname' => ['-a'],
-      'pwd' => [''],
-      'env' => ['']},
-      false
-    )
+def _run_env_queries
+  _run_commands('basic common env queries', {
+                  'uname' => ['-a'],
+                  'pwd' => [''],
+                  'env' => [''] },
+    false)
   case @platform
   when 'ubuntu16'
     _run_commands('basic debian env queries', {
-      'ifconfig' => [''],
-      'cat' => ['/etc/apt/sources.list']},
-      false
-    )
+                    'ifconfig' => [''],
+                    'cat' => ['/etc/apt/sources.list'] },
+      false)
   when 'centos7'
     _run_commands('basic rhel env queries', {
-      '/usr/sbin/ip' => ['addr'],
-      'cat' => ['/etc/yum.repos.d/*']},
-      false
-    )
+                    '/usr/sbin/ip' => ['addr'],
+                    'cat' => ['/etc/yum.repos.d/*'] },
+    false)
   end
 end
 
 # Helper for setting up basic query tests
-def _run_basic_queries # rubocop:disable Metrics/MethodLength
-  _run_commands('basic common test queries', {
-      'sudo netstat' => ['-nlp'],
-      'nova-manage' => ['version', 'db version'],
-      'nova' => %w(--version service-list hypervisor-list flavor-list),
-      'glance-manage' => %w(db_version),
-      'glance' => %w(--version image-list),
-      'keystone-manage' => %w(db_version),
-      'openstack' => ['--version', 'user list', 'endpoint list', 'role list',
-                      'service list', 'project list',
-                      'network agent list', 'extension list --network ',
-                      'network list', 'subnet list'],
-      'neutron' => %w(port-list quota-list),
-      'ovs-vsctl' => %w(show) }
-    )
+def _run_basic_queries
+  _run_commands('basic common test queries',
+                'sudo netstat' => ['-nlp'],
+                'nova-manage' => ['version', 'db version'],
+                'nova' => %w(--version service-list hypervisor-list flavor-list),
+                'glance-manage' => %w(db_version),
+                'glance' => %w(--version image-list),
+                'keystone-manage' => %w(db_version),
+                'openstack' => ['--version', 'user list', 'endpoint list', 'role list',
+                                'service list', 'project list',
+                                'network agent list', 'extension list --network ',
+                                'network list', 'subnet list'],
+                'ovs-vsctl' => %w(show))
   case @platform
   when 'ubuntu16'
-    _run_commands('basic debian test queries', {
-      'rabbitmqctl' => %w(cluster_status),
-      'ip' => ['addr', 'route', '-6 route']}
-    )
+    _run_commands('basic debian test queries',
+                  'rabbitmqctl' => %w(cluster_status),
+                  'ip' => ['addr', 'route', '-6 route'])
   when 'centos7'
-    _run_commands('basic rhel test queries', {
-      '/usr/sbin/rabbitmqctl' => %w(cluster_status),
-      '/usr/sbin/ip' => ['addr', 'route', '-6 route']}
-    )
+    _run_commands('basic rhel test queries',
+                  '/usr/sbin/rabbitmqctl' => %w(cluster_status),
+                  '/usr/sbin/ip' => ['addr', 'route', '-6 route'])
   end
 end
 
-# Helper for setting up basic nova tests
-def _run_nova_tests(pass) # rubocop:disable Metrics/MethodLength
-  _run_commands('nova server create', {
-    'openstack' => ['server list', "server create --image cirros --flavor m1.nano test#{pass}"],
-    'sleep' => ['40'] }
-  )
-  _run_commands('nova server cleanup', {
-    'openstack' => ['server list', "server show test#{pass}", "server delete test#{pass}"],
-    'sleep' => ['15'] }
-  )
- _run_commands('nova server query', {
-    'openstack' => ['server list'] }
-  )
-end
-
-# Helper for setting up neutron local network
-def _setup_local_network # rubocop:disable Metrics/MethodLength
-  _run_commands('neutron local network setup', {
-    'openstack' => ['network create --share local_net', 'subnet create --network local_net --subnet-range 192.168.180.0/24 local_subnet'] }
-  )
-end
-
-# Helper for setting up tempest and upload the default cirros image.
-def _setup_tempest(client_opts, log_dir)
-  sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'recipe[openstack-integration-test::setup]' > #{log_dir}/chef-client-setup-tempest.txt 2>&1 )
-end
-
 def _save_logs(prefix, log_dir)
   sh %(sleep 25)
-  %w(nova neutron keystone glance apache2 rabbitmq mysql-default openvswitch mariadb).each do |project|
+  %w(nova neutron keystone glance apache2 rabbitmq mysql mysql-default openvswitch mariadb).each do |project|
     sh %(mkdir -p #{log_dir}/#{prefix}/#{project})
     sh %(sudo cp -rL /etc/#{project} #{log_dir}/#{prefix}/#{project}/etc || true)
     sh %(sudo cp -rL /var/log/#{project} #{log_dir}/#{prefix}/#{project}/log || true)
   end
 end
 
-desc "Integration test on Infra"
-task :integration => [:create_key, :berks_vendor] do
-  log_dir = ENV['WORKSPACE']+'/logs'
+desc 'Integration test on Infra'
+task integration: [:create_key, :berks_vendor] do
+  log_dir = ENV['WORKSPACE'] + '/logs'
   sh %(mkdir #{log_dir})
   # This is a workaround for allowing chef-client to run in local mode
   sh %(sudo mkdir -p /etc/chef && sudo cp .chef/encrypted_data_bag_secret /etc/chef/openstack_data_bag_secret)
   _run_env_queries
 
-  # Install mysql2 gem to avoid hitting mirror issues
-  sh %(wget https://rubygems.org/downloads/mysql2-0.4.5.gem)
-  sh %(sudo apt-get install -y libmysqlclient-dev)
-  sh %(chef exec gem install -N ./mysql2-0.4.5.gem)
-
-  # Three passes to make sure of cookbooks idempotency
-  for i in 1..3
+  # Three passes to ensure idempotency. prefer each to times, even if it
+  # reads weird
+  for i in 1..3 do
     begin
-    puts "####### Pass #{i}"
-    # Kick off chef client in local mode, will converge OpenStack right on the gate job "in place"
-    sh %(sudo chef-client #{client_opts} -E integration-#{@platform} -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 )
-    if i == 1
-      _setup_tempest(client_opts, log_dir)
-      _setup_local_network
-    end
-    _run_basic_queries
-    _run_nova_tests(i)
-
+      puts "####### Pass #{i}"
+      # Kick off chef client in local mode, will converge OpenStack right on the gate job "in place"
+      sh %(sudo chef-client #{client_opts} -E integration -r 'role[minimal]' > #{log_dir}/chef-client-pass#{i}.txt 2>&1 )
+      _run_basic_queries
     rescue => e
       raise "####### Pass #{i} failed with #{e.message}"
     ensure
@@ -216,8 +156,4 @@ task :integration => [:create_key, :berks_vendor] do
       sh %(sudo chmod -R go+rx #{log_dir}/pass#{i})
     end
   end
-  # Run the tempest formal tests, setup with the openstack-integration-test cookbook
-   Dir.chdir('/opt/tempest') do
-     sh %(sudo -H /opt/tempest-venv/tempest.sh)
-   end
 end

environments/integration.json

@@ -1,6 +1,6 @@
 {
-  "name": "integration-ubuntu16",
-  "description": "Environment used in testing the upstream cookbooks and reference Chef repository with vagrant. To be used with the vagrantfile-allinone vagrantfile. Defines the necessary attributes for a working all-in-one openstack deployment, using neutron for the networking component, and the openvswitch neutron plugin",
+  "name": "integration",
+  "description": "Chef environment file for building OpenStack in CI settings. Certain Tempest services are disabled in this scenario",
   "cookbook_versions": {
   },
   "json_class": "Chef::Environment",
@@ -8,13 +8,11 @@
   "default_attributes": {
     "apache": {
       "listen": []
-    },
-    "apt": {
-      "compile_time_update" : true
     }
   },
   "override_attributes": {
     "openstack": {
+      "is_release": true,
       "apt": {
         "update_apt_cache": true
       },
@@ -41,7 +39,7 @@
       "mq": {
         "user": "admin"
       },
-      "network":{
+      "network": {
         "conf": {
           "DEFAULT": {
             "service_plugins": "router"
@@ -54,6 +52,7 @@
       "compute": {
         "conf": {
           "libvirt": {
+            "cpu_type": "none",
             "virt_type": "qemu"
           }
         }

minimal.json

@@ -0,0 +1,53 @@
+{
+  "name": "minimal",
+  "description": "This will deploy a minimal set of services for Openstack Compute to function for integration testing.",
+  "json_class": "Chef::Role",
+  "default_attributes": {
+  },
+  "override_attributes": {
+  },
+  "chef_type": "role",
+  "run_list": [
+    "recipe[openstack-common]",
+    "recipe[openstack-common::logging]",
+    "recipe[openstack-common::sysctl]",
+    "recipe[openstack-common::client]",
+    "recipe[openstack-ops-database::server]",
+    "recipe[openstack-ops-database::openstack-db]",
+    "recipe[openstack-ops-messaging::rabbitmq-server]",
+    "recipe[openstack-identity::server-apache]",
+    "recipe[openstack-identity::registration]",
+    "recipe[openstack-identity::openrc]",
+    "recipe[openstack-image::api]",
+    "recipe[openstack-image::registry]",
+    "recipe[openstack-image::identity_registration]",
+    "recipe[openstack-network::identity_registration]",
+    "recipe[openstack-network::ml2_core_plugin]",
+    "recipe[openstack-network::ml2_openvswitch]",
+    "recipe[openstack-network]",
+    "recipe[openstack-network::openvswitch]",
+    "recipe[openstack-network::_bridge_config_example]",
+    "recipe[openstack-network::plugin_config]",
+    "recipe[openstack-network::server]",
+    "recipe[openstack-network::l3_agent]",
+    "recipe[openstack-network::dhcp_agent]",
+    "recipe[openstack-network::metadata_agent]",
+    "recipe[openstack-network::openvswitch_agent]",
+    "recipe[openstack-compute::nova-setup]",
+    "recipe[openstack-compute::identity_registration]",
+    "recipe[openstack-compute::conductor]",
+    "recipe[openstack-compute::api-os-compute]",
+    "recipe[openstack-compute::api-metadata]",
+    "recipe[openstack-compute::placement_api]",
+    "recipe[openstack-compute::scheduler]",
+    "recipe[openstack-compute::vncproxy]",
+    "recipe[openstack-compute::compute]",
+    "recipe[openstack-compute::identity_registration]",
+    "recipe[openstack-image::image_upload]",
+    "recipe[openstack-integration-test::create_network]",
+    "recipe[openstack-integration-test::setup]",
+    "recipe[openstack-integration-test::run_tempest]"
+  ],
+  "env_run_lists": {
+  }
+}

playbooks/pre.yaml

@@ -7,14 +7,12 @@
 - hosts: all
   vars:
     chefdk: chefdk_1.6.1-1_amd64.deb
+    chefdkrel: 1.6.1
   tasks:
-  - name: Purge some packages
-    shell: apt-get -y purge libvirt0 libvirt-dev python-libvirt
-    become: yes
   - name: Fetch chefdk package
     get_url:
       dest: /tmp/{{ chefdk }}
-      url: https://packages.chef.io/stable/ubuntu/16.04/{{ chefdk }}
+      url: https://packages.chef.io/files/stable/chefdk/{{ chefdkrel }}/ubuntu/16.04/{{ chefdk }}
   - name: Install chefdk package
     shell: dpkg -i /tmp/{{ chefdk }}
     become: yes