Revert from packaged tempest to git, update image refs

- due to inconsistencies in RDO versus UCA packaging, Tempest performs more
  consistently from upstream git
- update cirros references to 0.4.0

Change-Id: I374497db1b602683f1f9c62489f523b18c50266c
Implements: blueprint modern-chef

attributes/default.rb

@@ -44,13 +44,13 @@ default['openstack']['integration-test'] = {
     'name' => 'cirros-test1',
     'id' => '1ac790f6-903a-4833-979f-a38f1819e3b1',
     'flavor' => 99,
-    'source' => 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img',
+    'source' => 'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img',
   },
   'image2' => {
     'name' => 'cirros-test2',
     'id' => 'f7c2ac6d-0011-499f-a9ec-ca71348bf2e4',
     'flavor' => 99,
-    'source' => 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img',
+    'source' => 'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img',
   },
 }
 
@@ -58,18 +58,20 @@ default['openstack']['integration-test'] = {
 case node['platform_family']
 when 'fedora', 'rhel' # :pragma-foodcritic: ~FC024 - won't fix this
   default['openstack']['integration-test']['platform'] = {
-    tempest_packages: %w(git libxslt-devel
+    tempest_packages: %w(git curl libxslt-devel
                          libxml2-devel python-testrepository
                          libffi-devel python-devel python-setuptools
-                         openstack-tempest),
+                         python-gabbi python-testscenarios
+                         python-ddt),
     package_overrides: '',
   }
 when 'debian'
   default['openstack']['integration-test']['platform'] = {
-    'tempest_packages' => %w(git libssl-dev libffi-dev python-dev libxml2-dev
+    'tempest_packages' => %w(git curl libssl-dev libffi-dev python-dev libxml2-dev
                              libxslt1-dev libpq-dev libxml2-dev libxslt-dev
                              testrepository python-dev libffi-dev
-                             python-gabbi tempest),
+                             python-gabbi python-testscenarios
+                             python-ddt),
     'package_overrides' => '',
   }
 end

attributes/tempest_conf.rb

@@ -1,5 +1,7 @@
 default['openstack']['integration-test']['conf_secrets'] = {}
 default['openstack']['integration-test']['conf'].tap do |conf|
+  conf['DEFAULT']['log_dir'] = '/opt/tempest/logs'
+  conf['DEFAULT']['log_file'] = 'tempest.log'
   conf['auth']['use_dynamic_credentials'] = node['openstack']['integration-test']['use_dynamic_credentials']
   conf['auth']['default_credentials_domain_name'] = 'Default'
   conf['auth']['admin_domain_name'] = 'Default'
@@ -91,4 +93,5 @@ default['openstack']['integration-test']['conf'].tap do |conf|
   conf['service_available']['glance'] = true
   conf['service_available']['cinder'] = false
   conf['service_available']['nova'] = true
+  conf['oslo_concurrency']['lock_path'] = '/opt/tempest/tempest_lock'
 end

recipes/run_tempest.rb

@@ -19,7 +19,7 @@
 
 execute 'run_tempest' do
   # Write log file to test VM's /root directory.
-  command 'tempest run --smoke --serial --config-file /etc/tempest/tempest.conf'
+  command 'cd /opt/tempest; /opt/tempest-venv/tempest.sh'
   action :nothing
 end
 

recipes/setup.rb

@@ -27,7 +27,8 @@ end
 
 platform_options = node['openstack']['integration-test']['platform']
 
-python_runtime '2' do
+python_runtime 'tempest' do
+  version '2'
   provider :system
 end
 
@@ -39,8 +40,6 @@ platform_options['tempest_packages'].each do |pkg|
   end
 end
 
-package 'curl'
-
 identity_admin_endpoint = admin_endpoint 'identity'
 identity_public_endpoint = public_endpoint 'identity'
 auth_url = ::URI.decode identity_admin_endpoint.to_s
@@ -70,35 +69,52 @@ connection_params = {
     connection_params connection_params
   end
 
-  openstack_role service_role do
-    connection_params connection_params
-  end
-
   openstack_user service_user do
+    role_name service_role
     project_name service_project
     domain_name service_domain
     password service_pass
     connection_params connection_params
-  end
+    action [:create, :grant_role, :grant_domain]
-
-  openstack_user service_user do
-    role_name service_role
-    project_name service_project
-    connection_params connection_params
-    action :grant_role
   end
 
   heat_stack_user_role = node['openstack']['integration-test']['heat_stack_user_role']
   openstack_role heat_stack_user_role do
     connection_params connection_params
   end
+end
 
-  openstack_user service_user do
+tempest_path = '/opt/tempest'
-    domain_name service_domain
+venv_path = '/opt/tempest-venv'
-    user_name service_user
+
-    connection_params connection_params
+python_virtualenv venv_path do
-    action :grant_domain
+  python 'tempest'
-  end
+  system_site_packages true
+end
+
+python_execute 'install tempest' do
+  action :nothing
+  command '-m pip install .'
+  cwd tempest_path
+  virtualenv venv_path
+end
+
+git tempest_path do
+  repository 'https://github.com/openstack/tempest'
+  reference 'master'
+  depth 1
+  action :sync
+  notifies :run, 'python_execute[install tempest]', :immediately
+end
+
+template "#{venv_path}/tempest.sh" do
+  source 'tempest.sh.erb'
+  user 'root'
+  group 'root'
+  mode 0o755
+  variables(
+    venv_path: venv_path
+  )
 end
 
 %w(image1 image2).each do |img|
@@ -150,7 +166,7 @@ end
 integration_test_conf_options = merge_config_options 'integration-test'
 
 # create the keystone.conf from attributes
-template '/etc/tempest/tempest.conf' do
+template '/opt/tempest/etc/tempest.conf' do
   source 'openstack-service.conf.erb'
   cookbook 'openstack-common'
   owner 'root'
@@ -161,6 +177,13 @@ template '/etc/tempest/tempest.conf' do
   )
 end
 
+directory '/opt/tempest/logs' do
+  owner 'root'
+  group 'root'
+  mode 0755
+  action :create
+end
+
 # delete all secrets saved in the attribute
 # node['openstack']['identity']['conf_secrets'] after creating the keystone.conf
 ruby_block "delete all attributes in node['openstack']['integration-test']['conf_secrets']" do

spec/setup-redhat_spec.rb

@@ -11,11 +11,12 @@ describe 'openstack-integration-test::setup' do
 
     include_context 'tempest-stubs'
 
-    it 'installs tempest from package' do
+    it 'installs tempest dependencies' do
-      %w(git libxslt-devel
+      %w(git curl libxslt-devel
          libxml2-devel python-testrepository
          libffi-devel python-devel python-setuptools
-         openstack-tempest).each do |pkg|
+         python-gabbi python-testscenarios
+         python-ddt).each do |pkg|
         expect(chef_run).to upgrade_package(pkg)
       end
     end

spec/setup_spec.rb

@@ -41,20 +41,13 @@ describe 'openstack-integration-test::setup' do
         'tempest_user1'
       ).with(
         domain_name: 'Default',
+        role_name: 'Member',
         project_name: 'tempest_project1',
         password: 'tempest_user1_pass',
         connection_params: connection_params
       )
     end
 
-    it 'create service role' do
-      expect(chef_run).to create_openstack_role(
-        'Member'
-      ).with(
-        connection_params: connection_params
-      )
-    end
-
     it do
       expect(chef_run).to grant_role_openstack_user(
         'tempest_user1'
@@ -79,6 +72,7 @@ describe 'openstack-integration-test::setup' do
         'tempest_user2'
       ).with(
         domain_name: 'Default',
+        role_name: 'Member',
         project_name: 'tempest_project2',
         password: 'tempest_user2_pass',
         connection_params: connection_params
@@ -96,11 +90,13 @@ describe 'openstack-integration-test::setup' do
       )
     end
 
-    it 'create service role' do
+    it 'syncs /opt/tempest from github' do
-      expect(chef_run).to create_openstack_role(
+      expect(chef_run).to sync_git(
-        'heat_stack_owner'
+        '/opt/tempest'
       ).with(
-        connection_params: connection_params
+        repository: 'https://github.com/openstack/tempest',
+        reference: 'master',
+        depth: 1
       )
     end
 
@@ -114,7 +110,7 @@ describe 'openstack-integration-test::setup' do
         identity_project_domain_name: 'default',
         image_name: 'cirros-test1',
         image_id: '1ac790f6-903a-4833-979f-a38f1819e3b1',
-        image_url: 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img'
+        image_url: 'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img'
       )
     end
 
@@ -128,12 +124,12 @@ describe 'openstack-integration-test::setup' do
         identity_project_domain_name: 'default',
         image_name: 'cirros-test2',
         image_id: 'f7c2ac6d-0011-499f-a9ec-ca71348bf2e4',
-        image_url: 'http://download.cirros-cloud.net/0.3.5/cirros-0.3.5-x86_64-disk.img'
+        image_url: 'http://download.cirros-cloud.net/0.4.0/cirros-0.4.0-x86_64-disk.img'
       )
     end
 
     describe 'tempest.conf default' do
-      let(:file) { chef_run.template('/etc/tempest/tempest.conf') }
+      let(:file) { chef_run.template('/opt/tempest/etc/tempest.conf') }
 
       it 'creates tempest.conf' do
         expect(chef_run).to create_template(file.name).with(
@@ -174,7 +170,7 @@ describe 'openstack-integration-test::setup' do
         runner.node.normal['openstack']['endpoints']['public']['identity']['scheme'] = 'https'
         runner.converge(described_recipe)
       end
-      let(:file) { chef_run.template('/etc/tempest/tempest.conf') }
+      let(:file) { chef_run.template('/opt/tempest/etc/tempest.conf') }
 
       it 'has a v2 auth URI with the secure scheme' do
         expect(chef_run).to render_file(file.name).with_content(

spec/spec_helper.rb

@@ -37,6 +37,12 @@ shared_context 'tempest-stubs' do
     allow_any_instance_of(Chef::Recipe).to receive(:get_password)
       .with('user', 'admin')
       .and_return('admin')
+    allow_any_instance_of(Chef::Recipe).to receive(:get_password)
+      .with('openstack_user', 'tempest_user1')
+      .and_return('tempest_user1_pass')
+    allow_any_instance_of(Chef::Recipe).to receive(:get_password)
+      .with('openstack_user', 'tempest_user2')
+      .and_return('tempest_user2_pass')
     allow_any_instance_of(Chef::Resource::RubyBlock).to receive(:openstack_command_env)
       .with('admin', 'admin', 'Default', 'Default')
       .and_return(env)

templates/default/tempest.sh.erb

@@ -1,2 +1,3 @@
 #!/bin/bash
-tempest run --smoke --serial --config-file /etc/tempest/tempest.conf
+source <%= @venv_path -%>/bin/activate
+tempest run --smoke --serial --config-file /opt/tempest/etc/tempest.conf