309 lines
12 KiB
Python
309 lines
12 KiB
Python
# Copyright 2015 OpenStack Foundation
|
|
# All Rights Reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
import os
|
|
import socket
|
|
import subprocess
|
|
import tempfile
|
|
|
|
from oslo_log import log as logging
|
|
from tempest.common import credentials_factory as credentials
|
|
from tempest import config
|
|
from tempest.lib import exceptions
|
|
from tempest import manager as tempestmanager
|
|
from tempest import test
|
|
from urllib3.exceptions import MaxRetryError
|
|
|
|
from congress_tempest_tests.services.policy import policy_client
|
|
from congress_tempest_tests.tests.scenario import helper
|
|
from congress_tempest_tests.tests.scenario import manager_congress
|
|
|
|
CONF = config.CONF
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
|
|
class TestHA(manager_congress.ScenarioPolicyBase):
|
|
|
|
def setUp(self):
|
|
super(TestHA, self).setUp()
|
|
self.keypairs = {}
|
|
self.servers = []
|
|
self.replicas = {}
|
|
self.services_client = self.admin_manager.identity_services_client
|
|
self.endpoints_client = self.admin_manager.endpoints_client
|
|
|
|
def _prepare_replica(self, port_num):
|
|
replica_url = "http://127.0.0.1:%d" % port_num
|
|
resp = self.services_client.create_service(
|
|
name='congressha',
|
|
type=CONF.congressha.replica_type,
|
|
description='policy ha service')
|
|
self.replica_service_id = resp['OS-KSADM:service']['id']
|
|
resp = self.endpoints_client.create_endpoint(
|
|
service_id=self.replica_service_id,
|
|
region=CONF.identity.region,
|
|
publicurl=replica_url,
|
|
adminurl=replica_url,
|
|
internalurl=replica_url)
|
|
self.replica_endpoint_id = resp['endpoint']['id']
|
|
|
|
def _cleanup_replica(self):
|
|
self.endpoints_client.delete_endpoint(self.replica_endpoint_id)
|
|
self.services_client.delete_service(self.replica_service_id)
|
|
|
|
def start_replica(self, port_num):
|
|
self._prepare_replica(port_num)
|
|
f = tempfile.NamedTemporaryFile(mode='w', suffix='.conf',
|
|
prefix='congress%d-' % port_num,
|
|
dir='/tmp', delete=False)
|
|
conf_file = f.name
|
|
template = open('/etc/congress/congress.conf')
|
|
conf = template.read()
|
|
|
|
# Add 'bind_port' and 'datasource_sync_period' to conf file.
|
|
index = conf.find('[DEFAULT]') + len('[DEFAULT]\n')
|
|
conf = (conf[:index] +
|
|
'bind_port = %d\n' % port_num +
|
|
'datasource_sync_period = 5\n' +
|
|
'bus_id = replica-node\n' +
|
|
conf[index:])
|
|
sindex = conf.find('signing_dir')
|
|
conf = conf[:sindex] + '#' + conf[sindex:]
|
|
LOG.debug("Configuration file for replica: %s\n", conf)
|
|
f.write(conf)
|
|
f.close()
|
|
|
|
args = ['/usr/bin/python',
|
|
'bin/congress-server',
|
|
'--config-file',
|
|
conf_file]
|
|
out = tempfile.NamedTemporaryFile(mode='w', suffix='.out',
|
|
prefix='congress%d-' % port_num,
|
|
dir='/tmp', delete=False)
|
|
err = tempfile.NamedTemporaryFile(mode='w', suffix='.err',
|
|
prefix='congress%d-' % port_num,
|
|
dir='/tmp', delete=False)
|
|
p = subprocess.Popen(args, stdout=out, stderr=err,
|
|
cwd=helper.root_path())
|
|
|
|
assert port_num not in self.replicas
|
|
self.replicas[port_num] = (p, conf_file)
|
|
|
|
def stop_replica(self, port_num):
|
|
proc, conf_file = self.replicas[port_num]
|
|
# Using proc.terminate() will block at proc.wait(), no idea why yet
|
|
proc.kill()
|
|
proc.wait()
|
|
os.unlink(conf_file)
|
|
self.replicas[port_num] = (None, conf_file)
|
|
self._cleanup_replica()
|
|
|
|
def create_client(self, client_type):
|
|
creds = credentials.get_configured_admin_credentials('identity_admin')
|
|
auth_prov = tempestmanager.get_auth_provider(creds)
|
|
|
|
return policy_client.PolicyClient(
|
|
auth_prov, client_type,
|
|
CONF.identity.region)
|
|
|
|
def datasource_exists(self, client, datasource_id):
|
|
try:
|
|
LOG.debug("datasource_exists begin")
|
|
body = client.list_datasource_status(datasource_id)
|
|
LOG.debug("list_datasource_status: %s", str(body))
|
|
except exceptions.NotFound:
|
|
LOG.debug("not found")
|
|
return False
|
|
except exceptions.Unauthorized:
|
|
LOG.debug("connection refused")
|
|
return False
|
|
except (socket.error, MaxRetryError):
|
|
LOG.debug("Replica server not ready")
|
|
return False
|
|
except Exception as e:
|
|
raise e
|
|
return True
|
|
|
|
def datasource_missing(self, client, datasource_id):
|
|
try:
|
|
LOG.debug("datasource_missing begin")
|
|
body = client.list_datasource_status(datasource_id)
|
|
LOG.debug("list_datasource_status: %s", str(body))
|
|
except exceptions.NotFound:
|
|
LOG.debug("not found")
|
|
return True
|
|
except exceptions.Unauthorized:
|
|
LOG.debug("connection refused")
|
|
return False
|
|
except (socket.error, MaxRetryError):
|
|
LOG.debug("Replica server not ready")
|
|
return False
|
|
except Exception as e:
|
|
raise e
|
|
return False
|
|
|
|
def find_fake(self, client):
|
|
datasources = client.list_datasources()
|
|
for r in datasources['results']:
|
|
if r['name'] == 'fake':
|
|
LOG.debug('existing fake driver: %s', str(r['id']))
|
|
return r['id']
|
|
return None
|
|
|
|
def create_fake(self, client):
|
|
# Create fake datasource if it does not exist. Returns the
|
|
# fake datasource id.
|
|
fake_id = self.find_fake(client)
|
|
if fake_id:
|
|
return fake_id
|
|
|
|
item = {'id': None,
|
|
'name': 'fake',
|
|
'driver': 'fake_datasource',
|
|
'config': '{"username":"fakeu", "tenant_name": "faket",' +
|
|
'"password": "fakep",' +
|
|
'"auth_url": "http://127.0.0.1:5000/v2"}',
|
|
'description': 'bar',
|
|
'enabled': True}
|
|
ret = client.create_datasource(item)
|
|
LOG.debug('created fake driver: %s', str(ret['id']))
|
|
return ret['id']
|
|
|
|
@test.attr(type='smoke')
|
|
def test_datasource_db_sync_add(self):
|
|
# Verify that a replica adds a datasource when a datasource
|
|
# appears in the database.
|
|
client1 = self.admin_manager.congress_client
|
|
|
|
# delete fake if it exists.
|
|
old_fake_id = self.find_fake(client1)
|
|
if old_fake_id:
|
|
client1.delete_datasource(old_fake_id)
|
|
|
|
# Verify that primary server has no fake datasource
|
|
if not test.call_until_true(
|
|
func=lambda: self.datasource_missing(client1, old_fake_id),
|
|
duration=60, sleep_for=1):
|
|
raise exceptions.TimeoutException(
|
|
"primary should not have fake, but does")
|
|
|
|
need_to_delete_fake = False
|
|
try:
|
|
# Create a new fake datasource
|
|
fake_id = self.create_fake(client1)
|
|
need_to_delete_fake = True
|
|
|
|
# Verify that primary server has fake datasource
|
|
if not test.call_until_true(
|
|
func=lambda: self.datasource_exists(client1, fake_id),
|
|
duration=60, sleep_for=1):
|
|
raise exceptions.TimeoutException(
|
|
"primary should have fake, but does not")
|
|
|
|
# start replica
|
|
self.start_replica(CONF.congressha.replica_port)
|
|
|
|
# Create session for second server.
|
|
client2 = self.create_client(CONF.congressha.replica_type)
|
|
|
|
# Verify that second server has fake datasource
|
|
if not test.call_until_true(
|
|
func=lambda: self.datasource_exists(client2, fake_id),
|
|
duration=60, sleep_for=1):
|
|
raise exceptions.TimeoutException(
|
|
"replica should have fake, but does not")
|
|
|
|
# Remove fake from primary server instance.
|
|
LOG.debug("removing fake datasource %s", str(fake_id))
|
|
client1.delete_datasource(fake_id)
|
|
need_to_delete_fake = False
|
|
|
|
# Confirm that fake is gone from primary server instance.
|
|
if not test.call_until_true(
|
|
func=lambda: self.datasource_missing(client1, fake_id),
|
|
duration=60, sleep_for=1):
|
|
self.stop_replica(CONF.congressha.replica_port)
|
|
raise exceptions.TimeoutException(
|
|
"primary instance still has fake")
|
|
LOG.debug("removed fake datasource from primary instance")
|
|
|
|
# Confirm that second service instance removes fake.
|
|
if not test.call_until_true(
|
|
func=lambda: self.datasource_missing(client2, fake_id),
|
|
duration=60, sleep_for=1):
|
|
raise exceptions.TimeoutException(
|
|
"replica should remove fake, but still has it")
|
|
|
|
finally:
|
|
self.stop_replica(CONF.congressha.replica_port)
|
|
if need_to_delete_fake:
|
|
self.admin_manager.congress_client.delete_datasource(fake_id)
|
|
|
|
# The following test is redundant because removal by db sync is already
|
|
# tested in test_datasource_db_sync_add above
|
|
# TODO(ekcs): decide whether to completely erase or update test
|
|
# @test.attr(type='smoke')
|
|
# def test_datasource_db_sync_remove(self):
|
|
# # Verify that a replica removes a datasource when a datasource
|
|
# # disappears from the database.
|
|
# client1 = self.admin_manager.congress_client
|
|
# fake_id = self.create_fake(client1)
|
|
# need_to_delete_fake = True
|
|
# try:
|
|
# self.start_replica(CONF.congressha.replica_port)
|
|
#
|
|
# # Verify that primary server has fake datasource
|
|
# if not test.call_until_true(
|
|
# func=lambda: self.datasource_exists(client1, fake_id),
|
|
# duration=60, sleep_for=1):
|
|
# raise exceptions.TimeoutException(
|
|
# "primary should have fake, but does not")
|
|
#
|
|
# # Create session for second server.
|
|
# client2 = self.create_client(CONF.congressha.replica_type)
|
|
#
|
|
# # Verify that second server has fake datasource
|
|
# if not test.call_until_true(
|
|
# func=lambda: self.datasource_exists(client2, fake_id),
|
|
# duration=60, sleep_for=1):
|
|
# raise exceptions.TimeoutException(
|
|
# "replica should have fake, but does not")
|
|
#
|
|
# # Remove fake from primary server instance.
|
|
# LOG.debug("removing fake datasource %s", str(fake_id))
|
|
# client1.delete_datasource(fake_id)
|
|
# need_to_delete_fake = False
|
|
#
|
|
# # Confirm that fake is gone from primary server instance.
|
|
# if not test.call_until_true(
|
|
# func=lambda: self.datasource_missing(client1, fake_id),
|
|
# duration=60, sleep_for=1):
|
|
# self.stop_replica(CONF.congressha.replica_port)
|
|
# raise exceptions.TimeoutException(
|
|
# "primary instance still has fake")
|
|
# LOG.debug("removed fake datasource from primary instance")
|
|
#
|
|
# # Confirm that second service instance removes fake.
|
|
# if not test.call_until_true(
|
|
# func=lambda: self.datasource_missing(client2, fake_id),
|
|
# duration=60, sleep_for=1):
|
|
# raise exceptions.TimeoutException(
|
|
# "replica should remove fake, but still has it")
|
|
#
|
|
# finally:
|
|
# self.stop_replica(CONF.congressha.replica_port)
|
|
# if need_to_delete_fake:
|
|
# self.admin_manager.congress_client.delete_datasource(fake_id)
|