Adds better support for service leaders.
* The service leader is determined depending on how keystone is currently clustered. If there are multiple units, but no hacluster subordinate, the oldest service unit is elected leader (lowest unit number). If hacluster exists and the service is clustered, the CRM is consulted and the node hosting the resources is designated the leader. * Only the leader may initialize or touch the database (create users, endpoints, etc) * The leader is responsible for synchronizing a list of service credentials to all peers. The list is stored on disk and resolves the issue of the passwd dump files in /var/lib/keystone/ being out-of-sync among peers. We can use the same approach in the rabbitmq-server charm if it works out here.
This commit is contained in:
James Page
commit
632c1369af
@ -77,12 +77,14 @@ def install_hook():
|
||||
driver='keystone.token.backends.sql.Token')
|
||||
update_config_block('ec2',
|
||||
driver='keystone.contrib.ec2.backends.sql.Ec2')
|
||||
|
||||
execute("service keystone stop", echo=True)
|
||||
execute("keystone-manage db_sync")
|
||||
execute("service keystone start", echo=True)
|
||||
time.sleep(5)
|
||||
ensure_initial_admin(config)
|
||||
|
||||
|
||||
def db_joined():
|
||||
relation_data = { "database": config["database"],
|
||||
"username": config["database-user"],
|
||||
@ -100,7 +102,14 @@ def db_changed():
|
||||
relation_data["password"],
|
||||
relation_data["private-address"],
|
||||
config["database"]))
|
||||
|
||||
execute("service keystone stop", echo=True)
|
||||
|
||||
if not eligible_leader():
|
||||
juju_log('Deferring DB initialization to service leader.')
|
||||
execute("service keystone start")
|
||||
return
|
||||
|
||||
execute("keystone-manage db_sync", echo=True)
|
||||
execute("service keystone start")
|
||||
time.sleep(5)
|
||||
@ -141,8 +150,8 @@ def identity_changed(relation_id=None, remote_unit=None):
|
||||
adminurl=adminurl,
|
||||
internalurl=internalurl)
|
||||
|
||||
if is_clustered() and not is_leader():
|
||||
# Only respond if service unit is the leader
|
||||
if not eligible_leader():
|
||||
juju_log('Deferring identity_changed() to service leader.')
|
||||
return
|
||||
|
||||
settings = relation_get_dict(relation_id=relation_id,
|
||||
@ -221,16 +230,7 @@ def identity_changed(relation_id=None, remote_unit=None):
|
||||
token = get_admin_token()
|
||||
juju_log("Creating service credentials for '%s'" % service_username)
|
||||
|
||||
# TODO: This needs to be changed as it won't work with ha keystone
|
||||
stored_passwd = '/var/lib/keystone/%s.passwd' % service_username
|
||||
if os.path.isfile(stored_passwd):
|
||||
juju_log("Loading stored service passwd from %s" % stored_passwd)
|
||||
service_password = open(stored_passwd, 'r').readline().strip('\n')
|
||||
else:
|
||||
juju_log("Generating a new service password for %s" % service_username)
|
||||
service_password = execute('pwgen -c 32 1', die=True)[0].strip()
|
||||
open(stored_passwd, 'w+').writelines("%s\n" % service_password)
|
||||
|
||||
service_password = get_service_password(service_username)
|
||||
create_user(service_username, service_password, config['service-tenant'])
|
||||
grant_role(service_username, config['admin-role'], config['service-tenant'])
|
||||
|
||||
@ -258,6 +258,7 @@ def identity_changed(relation_id=None, remote_unit=None):
|
||||
relation_data["service_port"] = SERVICE_PORTS['keystone_service']
|
||||
|
||||
relation_set(relation_data)
|
||||
synchronize_service_credentials()
|
||||
|
||||
def config_changed():
|
||||
|
||||
@ -272,11 +273,9 @@ def config_changed():
|
||||
|
||||
set_admin_token(config['admin-token'])
|
||||
|
||||
if is_clustered() and is_leader():
|
||||
if eligible_leader():
|
||||
juju_log('Cluster leader - ensuring endpoint configuration is up to date')
|
||||
ensure_initial_admin(config)
|
||||
elif not is_clustered():
|
||||
ensure_initial_admin(config)
|
||||
|
||||
update_config_block('logger_root', level=config['log-level'],
|
||||
file='/etc/keystone/logging.conf')
|
||||
@ -290,11 +289,9 @@ def config_changed():
|
||||
|
||||
def upgrade_charm():
|
||||
cluster_changed()
|
||||
if is_clustered() and is_leader():
|
||||
if eligible_leader():
|
||||
juju_log('Cluster leader - ensuring endpoint configuration is up to date')
|
||||
ensure_initial_admin(config)
|
||||
elif not is_clustered():
|
||||
ensure_initial_admin(config)
|
||||
|
||||
|
||||
SERVICE_PORTS = {
|
||||
@ -314,6 +311,8 @@ def cluster_changed():
|
||||
configure_haproxy(cluster_hosts,
|
||||
SERVICE_PORTS)
|
||||
|
||||
synchronize_service_credentials()
|
||||
|
||||
|
||||
def ha_relation_changed():
|
||||
relation_data = relation_get_dict()
|
||||
@ -391,7 +390,7 @@ hooks = {
|
||||
|
||||
# keystone-hooks gets called by symlink corresponding to the requested relation
|
||||
# hook.
|
||||
arg0 = sys.argv[0].split("/").pop()
|
||||
if arg0 not in hooks.keys():
|
||||
error_out("Unsupported hook: %s" % arg0)
|
||||
hooks[arg0]()
|
||||
hook = os.path.basename(sys.argv[0])
|
||||
if hook not in hooks.keys():
|
||||
error_out("Unsupported hook: %s" % hook)
|
||||
hooks[hook]()
|
||||
|
||||
@ -11,10 +11,10 @@ from lib.openstack_common import *
|
||||
keystone_conf = "/etc/keystone/keystone.conf"
|
||||
stored_passwd = "/var/lib/keystone/keystone.passwd"
|
||||
stored_token = "/var/lib/keystone/keystone.token"
|
||||
|
||||
SERVICE_PASSWD_PATH = '/var/lib/keystone/services.passwd'
|
||||
|
||||
def execute(cmd, die=False, echo=False):
|
||||
""" Executes a command
|
||||
""" Executes a command
|
||||
|
||||
if die=True, script will exit(1) if command does not return 0
|
||||
if echo=True, output of command will be printed to stdout
|
||||
@ -398,6 +398,31 @@ def update_user_password(username, password):
|
||||
manager.api.users.update_password(user=user_id, password=password)
|
||||
juju_log("Successfully updated password for user '%s'" % username)
|
||||
|
||||
def load_stored_passwords(path=SERVICE_PASSWD_PATH):
|
||||
creds = {}
|
||||
if not os.path.isfile(path):
|
||||
return creds
|
||||
|
||||
stored_passwd = open(path, 'r')
|
||||
for l in stored_passwd.readlines():
|
||||
user, passwd = l.strip().split(':')
|
||||
creds[user] = passwd
|
||||
return creds
|
||||
|
||||
def save_stored_passwords(path=SERVICE_PASSWD_PATH, **creds):
|
||||
with open(path, 'wb') as stored_passwd:
|
||||
[stored_passwd.write('%s:%s\n' % (u, p)) for u, p in creds.iteritems()]
|
||||
|
||||
def get_service_password(service_username):
|
||||
creds = load_stored_passwords()
|
||||
if service_username in creds:
|
||||
return creds[service_username]
|
||||
|
||||
passwd = subprocess.check_output(['pwgen', '-c', '32', '1']).strip()
|
||||
creds[service_username] = passwd
|
||||
save_stored_passwords(**creds)
|
||||
|
||||
return passwd
|
||||
|
||||
def configure_pki_tokens(config):
|
||||
'''Configure PKI token signing, if enabled.'''
|
||||
@ -482,3 +507,53 @@ def is_leader():
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
|
||||
def peer_units():
|
||||
peers = []
|
||||
for r_id in (relation_ids('cluster') or []):
|
||||
for unit in (relation_list(r_id) or []):
|
||||
peers.append(unit)
|
||||
return peers
|
||||
|
||||
def oldest_peer(peers):
|
||||
local_unit_no = os.getenv('JUJU_UNIT_NAME').split('/')[1]
|
||||
for peer in peers:
|
||||
remote_unit_no = peer.split('/')[1]
|
||||
if remote_unit_no < local_unit_no:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def eligible_leader():
|
||||
if is_clustered():
|
||||
if not is_leader():
|
||||
juju_log('Deferring action to CRM leader.')
|
||||
return False
|
||||
else:
|
||||
peers = peer_units()
|
||||
if peers and not oldest_peer(peers):
|
||||
juju_log('Deferring action to oldest service unit.')
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def synchronize_service_credentials():
|
||||
'''
|
||||
Broadcast service credentials to peers or consume those that have been
|
||||
broadcasted by peer, depending on hook context.
|
||||
'''
|
||||
if os.path.basename(sys.argv[0]) == 'cluster-relation-changed':
|
||||
r_data = relation_get_dict()
|
||||
if 'service_credentials' in r_data:
|
||||
juju_log('Saving service passwords from peer.')
|
||||
save_stored_passwords(**json.loads(r_data['service_credentials']))
|
||||
return
|
||||
|
||||
creds = load_stored_passwords()
|
||||
if not creds:
|
||||
return
|
||||
juju_log('Synchronizing service passwords to all peers.')
|
||||
creds = json.dumps(creds)
|
||||
for r_id in (relation_ids('cluster') or []):
|
||||
relation_set_2(rid=r_id, service_credentials=creds)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user