456 lines
16 KiB
Python
Executable File
456 lines
16 KiB
Python
Executable File
#!/usr/bin/env python
|
|
|
|
import ConfigParser
|
|
import logging
|
|
import netifaces
|
|
import os
|
|
import re
|
|
from socket import inet_ntoa
|
|
from struct import pack
|
|
import subprocess
|
|
import sys
|
|
import yaml
|
|
|
|
|
|
ASTUTE_PATH = '/etc/astute.yaml'
|
|
ASTUTE_SECTION = 'fuel-plugin-xenserver'
|
|
LOG_ROOT = '/var/log/fuel-plugin-xenserver'
|
|
LOG_FILE = 'compute_post_deployment.log'
|
|
HIMN_IP = '169.254.0.1'
|
|
INT_BRIDGE = 'br-int'
|
|
XS_PLUGIN_ISO = 'xenserverplugins-liberty.iso'
|
|
DIST_PACKAGES_DIR = '/usr/lib/python2.7/dist-packages/'
|
|
|
|
if not os.path.exists(LOG_ROOT):
|
|
os.mkdir(LOG_ROOT)
|
|
|
|
logging.basicConfig(filename=os.path.join(LOG_ROOT, LOG_FILE),
|
|
level=logging.DEBUG)
|
|
|
|
|
|
def reportError(err):
|
|
logging.warning(err)
|
|
raise Exception(err)
|
|
|
|
|
|
def execute(*cmd, **kwargs):
|
|
cmd = map(str, cmd)
|
|
logging.info(' '.join(cmd))
|
|
proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE)
|
|
|
|
if 'prompt' in kwargs:
|
|
prompt = kwargs.get('prompt')
|
|
proc.stdout.flush()
|
|
(out, err) = proc.communicate(prompt)
|
|
else:
|
|
out = proc.stdout.readlines()
|
|
err = proc.stderr.readlines()
|
|
(out, err) = map(' '.join, [out, err])
|
|
|
|
# Both if/else need to deal with "\n" scenario
|
|
(out, err) = (out.replace('\n', ''), err.replace('\n', ''))
|
|
|
|
if out:
|
|
logging.debug(out)
|
|
|
|
if proc.returncode is not None and proc.returncode != 0:
|
|
reportError(err)
|
|
|
|
return out
|
|
|
|
|
|
def ssh(host, username, password, *cmd, **kwargs):
|
|
cmd = map(str, cmd)
|
|
|
|
return execute('sshpass', '-p', password, 'ssh',
|
|
'-o', 'StrictHostKeyChecking=no',
|
|
'%s@%s' % (username, host), *cmd,
|
|
prompt=kwargs.get('prompt'))
|
|
|
|
|
|
def scp(host, username, password, target_path, filename):
|
|
return execute('sshpass', '-p', password, 'scp',
|
|
'-o', 'StrictHostKeyChecking=no', filename,
|
|
'%s@%s:%s' % (username, host, target_path))
|
|
|
|
|
|
def get_astute(astute_path):
|
|
"""Return the root object read from astute.yaml"""
|
|
if not os.path.exists(astute_path):
|
|
reportError('%s not found' % astute_path)
|
|
|
|
astute = yaml.load(open(astute_path))
|
|
return astute
|
|
|
|
|
|
def astute_get(dct, keys, default=None, fail_if_missing=True):
|
|
"""A safe dictionary getter"""
|
|
for key in keys:
|
|
if key in dct:
|
|
dct = dct[key]
|
|
else:
|
|
if fail_if_missing:
|
|
reportError('Value of "%s" is missing' % key)
|
|
return default
|
|
return dct
|
|
|
|
|
|
def get_options(astute, astute_section):
|
|
"""Return username and password filled in plugin."""
|
|
if astute_section not in astute:
|
|
reportError('%s not found' % astute_section)
|
|
|
|
options = astute[astute_section]
|
|
logging.info('username: {username}'.format(**options))
|
|
logging.info('password: {password}'.format(**options))
|
|
logging.info('install_xapi: {install_xapi}'.format(**options))
|
|
return options['username'], options['password'], \
|
|
options['install_xapi']
|
|
|
|
|
|
def get_endpoints(astute):
|
|
"""Return the IP addresses of the endpoints connected to
|
|
storage/mgmt network.
|
|
"""
|
|
endpoints = astute['network_scheme']['endpoints']
|
|
endpoints = dict([(
|
|
k.replace('br-', ''),
|
|
endpoints[k]['IP'][0]
|
|
) for k in endpoints])
|
|
|
|
logging.info('storage network: {storage}'.format(**endpoints))
|
|
logging.info('mgmt network: {mgmt}'.format(**endpoints))
|
|
return endpoints
|
|
|
|
|
|
def init_eth():
|
|
"""Initialize the net interface connected to HIMN
|
|
|
|
Returns:
|
|
the IP addresses of local host and XenServer.
|
|
"""
|
|
|
|
domid = execute('xenstore-read', 'domid')
|
|
himn_mac = execute(
|
|
'xenstore-read',
|
|
'/local/domain/%s/vm-data/himn_mac' % domid)
|
|
logging.info('himn_mac: %s' % himn_mac)
|
|
|
|
_mac = lambda eth: \
|
|
netifaces.ifaddresses(eth).get(netifaces.AF_LINK)[0]['addr']
|
|
eths = [eth for eth in netifaces.interfaces() if _mac(eth) == himn_mac]
|
|
if len(eths) != 1:
|
|
reportError('Cannot find eth matches himn_mac')
|
|
|
|
eth = eths[0]
|
|
logging.info('himn_eth: %s' % eth)
|
|
|
|
ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)
|
|
|
|
if not ip:
|
|
execute('dhclient', eth)
|
|
fname = '/etc/network/interfaces.d/ifcfg-' + eth
|
|
s = ('auto {eth}\n'
|
|
'iface {eth} inet dhcp\n'
|
|
'post-up route del default dev {eth}').format(eth=eth)
|
|
with open(fname, 'w') as f:
|
|
f.write(s)
|
|
logging.info('%s created' % fname)
|
|
execute('ifdown', eth)
|
|
execute('ifup', eth)
|
|
ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)
|
|
|
|
if ip:
|
|
himn_local = ip[0]['addr']
|
|
himn_xs = '.'.join(himn_local.split('.')[:-1] + ['1'])
|
|
if HIMN_IP == himn_xs:
|
|
logging.info('himn_local: %s' % himn_local)
|
|
return eth, himn_local
|
|
|
|
reportError('HIMN failed to get IP address from XenServer')
|
|
|
|
|
|
def check_host_compatibility(himn, username, password):
|
|
hotfix = 'XS65ESP1013'
|
|
installed = ssh(himn, username, password,
|
|
'xe patch-list name-label=%s --minimal' % hotfix)
|
|
ver = ssh(himn, username, password,
|
|
('xe host-param-get uuid=$(xe host-list --minimal) '
|
|
'param-name=software-version param-key=product_version_text'))
|
|
|
|
if not installed and ver == "6.5":
|
|
reportError(('Hotfix %s has not been installed '
|
|
'and product version is %s') % (hotfix, ver))
|
|
|
|
|
|
def install_xenapi_sdk():
|
|
"""Install XenAPI Python SDK"""
|
|
execute('cp', 'XenAPI.py', DIST_PACKAGES_DIR)
|
|
|
|
|
|
def create_novacompute_conf(himn, username, password, public_ip, services_ssl):
|
|
"""Fill nova-compute.conf with HIMN IP and root password. """
|
|
mgmt_if = netifaces.ifaddresses('br-mgmt')
|
|
if mgmt_if and mgmt_if.get(netifaces.AF_INET) \
|
|
and mgmt_if.get(netifaces.AF_INET)[0]['addr']:
|
|
mgmt_ip = mgmt_if.get(netifaces.AF_INET)[0]['addr']
|
|
else:
|
|
reportError('Cannot get IP Address on Management Network')
|
|
|
|
filename = '/etc/nova/nova-compute.conf'
|
|
cf = ConfigParser.ConfigParser()
|
|
try:
|
|
cf.read(filename)
|
|
cf.set('DEFAULT', 'compute_driver', 'xenapi.XenAPIDriver')
|
|
cf.set('DEFAULT', 'force_config_drive', 'True')
|
|
scheme = "https" if services_ssl else "http"
|
|
cf.set('DEFAULT', 'novncproxy_base_url',
|
|
'%s://%s:6080/vnc_auto.html' % (scheme, public_ip))
|
|
cf.set('DEFAULT', 'vncserver_proxyclient_address', mgmt_ip)
|
|
if not cf.has_section('xenserver'):
|
|
cf.add_section('xenserver')
|
|
cf.set('xenserver', 'connection_url', 'http://%s' % himn)
|
|
cf.set('xenserver', 'connection_username', username)
|
|
cf.set('xenserver', 'connection_password', password)
|
|
cf.set('xenserver', 'vif_driver',
|
|
'nova.virt.xenapi.vif.XenAPIOpenVswitchDriver')
|
|
cf.set('xenserver', 'ovs_integration_bridge', INT_BRIDGE)
|
|
cf.write(open(filename, 'w'))
|
|
except Exception:
|
|
reportError('Cannot set configurations to %s' % filename)
|
|
logging.info('%s created' % filename)
|
|
|
|
|
|
def route_to_compute(endpoints, himn_xs, himn_local, username, password):
|
|
"""Route storage/mgmt requests to compute nodes. """
|
|
out = ssh(himn_xs, username, password, 'route', '-n')
|
|
_net = lambda ip: '.'.join(ip.split('.')[:-1] + ['0'])
|
|
_mask = lambda cidr: inet_ntoa(pack(
|
|
'>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1))
|
|
_routed = lambda net, mask, gw: re.search(r'%s\s+%s\s+%s\s+' % (
|
|
net.replace('.', r'\.'),
|
|
gw.replace('.', r'\.'),
|
|
mask
|
|
), out)
|
|
|
|
endpoint_names = ['storage', 'mgmt']
|
|
for endpoint_name in endpoint_names:
|
|
endpoint = endpoints.get(endpoint_name)
|
|
if endpoint:
|
|
ip, cidr = endpoint.split('/')
|
|
net, mask = _net(ip), _mask(cidr)
|
|
if not _routed(net, mask, himn_local):
|
|
params = ['route', 'add', '-net', net, 'netmask',
|
|
mask, 'gw', himn_local]
|
|
ssh(himn_xs, username, password, *params)
|
|
sh = 'echo \'%s\' >> /etc/sysconfig/static-routes' \
|
|
% ' '.join(params)
|
|
ssh(himn_xs, username, password, sh)
|
|
else:
|
|
logging.info('%s network ip is missing' % endpoint_name)
|
|
|
|
|
|
def install_suppack(himn, username, password):
|
|
"""Install xapi driver supplemental pack. """
|
|
# TODO(Johnhua): check if installed
|
|
scp(himn, username, password, '/tmp/', XS_PLUGIN_ISO)
|
|
ssh(
|
|
himn, username, password, 'xe-install-supplemental-pack',
|
|
'/tmp/%s' % XS_PLUGIN_ISO, prompt='Y\n')
|
|
ssh(himn, username, password, 'rm', '/tmp/%s' % XS_PLUGIN_ISO)
|
|
|
|
|
|
def forward_from_himn(eth):
|
|
"""Forward packets from HIMN to storage/mgmt network. """
|
|
execute('sed', '-i', 's/#net.ipv4.ip_forward/net.ipv4.ip_forward/g',
|
|
'/etc/sysctl.conf')
|
|
execute('sysctl', '-p', '/etc/sysctl.conf')
|
|
|
|
endpoint_names = ['br-storage', 'br-mgmt']
|
|
for endpoint_name in endpoint_names:
|
|
execute('iptables', '-t', 'nat', '-A', 'POSTROUTING',
|
|
'-o', endpoint_name, '-j', 'MASQUERADE')
|
|
execute('iptables', '-A', 'FORWARD',
|
|
'-i', endpoint_name, '-o', eth,
|
|
'-m', 'state', '--state', 'RELATED,ESTABLISHED',
|
|
'-j', 'ACCEPT')
|
|
execute('iptables', '-A', 'FORWARD',
|
|
'-i', eth, '-o', endpoint_name,
|
|
'-j', 'ACCEPT')
|
|
|
|
execute('iptables', '-A', 'INPUT', '-i', eth, '-j', 'ACCEPT')
|
|
execute('iptables', '-t', 'filter', '-S', 'FORWARD')
|
|
execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
|
|
execute('service', 'iptables-persistent', 'save')
|
|
|
|
|
|
def forward_port(eth_in, eth_out, target_host, target_port):
|
|
"""Forward packets from eth_in to eth_out on target_host:target_port. """
|
|
execute('iptables', '-t', 'nat', '-A', 'PREROUTING',
|
|
'-i', eth_in, '-p', 'tcp', '--dport', target_port,
|
|
'-j', 'DNAT', '--to', target_host)
|
|
execute('iptables', '-A', 'FORWARD',
|
|
'-i', eth_out, '-o', eth_in,
|
|
'-m', 'state', '--state', 'RELATED,ESTABLISHED',
|
|
'-j', 'ACCEPT')
|
|
execute('iptables', '-A', 'FORWARD',
|
|
'-i', eth_in, '-o', eth_out,
|
|
'-j', 'ACCEPT')
|
|
|
|
execute('iptables', '-t', 'filter', '-S', 'FORWARD')
|
|
execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
|
|
execute('service', 'iptables-persistent', 'save')
|
|
|
|
|
|
def install_logrotate_script(himn, username, password):
|
|
"Install console logrotate script"
|
|
scp(himn, username, password, '/root/', 'rotate_xen_guest_logs.sh')
|
|
ssh(himn, username, password, 'mkdir -p /var/log/xen/guest')
|
|
ssh(himn, username, password, '''crontab - << CRONTAB
|
|
* * * * * /root/rotate_xen_guest_logs.sh
|
|
CRONTAB''')
|
|
|
|
|
|
def modify_neutron_rootwrap_conf(himn, username, password):
|
|
"""Set xenapi configurations"""
|
|
filename = '/etc/neutron/rootwrap.conf'
|
|
cf = ConfigParser.ConfigParser()
|
|
try:
|
|
cf.read(filename)
|
|
cf.set('xenapi', 'xenapi_connection_url', 'http://%s' % himn)
|
|
cf.set('xenapi', 'xenapi_connection_username', username)
|
|
cf.set('xenapi', 'xenapi_connection_password', password)
|
|
cf.write(open(filename, 'w'))
|
|
except Exception:
|
|
reportError("Fail to modify file %s", filename)
|
|
logging.info('Modify file %s successfully', filename)
|
|
|
|
|
|
def modify_neutron_ovs_agent_conf(int_br, br_mappings):
|
|
filename = '/etc/neutron/plugins/ml2/ml2_conf.ini'
|
|
cf = ConfigParser.ConfigParser()
|
|
try:
|
|
cf.read(filename)
|
|
cf.set('agent', 'root_helper',
|
|
'neutron-rootwrap-xen-dom0 /etc/neutron/rootwrap.conf')
|
|
cf.set('agent', 'root_helper_daemon', '')
|
|
cf.set('agent', 'minimize_polling', False)
|
|
cf.set('ovs', 'integration_bridge', int_br)
|
|
cf.set('ovs', 'bridge_mappings', br_mappings)
|
|
cf.write(open(filename, 'w'))
|
|
except Exception:
|
|
reportError("Fail to modify %s", filename)
|
|
logging.info('Modify %s successfully', filename)
|
|
|
|
|
|
def get_private_network_ethX():
|
|
# find out bridge which is used for private network
|
|
values = astute['network_scheme']['transformations']
|
|
for item in values:
|
|
if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
|
|
return item['name']
|
|
|
|
|
|
def find_bridge_mappings(astute, himn, username, password):
|
|
ethX = get_private_network_ethX()
|
|
if not ethX:
|
|
reportError("Cannot find eth used for private network")
|
|
|
|
# find the ethX mac in /sys/class/net/ethX/address
|
|
fo = open('/sys/class/net/%s/address' % ethX, 'r')
|
|
mac = fo.readline()
|
|
fo.close()
|
|
network_uuid = ssh(himn, username, password,
|
|
'xe vif-list params=network-uuid minimal=true MAC=%s' % mac)
|
|
bridge = ssh(himn, username, password,
|
|
'xe network-param-get param-name=bridge uuid=%s' % network_uuid)
|
|
|
|
# find physical network name
|
|
phynet_setting = astute['quantum_settings']['L2']['phys_nets']
|
|
physnet = phynet_setting.keys()[0]
|
|
return physnet + ':' + bridge
|
|
|
|
|
|
def restart_services(service_name):
|
|
execute('stop', service_name)
|
|
execute('start', service_name)
|
|
|
|
|
|
def enable_linux_bridge(himn, username, password):
|
|
# When using OVS under XS6.5, it will prevent use of Linux bridge in
|
|
# Dom0, but neutron-openvswitch-agent in compute node will use Linux
|
|
# bridge, so we remove this restriction here
|
|
ssh(himn, username, password, 'rm -f /etc/modprobe.d/blacklist-bridge*')
|
|
|
|
|
|
def patch_compute_xenapi():
|
|
"""replace folder xenapi to add patches which are not merged to upstream"""
|
|
# TODO(huanxie): need to confirm the overall patchset list
|
|
patchset_dir = sys.path[0]
|
|
patchfile_list = ['%s/patchset/vif-plug.patch' % patchset_dir,
|
|
'%s/patchset/nova-neutron-race-condition.patch' % patchset_dir,
|
|
'%s/patchset/ovs-interim-bridge.patch' % patchset_dir,
|
|
'%s/patchset/neutron-security-group.patch' % patchset_dir,
|
|
'%s/patchset/speed-up-writing-config-drive.patch' % patchset_dir]
|
|
for patch_file in patchfile_list:
|
|
execute('patch', '-d', DIST_PACKAGES_DIR, '-p1', '-i', patch_file)
|
|
|
|
|
|
def patch_neutron_ovs_agent():
|
|
patchset_dir = sys.path[0]
|
|
patch_file = '%s/patchset/neutron-rootwrap-xen-dom0.patch' % patchset_dir
|
|
execute('patch', '-d', '/usr/', '-p1', '-i', patch_file)
|
|
|
|
|
|
def apply_sm_patch(himn, username, password):
|
|
ver = ssh(himn, username, password,
|
|
('xe host-param-get uuid=$(xe host-list --minimal) '
|
|
'param-name=software-version param-key=product_version_text'))
|
|
if ver == "6.5":
|
|
ssh(himn, username, password, "sed -i s/'phy'/'aio'/g /opt/xensource/sm/ISCSISR.py")
|
|
|
|
|
|
if __name__ == '__main__':
|
|
install_xenapi_sdk()
|
|
astute = get_astute(ASTUTE_PATH)
|
|
if astute:
|
|
username, password, install_xapi = get_options(astute, ASTUTE_SECTION)
|
|
endpoints = get_endpoints(astute)
|
|
himn_eth, himn_local = init_eth()
|
|
|
|
public_ip = astute_get(
|
|
astute, ('network_metadata', 'vips', 'public', 'ipaddr'))
|
|
|
|
services_ssl = astute_get(
|
|
astute, ('public_ssl', 'services'))
|
|
|
|
if username and password and endpoints and himn_local:
|
|
check_host_compatibility(HIMN_IP, username, password)
|
|
route_to_compute(
|
|
endpoints, HIMN_IP, himn_local, username, password)
|
|
if install_xapi:
|
|
install_suppack(HIMN_IP, username, password)
|
|
enable_linux_bridge(HIMN_IP, username, password)
|
|
forward_from_himn(himn_eth)
|
|
|
|
# port forwarding for novnc
|
|
forward_port('br-mgmt', himn_eth, HIMN_IP, '80')
|
|
|
|
# apply sm patch
|
|
apply_sm_patch(HIMN_IP, username, password)
|
|
|
|
create_novacompute_conf(HIMN_IP, username, password, public_ip, services_ssl)
|
|
patch_compute_xenapi()
|
|
restart_services('nova-compute')
|
|
|
|
install_logrotate_script(HIMN_IP, username, password)
|
|
|
|
# neutron-l2-agent in compute node
|
|
modify_neutron_rootwrap_conf(HIMN_IP, username, password)
|
|
br_mappings = find_bridge_mappings(astute, HIMN_IP,
|
|
username, password)
|
|
modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
|
|
patch_neutron_ovs_agent()
|
|
restart_services('neutron-plugin-openvswitch-agent')
|