Fuel plugin for XenServer integration
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.

compute_post_deployment.py 16KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456
  1. #!/usr/bin/env python
  2. import ConfigParser
  3. import logging
  4. import netifaces
  5. import os
  6. import re
  7. from socket import inet_ntoa
  8. from struct import pack
  9. import subprocess
  10. import sys
  11. import yaml
  12. ASTUTE_PATH = '/etc/astute.yaml'
  13. ASTUTE_SECTION = 'fuel-plugin-xenserver'
  14. LOG_ROOT = '/var/log/fuel-plugin-xenserver'
  15. LOG_FILE = 'compute_post_deployment.log'
  16. HIMN_IP = '169.254.0.1'
  17. INT_BRIDGE = 'br-int'
  18. XS_PLUGIN_ISO = 'xenserverplugins-liberty.iso'
  19. DIST_PACKAGES_DIR = '/usr/lib/python2.7/dist-packages/'
  20. if not os.path.exists(LOG_ROOT):
  21. os.mkdir(LOG_ROOT)
  22. logging.basicConfig(filename=os.path.join(LOG_ROOT, LOG_FILE),
  23. level=logging.DEBUG)
  24. def reportError(err):
  25. logging.warning(err)
  26. raise Exception(err)
  27. def execute(*cmd, **kwargs):
  28. cmd = map(str, cmd)
  29. logging.info(' '.join(cmd))
  30. proc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
  31. stderr=subprocess.PIPE)
  32. if 'prompt' in kwargs:
  33. prompt = kwargs.get('prompt')
  34. proc.stdout.flush()
  35. (out, err) = proc.communicate(prompt)
  36. else:
  37. out = proc.stdout.readlines()
  38. err = proc.stderr.readlines()
  39. (out, err) = map(' '.join, [out, err])
  40. # Both if/else need to deal with "\n" scenario
  41. (out, err) = (out.replace('\n', ''), err.replace('\n', ''))
  42. if out:
  43. logging.debug(out)
  44. if proc.returncode is not None and proc.returncode != 0:
  45. reportError(err)
  46. return out
  47. def ssh(host, username, password, *cmd, **kwargs):
  48. cmd = map(str, cmd)
  49. return execute('sshpass', '-p', password, 'ssh',
  50. '-o', 'StrictHostKeyChecking=no',
  51. '%s@%s' % (username, host), *cmd,
  52. prompt=kwargs.get('prompt'))
  53. def scp(host, username, password, target_path, filename):
  54. return execute('sshpass', '-p', password, 'scp',
  55. '-o', 'StrictHostKeyChecking=no', filename,
  56. '%s@%s:%s' % (username, host, target_path))
  57. def get_astute(astute_path):
  58. """Return the root object read from astute.yaml"""
  59. if not os.path.exists(astute_path):
  60. reportError('%s not found' % astute_path)
  61. astute = yaml.load(open(astute_path))
  62. return astute
  63. def astute_get(dct, keys, default=None, fail_if_missing=True):
  64. """A safe dictionary getter"""
  65. for key in keys:
  66. if key in dct:
  67. dct = dct[key]
  68. else:
  69. if fail_if_missing:
  70. reportError('Value of "%s" is missing' % key)
  71. return default
  72. return dct
  73. def get_options(astute, astute_section):
  74. """Return username and password filled in plugin."""
  75. if astute_section not in astute:
  76. reportError('%s not found' % astute_section)
  77. options = astute[astute_section]
  78. logging.info('username: {username}'.format(**options))
  79. logging.info('password: {password}'.format(**options))
  80. logging.info('install_xapi: {install_xapi}'.format(**options))
  81. return options['username'], options['password'], \
  82. options['install_xapi']
  83. def get_endpoints(astute):
  84. """Return the IP addresses of the endpoints connected to
  85. storage/mgmt network.
  86. """
  87. endpoints = astute['network_scheme']['endpoints']
  88. endpoints = dict([(
  89. k.replace('br-', ''),
  90. endpoints[k]['IP'][0]
  91. ) for k in endpoints])
  92. logging.info('storage network: {storage}'.format(**endpoints))
  93. logging.info('mgmt network: {mgmt}'.format(**endpoints))
  94. return endpoints
  95. def init_eth():
  96. """Initialize the net interface connected to HIMN
  97. Returns:
  98. the IP addresses of local host and XenServer.
  99. """
  100. domid = execute('xenstore-read', 'domid')
  101. himn_mac = execute(
  102. 'xenstore-read',
  103. '/local/domain/%s/vm-data/himn_mac' % domid)
  104. logging.info('himn_mac: %s' % himn_mac)
  105. _mac = lambda eth: \
  106. netifaces.ifaddresses(eth).get(netifaces.AF_LINK)[0]['addr']
  107. eths = [eth for eth in netifaces.interfaces() if _mac(eth) == himn_mac]
  108. if len(eths) != 1:
  109. reportError('Cannot find eth matches himn_mac')
  110. eth = eths[0]
  111. logging.info('himn_eth: %s' % eth)
  112. ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)
  113. if not ip:
  114. execute('dhclient', eth)
  115. fname = '/etc/network/interfaces.d/ifcfg-' + eth
  116. s = ('auto {eth}\n'
  117. 'iface {eth} inet dhcp\n'
  118. 'post-up route del default dev {eth}').format(eth=eth)
  119. with open(fname, 'w') as f:
  120. f.write(s)
  121. logging.info('%s created' % fname)
  122. execute('ifdown', eth)
  123. execute('ifup', eth)
  124. ip = netifaces.ifaddresses(eth).get(netifaces.AF_INET)
  125. if ip:
  126. himn_local = ip[0]['addr']
  127. himn_xs = '.'.join(himn_local.split('.')[:-1] + ['1'])
  128. if HIMN_IP == himn_xs:
  129. logging.info('himn_local: %s' % himn_local)
  130. return eth, himn_local
  131. reportError('HIMN failed to get IP address from XenServer')
  132. def check_host_compatibility(himn, username, password):
  133. hotfix = 'XS65ESP1013'
  134. installed = ssh(himn, username, password,
  135. 'xe patch-list name-label=%s --minimal' % hotfix)
  136. ver = ssh(himn, username, password,
  137. ('xe host-param-get uuid=$(xe host-list --minimal) '
  138. 'param-name=software-version param-key=product_version_text'))
  139. if not installed and ver == "6.5":
  140. reportError(('Hotfix %s has not been installed '
  141. 'and product version is %s') % (hotfix, ver))
  142. def install_xenapi_sdk():
  143. """Install XenAPI Python SDK"""
  144. execute('cp', 'XenAPI.py', DIST_PACKAGES_DIR)
  145. def create_novacompute_conf(himn, username, password, public_ip, services_ssl):
  146. """Fill nova-compute.conf with HIMN IP and root password. """
  147. mgmt_if = netifaces.ifaddresses('br-mgmt')
  148. if mgmt_if and mgmt_if.get(netifaces.AF_INET) \
  149. and mgmt_if.get(netifaces.AF_INET)[0]['addr']:
  150. mgmt_ip = mgmt_if.get(netifaces.AF_INET)[0]['addr']
  151. else:
  152. reportError('Cannot get IP Address on Management Network')
  153. filename = '/etc/nova/nova-compute.conf'
  154. cf = ConfigParser.ConfigParser()
  155. try:
  156. cf.read(filename)
  157. cf.set('DEFAULT', 'compute_driver', 'xenapi.XenAPIDriver')
  158. cf.set('DEFAULT', 'force_config_drive', 'True')
  159. scheme = "https" if services_ssl else "http"
  160. cf.set('DEFAULT', 'novncproxy_base_url',
  161. '%s://%s:6080/vnc_auto.html' % (scheme, public_ip))
  162. cf.set('DEFAULT', 'vncserver_proxyclient_address', mgmt_ip)
  163. if not cf.has_section('xenserver'):
  164. cf.add_section('xenserver')
  165. cf.set('xenserver', 'connection_url', 'http://%s' % himn)
  166. cf.set('xenserver', 'connection_username', username)
  167. cf.set('xenserver', 'connection_password', password)
  168. cf.set('xenserver', 'vif_driver',
  169. 'nova.virt.xenapi.vif.XenAPIOpenVswitchDriver')
  170. cf.set('xenserver', 'ovs_integration_bridge', INT_BRIDGE)
  171. cf.write(open(filename, 'w'))
  172. except Exception:
  173. reportError('Cannot set configurations to %s' % filename)
  174. logging.info('%s created' % filename)
  175. def route_to_compute(endpoints, himn_xs, himn_local, username, password):
  176. """Route storage/mgmt requests to compute nodes. """
  177. out = ssh(himn_xs, username, password, 'route', '-n')
  178. _net = lambda ip: '.'.join(ip.split('.')[:-1] + ['0'])
  179. _mask = lambda cidr: inet_ntoa(pack(
  180. '>I', 0xffffffff ^ (1 << 32 - int(cidr)) - 1))
  181. _routed = lambda net, mask, gw: re.search(r'%s\s+%s\s+%s\s+' % (
  182. net.replace('.', r'\.'),
  183. gw.replace('.', r'\.'),
  184. mask
  185. ), out)
  186. endpoint_names = ['storage', 'mgmt']
  187. for endpoint_name in endpoint_names:
  188. endpoint = endpoints.get(endpoint_name)
  189. if endpoint:
  190. ip, cidr = endpoint.split('/')
  191. net, mask = _net(ip), _mask(cidr)
  192. if not _routed(net, mask, himn_local):
  193. params = ['route', 'add', '-net', net, 'netmask',
  194. mask, 'gw', himn_local]
  195. ssh(himn_xs, username, password, *params)
  196. sh = 'echo \'%s\' >> /etc/sysconfig/static-routes' \
  197. % ' '.join(params)
  198. ssh(himn_xs, username, password, sh)
  199. else:
  200. logging.info('%s network ip is missing' % endpoint_name)
  201. def install_suppack(himn, username, password):
  202. """Install xapi driver supplemental pack. """
  203. # TODO(Johnhua): check if installed
  204. scp(himn, username, password, '/tmp/', XS_PLUGIN_ISO)
  205. ssh(
  206. himn, username, password, 'xe-install-supplemental-pack',
  207. '/tmp/%s' % XS_PLUGIN_ISO, prompt='Y\n')
  208. ssh(himn, username, password, 'rm', '/tmp/%s' % XS_PLUGIN_ISO)
  209. def forward_from_himn(eth):
  210. """Forward packets from HIMN to storage/mgmt network. """
  211. execute('sed', '-i', 's/#net.ipv4.ip_forward/net.ipv4.ip_forward/g',
  212. '/etc/sysctl.conf')
  213. execute('sysctl', '-p', '/etc/sysctl.conf')
  214. endpoint_names = ['br-storage', 'br-mgmt']
  215. for endpoint_name in endpoint_names:
  216. execute('iptables', '-t', 'nat', '-A', 'POSTROUTING',
  217. '-o', endpoint_name, '-j', 'MASQUERADE')
  218. execute('iptables', '-A', 'FORWARD',
  219. '-i', endpoint_name, '-o', eth,
  220. '-m', 'state', '--state', 'RELATED,ESTABLISHED',
  221. '-j', 'ACCEPT')
  222. execute('iptables', '-A', 'FORWARD',
  223. '-i', eth, '-o', endpoint_name,
  224. '-j', 'ACCEPT')
  225. execute('iptables', '-A', 'INPUT', '-i', eth, '-j', 'ACCEPT')
  226. execute('iptables', '-t', 'filter', '-S', 'FORWARD')
  227. execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
  228. execute('service', 'iptables-persistent', 'save')
  229. def forward_port(eth_in, eth_out, target_host, target_port):
  230. """Forward packets from eth_in to eth_out on target_host:target_port. """
  231. execute('iptables', '-t', 'nat', '-A', 'PREROUTING',
  232. '-i', eth_in, '-p', 'tcp', '--dport', target_port,
  233. '-j', 'DNAT', '--to', target_host)
  234. execute('iptables', '-A', 'FORWARD',
  235. '-i', eth_out, '-o', eth_in,
  236. '-m', 'state', '--state', 'RELATED,ESTABLISHED',
  237. '-j', 'ACCEPT')
  238. execute('iptables', '-A', 'FORWARD',
  239. '-i', eth_in, '-o', eth_out,
  240. '-j', 'ACCEPT')
  241. execute('iptables', '-t', 'filter', '-S', 'FORWARD')
  242. execute('iptables', '-t', 'nat', '-S', 'POSTROUTING')
  243. execute('service', 'iptables-persistent', 'save')
  244. def install_logrotate_script(himn, username, password):
  245. "Install console logrotate script"
  246. scp(himn, username, password, '/root/', 'rotate_xen_guest_logs.sh')
  247. ssh(himn, username, password, 'mkdir -p /var/log/xen/guest')
  248. ssh(himn, username, password, '''crontab - << CRONTAB
  249. * * * * * /root/rotate_xen_guest_logs.sh
  250. CRONTAB''')
  251. def modify_neutron_rootwrap_conf(himn, username, password):
  252. """Set xenapi configurations"""
  253. filename = '/etc/neutron/rootwrap.conf'
  254. cf = ConfigParser.ConfigParser()
  255. try:
  256. cf.read(filename)
  257. cf.set('xenapi', 'xenapi_connection_url', 'http://%s' % himn)
  258. cf.set('xenapi', 'xenapi_connection_username', username)
  259. cf.set('xenapi', 'xenapi_connection_password', password)
  260. cf.write(open(filename, 'w'))
  261. except Exception:
  262. reportError("Fail to modify file %s", filename)
  263. logging.info('Modify file %s successfully', filename)
  264. def modify_neutron_ovs_agent_conf(int_br, br_mappings):
  265. filename = '/etc/neutron/plugins/ml2/ml2_conf.ini'
  266. cf = ConfigParser.ConfigParser()
  267. try:
  268. cf.read(filename)
  269. cf.set('agent', 'root_helper',
  270. 'neutron-rootwrap-xen-dom0 /etc/neutron/rootwrap.conf')
  271. cf.set('agent', 'root_helper_daemon', '')
  272. cf.set('agent', 'minimize_polling', False)
  273. cf.set('ovs', 'integration_bridge', int_br)
  274. cf.set('ovs', 'bridge_mappings', br_mappings)
  275. cf.write(open(filename, 'w'))
  276. except Exception:
  277. reportError("Fail to modify %s", filename)
  278. logging.info('Modify %s successfully', filename)
  279. def get_private_network_ethX():
  280. # find out bridge which is used for private network
  281. values = astute['network_scheme']['transformations']
  282. for item in values:
  283. if item['action'] == 'add-port' and item['bridge'] == 'br-aux':
  284. return item['name']
  285. def find_bridge_mappings(astute, himn, username, password):
  286. ethX = get_private_network_ethX()
  287. if not ethX:
  288. reportError("Cannot find eth used for private network")
  289. # find the ethX mac in /sys/class/net/ethX/address
  290. fo = open('/sys/class/net/%s/address' % ethX, 'r')
  291. mac = fo.readline()
  292. fo.close()
  293. network_uuid = ssh(himn, username, password,
  294. 'xe vif-list params=network-uuid minimal=true MAC=%s' % mac)
  295. bridge = ssh(himn, username, password,
  296. 'xe network-param-get param-name=bridge uuid=%s' % network_uuid)
  297. # find physical network name
  298. phynet_setting = astute['quantum_settings']['L2']['phys_nets']
  299. physnet = phynet_setting.keys()[0]
  300. return physnet + ':' + bridge
  301. def restart_services(service_name):
  302. execute('stop', service_name)
  303. execute('start', service_name)
  304. def enable_linux_bridge(himn, username, password):
  305. # When using OVS under XS6.5, it will prevent use of Linux bridge in
  306. # Dom0, but neutron-openvswitch-agent in compute node will use Linux
  307. # bridge, so we remove this restriction here
  308. ssh(himn, username, password, 'rm -f /etc/modprobe.d/blacklist-bridge*')
  309. def patch_compute_xenapi():
  310. """replace folder xenapi to add patches which are not merged to upstream"""
  311. # TODO(huanxie): need to confirm the overall patchset list
  312. patchset_dir = sys.path[0]
  313. patchfile_list = ['%s/patchset/vif-plug.patch' % patchset_dir,
  314. '%s/patchset/nova-neutron-race-condition.patch' % patchset_dir,
  315. '%s/patchset/ovs-interim-bridge.patch' % patchset_dir,
  316. '%s/patchset/neutron-security-group.patch' % patchset_dir,
  317. '%s/patchset/speed-up-writing-config-drive.patch' % patchset_dir]
  318. for patch_file in patchfile_list:
  319. execute('patch', '-d', DIST_PACKAGES_DIR, '-p1', '-i', patch_file)
  320. def patch_neutron_ovs_agent():
  321. patchset_dir = sys.path[0]
  322. patch_file = '%s/patchset/neutron-rootwrap-xen-dom0.patch' % patchset_dir
  323. execute('patch', '-d', '/usr/', '-p1', '-i', patch_file)
  324. def apply_sm_patch(himn, username, password):
  325. ver = ssh(himn, username, password,
  326. ('xe host-param-get uuid=$(xe host-list --minimal) '
  327. 'param-name=software-version param-key=product_version_text'))
  328. if ver == "6.5":
  329. ssh(himn, username, password,
  330. "sed -i s/\\'phy\\'/\\'aio\\'/g /opt/xensource/sm/ISCSISR.py")
  331. if __name__ == '__main__':
  332. install_xenapi_sdk()
  333. astute = get_astute(ASTUTE_PATH)
  334. if astute:
  335. username, password, install_xapi = get_options(astute, ASTUTE_SECTION)
  336. endpoints = get_endpoints(astute)
  337. himn_eth, himn_local = init_eth()
  338. public_ip = astute_get(
  339. astute, ('network_metadata', 'vips', 'public', 'ipaddr'))
  340. services_ssl = astute_get(
  341. astute, ('public_ssl', 'services'))
  342. if username and password and endpoints and himn_local:
  343. check_host_compatibility(HIMN_IP, username, password)
  344. route_to_compute(
  345. endpoints, HIMN_IP, himn_local, username, password)
  346. if install_xapi:
  347. install_suppack(HIMN_IP, username, password)
  348. enable_linux_bridge(HIMN_IP, username, password)
  349. forward_from_himn(himn_eth)
  350. # port forwarding for novnc
  351. forward_port('br-mgmt', himn_eth, HIMN_IP, '80')
  352. # apply sm patch
  353. apply_sm_patch(HIMN_IP, username, password)
  354. create_novacompute_conf(HIMN_IP, username, password, public_ip, services_ssl)
  355. patch_compute_xenapi()
  356. restart_services('nova-compute')
  357. install_logrotate_script(HIMN_IP, username, password)
  358. # neutron-l2-agent in compute node
  359. modify_neutron_rootwrap_conf(HIMN_IP, username, password)
  360. br_mappings = find_bridge_mappings(astute, HIMN_IP,
  361. username, password)
  362. modify_neutron_ovs_agent_conf(INT_BRIDGE, br_mappings)
  363. patch_neutron_ovs_agent()
  364. restart_services('neutron-plugin-openvswitch-agent')