From 5caad8e8353c274a7be715ee48e19df672624f00 Mon Sep 17 00:00:00 2001 From: Kirill Omelchenko Date: Tue, 2 Sep 2014 17:35:48 +0300 Subject: [PATCH] Add time-sync functionality to dos.py Add functionality to sync time on specific env nodes on demand using dos.py Related-Bug: #1318641 Change-Id: If26abc316a80e31fb575fec0ebffa1685025d556 --- devops/helpers/helpers.py | 68 +++++++++++++++++++++++++++++++++++++++ devops/settings.py | 6 ++++ devops/shell.py | 44 +++++++++++++++++++++++++ 3 files changed, 118 insertions(+) diff --git a/devops/helpers/helpers.py b/devops/helpers/helpers.py index dcb2cc6a..13e20f7a 100644 --- a/devops/helpers/helpers.py +++ b/devops/helpers/helpers.py @@ -34,6 +34,7 @@ from devops.error import DevopsError from devops.error import TimeoutError from devops.helpers.retry import retry from devops import logger +from devops.settings import SSH_CREDENTIALS def get_free_port(): @@ -128,6 +129,73 @@ def http(host='localhost', port=80, method='GET', url='/', waited_code=200): return False +def get_private_keys(env): + _ssh_keys = [] + admin_remote = get_admin_remote(env) + for key_string in ['/root/.ssh/id_rsa', + '/root/.ssh/bootstrap.rsa']: + with admin_remote.open(key_string) as f: + _ssh_keys.append(paramiko.RSAKey.from_private_key(f)) + return _ssh_keys + + +def get_admin_remote(env): + wait(lambda: tcp_ping(env.node_by_name( + 'admin').get_ip_address_by_network_name('admin'), 22), timeout=180) + return env.node_by_name( + 'admin').remote(network_name=SSH_CREDENTIALS['admin_network'], + login=SSH_CREDENTIALS['login'], + password=SSH_CREDENTIALS['password']) + + +def get_node_remote(env, node_name): + ip = get_slave_ip(env, env.node_by_name( + node_name).interfaces[0].mac_address) + wait(lambda: tcp_ping(ip, 22), timeout=180) + return SSHClient(ip, + username=SSH_CREDENTIALS['login'], + password=SSH_CREDENTIALS['password'], + private_keys=get_private_keys(env)) + + +def sync_node_time(env, node_name='admin', cmd=None): + if cmd is None: + cmd = "hwclock -s && NTPD=$(find /etc/init.d/ -regex " + cmd += "'/etc/init.d/ntp.?'); $NTPD stop; killall ntpd;" + cmd += " ntpd -qg && $NTPD start" + + if node_name == 'admin': + try: + # If public NTP servers aren't accessible ntpdate will fail and + # ntpd daemon shouldn't be restarted to avoid 'Server has gone + # too long without sync' error while syncing time from slaves + remote = get_admin_remote(env) + remote.execute("ntpdate -d $(awk '/^server/{print" + " $2}' /etc/ntp.conf)") + except AssertionError as e: + logger.warning('Error occurred while synchronizing time on master' + ': {0}'.format(e)) + else: + remote = get_admin_remote(env) + remote.execute('service ntpd stop && ntpd -qg && ' + 'service ntpd start') + else: + remote = get_node_remote(env, node_name) + remote.execute(cmd) + remote.execute('hwclock -w') + remote_date = remote.execute('date')['stdout'] + logger.info("Node time: {0}".format(remote_date)) + + +def get_slave_ip(env, node_mac_address): + remote = get_admin_remote(env) + ip = remote.execute( + "fuel nodes --node-id {0} | awk -F'|' " + "'END{{gsub(\" \", \"\", $5); print $5}}'". + format(node_mac_address))['stdout'] + return ip[0].rstrip() + + class KeyPolicy(paramiko.WarningPolicy): def missing_host_key(self, client, hostname, key): return diff --git a/devops/settings.py b/devops/settings.py index 9744a226..b8a99e50 100644 --- a/devops/settings.py +++ b/devops/settings.py @@ -36,6 +36,12 @@ DATABASES = { } } +SSH_CREDENTIALS = { + 'admin_network': environ.get('ENV_ADMIN_NETWORK', 'admin'), + 'login': environ.get('ENV_FUEL_LOGIN', 'root'), + 'password': environ.get('ENV_FUEL_PASSWORD', 'r00tme') +} + SECRET_KEY = 'dummykey' VNC_PASSWORD = environ.get('VNC_PASSWORD', None) diff --git a/devops/shell.py b/devops/shell.py index 6974ebc1..7d279717 100644 --- a/devops/shell.py +++ b/devops/shell.py @@ -17,6 +17,8 @@ from os import environ from devops.manager import Manager +from helpers.helpers import sync_node_time + class Shell(object): def __init__(self): @@ -105,6 +107,24 @@ class Shell(object): for network in networks: print("%15s %10s" % (network.name, network.ip_network)) + def do_timesync(self): + env = self.manager.environment_get(self.params.name) + if not self.params.node_name: + _nodes = {node.name: node.get_vnc_port() for node in env.nodes} + for node_name in sorted(_nodes.keys()): + if _nodes[node_name] != '-1': + sync_node_time(env, node_name) + else: + sync_node_time(env, self.params.node_name) + + def do_revert_resume(self): + self.manager.environment_get(self.params.name).revert( + self.params.snapshot_name) + self.manager.environment_get(self.params.name).resume(verbose=False) + if not self.params.no_timesync: + print('time synchronization is starting') + self.do_timesync() + commands = { 'list': do_list, 'show': do_show, @@ -119,6 +139,8 @@ class Shell(object): 'snapshot-list': do_snapshot_list, 'snapshot-delete': do_snapshot_delete, 'net-list': do_net_list, + 'time-sync': do_timesync, + 'revert-resume': do_revert_resume } def get_params(self): @@ -130,6 +152,15 @@ class Shell(object): snapshot_name_parser.add_argument('--snapshot-name', help='snapshot name', default=environ.get('SNAPSHOT_NAME')) + node_name_parser = argparse.ArgumentParser(add_help=False) + node_name_parser.add_argument('--node-name', + help='node name', + default=None) + no_timesync_parser = argparse.ArgumentParser(add_help=False) + no_timesync_parser.add_argument('--no-timesync', dest='no_timesync', + action='store_const', const=True, + help='revert without timesync', + default=False) parser = argparse.ArgumentParser( description="Manage virtual environments. " "For addional help use command with -h/--help") @@ -186,4 +217,17 @@ class Shell(object): help="Show networks in environment", description="Display allocated networks for " "environment") + subparsers.add_parser('time-sync', + parents=[name_parser, node_name_parser], + help="Sync time on all env nodes", + description="Sync time on all active nodes " + "of environment starting from " + "admin") + subparsers.add_parser('revert-resume', + parents=[name_parser, snapshot_name_parser, + node_name_parser, no_timesync_parser], + help="Revert, resume, sync time on VMs", + description="Revert and resume VMs in selected" + "environment, then" + " sync time on VMs") return parser.parse_args()