diff --git a/ChangeLog b/ChangeLog index e0941e73..c0ea46d9 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,6 +1,23 @@ CHANGES ======= +* Freezer Scheduler for Windows +* Import pep3134daemon as local module +* Freezer instructions for Windows +* Fix metadata storage +* Updated python-keystoneclient requirement +* Updated requirements to match Liberty's one +* Fix tests +* Improvements for windows snapshots +* Fix versions for Liberty + +1.1.3 +----- + +* Add backup metadata fields +* Updated LICENSE file and headers +* PBR version needs to be explicitly set on setup.py +* Fix bug on setup.cfg after bin removal * Fixes for cinder backup * Removed freezer/bin directory from the repo * Add cygwin source of cygwin .dll diff --git a/README.rst b/README.rst index 93c3f905..e696f870 100644 --- a/README.rst +++ b/README.rst @@ -45,25 +45,35 @@ Requirements Windows Requirements ==================== - - Python 2.7 - - GNU Tar binaries (we recommend to follow [this guide](https://github.com/memogarcia/freezer-windows-binaries#windows-binaries-for-freezer) to install them) - - [OpenSSL pre-compiled for windows](https://wiki.openssl.org/index.php/Binaries) or [direct download](https://indy.fulgan.com/SSL/openssl-1.0.1-i386-win32.zip) - - [Sync](https://technet.microsoft.com/en-us/sysinternals/bb897438.aspx) - - [Microsoft Visual C++ Compiler for Python 2.7](http://aka.ms/vcpython27) - - [PyWin32 for python 2.7](http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/) +- Python 2.7 +- GNU Tar binaries (we recommend to follow [this guide](https://github.com/memogarcia/freezer-windows-binaries#windows-binaries-for-freezer) to install them) +- [OpenSSL pre-compiled for windows](https://wiki.openssl.org/index.php/Binaries) or [direct download](https://indy.fulgan.com/SSL/openssl-1.0.1-i386-win32.zip) +- [Sync](https://technet.microsoft.com/en-us/sysinternals/bb897438.aspx) +- [Microsoft Visual C++ Compiler for Python 2.7](http://aka.ms/vcpython27) +- [PyWin32 for python 2.7](http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/) Add binaries to Windows Path ---------------------------- Go to **Control Panel\System and Security\System** and then **Advanced System Settings**, and click **Environment Variables** under **System Variables** edit **Path** and append in the end. - - ;C:\Sync - - ;C:\OpenSSL-Win64\bin - - ;C:\Python27;C:\Python27\Lib\site-packages\;C:\Python27\Scripts\ +- ;C:\Sync +- ;C:\OpenSSL-Win64\bin +- ;C:\Python27;C:\Python27\Lib\site-packages\;C:\Python27\Scripts\ The following components support Windows OS Platform: - freezer-agent - freezer-scheduler +Install Windows Scheduler +------------------------- + +Freezer scheduler on windows run as a windows service and it needs to be installed as a user service. + +- open cmd as admin +- whoami +- cd C:\Python27\Lib\site-packages\freezer\scheduler +- python win_service.py --username {whoami} --password {pc-password} install + Installation & Env Setup ======================== diff --git a/freezer/scheduler/arguments.py b/freezer/scheduler/arguments.py index ea86b888..53cc016e 100644 --- a/freezer/scheduler/arguments.py +++ b/freezer/scheduler/arguments.py @@ -19,8 +19,12 @@ import argparse import os from freezer.apiclient import client as api_client +from freezer import winutils -DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d' +if winutils.is_windows(): + DEFAULT_FREEZER_SCHEDULER_CONF_D = r'C:\.freezer\scheduler\conf.d' +else: + DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d' def base_parser(parser): diff --git a/freezer/scheduler/freezer_scheduler.py b/freezer/scheduler/freezer_scheduler.py index 9dbe78ec..e7edf56b 100755 --- a/freezer/scheduler/freezer_scheduler.py +++ b/freezer/scheduler/freezer_scheduler.py @@ -30,7 +30,11 @@ import arguments import shell import utils -from daemon import Daemon, NoDaemon +from freezer import winutils +if winutils.is_windows(): + from win_daemon import Daemon, NoDaemon +else: + from daemon import Daemon, NoDaemon from scheduler_job import Job @@ -176,7 +180,7 @@ def main(): if args.action is None: print ('No action') - return os.EX_DATAERR + return 65 # os.EX_DATAERR apiclient = None @@ -184,13 +188,17 @@ def main(): apiclient = client.Client(opts=args) if args.client_id: apiclient.client_id = args.client_id + else: + if winutils.is_windows(): + print("--no-api mode is not available on windows") + return 69 # os.EX_UNAVAILABLE if args.action in doers: try: return doers[args.action](apiclient, args) except Exception as e: print ('ERROR {0}'.format(e)) - return os.EX_SOFTWARE + return 70 # os.EX_SOFTWARE freezer_scheduler = FreezerScheduler(apiclient=apiclient, interval=int(args.interval), @@ -200,7 +208,12 @@ def main(): print ('Freezer Scheduler running in no-daemon mode') daemon = NoDaemon(daemonizable=freezer_scheduler) else: - daemon = Daemon(daemonizable=freezer_scheduler) + if winutils.is_windows(): + daemon = Daemon(daemonizable=freezer_scheduler, + interval=int(args.interval), + job_path=args.jobs_dir) + else: + daemon = Daemon(daemonizable=freezer_scheduler) if args.action == 'start': daemon.start(log_file=args.log_file) @@ -211,7 +224,9 @@ def main(): elif args.action == 'status': daemon.status() - return os.EX_OK + # os.RETURN_CODES are only available to posix like systems, on windows + # we need to translate the code to an actual number which is the equivalent + return 0 # os.EX_OK if __name__ == '__main__': diff --git a/freezer/scheduler/scheduler_job.py b/freezer/scheduler/scheduler_job.py index 7ba53cb6..a562e70b 100644 --- a/freezer/scheduler/scheduler_job.py +++ b/freezer/scheduler/scheduler_job.py @@ -25,6 +25,8 @@ import time from ConfigParser import ConfigParser +from freezer import utils + class StopState(object): @@ -279,11 +281,12 @@ class Job(object): freezer_action = job_action.get('freezer_action', {}) max_retries_interval = job_action.get('max_retries_interval', 60) action_name = freezer_action.get('action', '') + config_file_name = None while tries: - with tempfile.NamedTemporaryFile() as config_file: + with tempfile.NamedTemporaryFile(delete=False) as config_file: self.save_action_to_file(freezer_action, config_file) - + config_file_name = config_file.name freezer_command = '{0} --metadata-out - --config {1}'.\ format(self.executable, config_file.name) self.process = subprocess.Popen(freezer_command.split(), @@ -291,6 +294,8 @@ class Job(object): stderr=subprocess.PIPE, env=os.environ.copy()) output, error = self.process.communicate() + # ensure the tempfile gets deleted + utils.delete_file(config_file_name) if error: logging.error("[*] Freezer client error: {0}".format(error)) @@ -316,6 +321,7 @@ class Job(object): return Job.SUCCESS_RESULT logging.error('[*] Job {0} action {1} failed after {2} tries' .format(self.id, action_name, max_retries)) + return Job.FAIL_RESULT def execute(self): diff --git a/freezer/scheduler/utils.py b/freezer/scheduler/utils.py index c0280b68..7f3610dd 100644 --- a/freezer/scheduler/utils.py +++ b/freezer/scheduler/utils.py @@ -35,8 +35,8 @@ def do_register(client, args=None): except freezer.apiclient.exceptions.ApiClientException as e: if e.status_code == 409: print "Client already registered" - return os.EX_CANTCREAT - return os.EX_OK + return 73 # os.EX_CANTCREAT + return 0 # os.EX_OK def find_config_files(path): diff --git a/freezer/scheduler/win_daemon.py b/freezer/scheduler/win_daemon.py new file mode 100644 index 00000000..bd28683c --- /dev/null +++ b/freezer/scheduler/win_daemon.py @@ -0,0 +1,155 @@ +# Copyright 2015 Hewlett-Packard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import logging +import os +import signal +import subprocess + +import win32serviceutil + +from freezer.utils import shield +from freezer.utils import create_subprocess +from freezer.utils import create_dir +from freezer.winutils import save_environment + + +def setup_logging(log_file): + + class NoLogFilter(logging.Filter): + def filter(self, record): + return False + + def configure_logging(file_name): + expanded_file_name = os.path.expanduser(file_name) + expanded_dir_name = os.path.dirname(expanded_file_name) + create_dir(expanded_dir_name, do_log=False) + logging.basicConfig( + filename=expanded_file_name, + level=logging.INFO, + format=('%(asctime)s %(name)s %(levelname)s %(message)s')) + # filter out some annoying messages + # not the best position for this code + log_filter = NoLogFilter() + logging.getLogger("apscheduler.scheduler").\ + addFilter(log_filter) + logging.getLogger("apscheduler.executors.default").\ + addFilter(log_filter) + logging.getLogger("requests.packages.urllib3.connectionpool").\ + addFilter(log_filter) + return expanded_file_name + + log_file_paths = [log_file] if log_file else [ + r'C:\.freezer\freezer-scheduler.log'] + for file_name in log_file_paths: + try: + return configure_logging(file_name) + except IOError: + pass + + raise Exception("Unable to write to log file") + + +class Daemon(object): + """Daemon interface to start a windows service with a freezer-scheduler + instance + """ + def __init__(self, daemonizable=None, interval=None, job_path=None): + self.service_name = 'FreezerService' + self.home = r'C:\.freezer' + # this is only need it in order to have the same interface as in linux + self.daemonizable = daemonizable + self.interval = interval or 60 + self.job_path = job_path or r'C:\.freezer\scheduler\conf.d' + + @shield + def start(self, log_file=None): + """Initialize freezer-scheduler instance inside a windows service + """ + setup_logging(log_file) + + create_dir(self.home) + + # send arguments info to the windows service + os.environ['SERVICE_JOB_PATH'] = self.job_path + os.environ['SERVICE_INTERVAL'] = str(self.interval) + save_environment(self.home) + + print('Freezer Service is starting') + win32serviceutil.StartService(self.service_name) + + @shield + def reload(self): + """Reload the windows service + """ + win32serviceutil.RestartService(self.service_name) + + @shield + def stop(self): + """Stop the windows service by using sc queryex command, if we use + win32serviceutil.StoptService(self.service_name) it never gets stopped + becuase freezer_scheduler.start() blocks the windows service and + prevents any new signal to reach the service. + """ + query = 'sc queryex {0}'.format(self.service_name) + out = create_subprocess(query)[0] + pid = None + for line in out.split('\n'): + if 'PID' in line: + pid = line.split(':')[1].strip() + + command = 'taskkill /f /pid {0}'.format(pid) + create_subprocess(command) + print('Freezer Service has stopped') + + @shield + def status(self): + """Return running status of Freezer Service + by querying win32serviceutil.QueryServiceStatus() + possible running status: + 1 == stop + 4 == running + """ + if win32serviceutil.QueryServiceStatus(self.service_name)[1] == 4: + print("{0} is running normally".format(self.service_name)) + else: + print("{0} is *NOT* running".format(self.service_name)) + + +class NoDaemon(object): + """A class that share the same interface as the Daemon class but doesn't + initialize a windows service to execute the scheduler, it runs in the + foreground + """ + def __init__(self, daemonizable=None): + # this is only need it in order to have the same interface as in linux + self.daemonizable = daemonizable + + @shield + def stop(self): + self.daemonizable.stop() + + def status(self): + """Need it to have the same interface as Daemon + """ + pass + + def reload(self): + """Need it to have the same interface as Daemon + """ + pass + + @shield + def start(self, log_file=None): + self.daemonizable.start() diff --git a/freezer/scheduler/win_service.py b/freezer/scheduler/win_service.py new file mode 100644 index 00000000..43d72755 --- /dev/null +++ b/freezer/scheduler/win_service.py @@ -0,0 +1,91 @@ +# Copyright 2014 Hewlett-Packard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import sys +import servicemanager +import win32service +import win32serviceutil +import win32event + +from freezer.winutils import set_environment + + +class PySvc(win32serviceutil.ServiceFramework): + _svc_name_ = "FreezerService" + _svc_display_name_ = "Freezer Service" + _svc_description_ = "Freezer Service" + + def __init__(self, args): + win32serviceutil.ServiceFramework.__init__(self, args) + # create an event to listen for stop requests on + self.hWaitStop = win32event.CreateEvent(None, 0, 0, None) + self.home = r'C:\.freezer' + + + def SvcDoRun(self): + """Run the windows service and start the scheduler in the background + """ + rc = None + + self.main() + + # if the stop event hasn't been fired keep looping + while rc != win32event.WAIT_OBJECT_0: + # block for 5 seconds and listen for a stop event + rc = win32event.WaitForSingleObject(self.hWaitStop, 5000) + + def SvcStop(self): + """Stop the windows service and stop the scheduler instance + """ + # tell the SCM we're shutting down + self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING) + + # fire the stop event + servicemanager.LogInfoMsg("freezer-scheduler stopped") + win32event.SetEvent(self.hWaitStop) + + def main(self): + import freezer.apiclient.client + from freezer.scheduler.freezer_scheduler import FreezerScheduler + + servicemanager.LogMsg( + servicemanager.EVENTLOG_INFORMATION_TYPE, + servicemanager.PYS_SERVICE_STARTED, + (self._svc_name_, '')) + + set_environment(self.home) + + client = freezer.apiclient.client.Client( + version='2', + username=os.environ['OS_USERNAME'], + password=os.environ['OS_PASSWORD'], + tenant_name=os.environ['OS_TENANT_NAME'], + auth_url=os.environ['OS_AUTH_URL'], + endpoint=os.environ['OS_BACKUP_URL']) + + scheduler = FreezerScheduler( + apiclient=client, interval=int(os.environ['SERVICE_INTERVAL']), + job_path=os.environ['SERVICE_JOB_PATH']) + + scheduler.start() + + +if __name__ == '__main__': + if len(sys.argv) == 1: + servicemanager.Initialize() + servicemanager.PrepareToHostSingle(PySvc) + servicemanager.StartServiceCtrlDispatcher() + else: + win32serviceutil.HandleCommandLine(PySvc) diff --git a/freezer/utils.py b/freezer/utils.py index a8cdd380..4ca753e8 100644 --- a/freezer/utils.py +++ b/freezer/utils.py @@ -26,6 +26,8 @@ from ConfigParser import ConfigParser from distutils import spawn as distspawn import sys +from functools import wraps + class OpenstackOptions: """ @@ -420,3 +422,23 @@ def alter_proxy(proxy): def is_bsd(): return 'darwin' in sys.platform or 'bsd' in sys.platform + + +def shield(func): + """Remove try except boilerplate code from functions""" + @wraps(func) + def wrapper(*args, **kwargs): + try: + return func(*args, **kwargs) + except Exception as error: + logging.error(error) + return wrapper + + +def delete_file(path_to_file): + """Delete a file from the file system + """ + try: + os.remove(path_to_file) + except Exception: + logging.warning("Error deleting file {0}".format(path_to_file)) diff --git a/freezer/winutils.py b/freezer/winutils.py index ef7d34c2..88bb0f3c 100644 --- a/freezer/winutils.py +++ b/freezer/winutils.py @@ -16,6 +16,8 @@ import ctypes import logging import sys +import os +import json from freezer.utils import create_subprocess @@ -57,21 +59,6 @@ def use_shadow(to_backup, windows_volume): .format(windows_volume)) -def clean_tar_command(tar_cmd): - """ Delete tar arguments that are not supported by GnuWin32 tar""" - tar_cmd = tar_cmd.replace('--hard-dereference', '') - tar_cmd = tar_cmd.replace('--no-check-device', '') - tar_cmd = tar_cmd.replace('--warning=none', '') - tar_cmd = tar_cmd.replace('--seek', '') - tar_cmd = tar_cmd.replace('-z', '') - return tar_cmd - - -def add_gzip_to_command(tar_cmd): - gzip_cmd = 'gzip -7' - return '{0} | {1}'.format(tar_cmd, gzip_cmd) - - def stop_sql_server(sql_server_instance): """ Stop a SQL Server instance to perform the backup of the db files """ @@ -95,3 +82,24 @@ def start_sql_server(sql_server_instance): raise Exception('[*] Error while starting SQL Server' ', error {0}'.format(err)) logging.info('[*] SQL Server back to normal') + + +def save_environment(home): + """Read the environment from the terminal where the scheduler is + initialized and save the environment variables to be reused within the + windows service + """ + env_path = os.path.join(home, 'env.json') + with open(env_path, 'wb') as tmp: + json.dump(os.environ.copy(), tmp) + + +def set_environment(home): + """Read the environment variables saved by the win_daemon and restore it + here for the windows service + """ + json_env = os.path.join(home, 'env.json') + with open(json_env, 'rb') as fp: + env = json.loads(fp.read()) + for k, v in env.iteritems(): + os.environ[str(k).strip()] = str(v).strip() diff --git a/tests/test_winutils.py b/tests/test_winutils.py index 81b410e0..80eeab8d 100644 --- a/tests/test_winutils.py +++ b/tests/test_winutils.py @@ -14,8 +14,6 @@ from freezer.winutils import is_windows from freezer.winutils import use_shadow -from freezer.winutils import clean_tar_command -from freezer.winutils import add_gzip_to_command from freezer.winutils import DisableFileSystemRedirection from freezer import winutils from commons import * @@ -54,26 +52,6 @@ class TestWinutils(unittest.TestCase): # test if the volume format is incorrect self.assertRaises(Exception, use_shadow(path, test_volume)) - def test_clean_tar_command(self): - test_tar_command = 'tar --create -z --warning=none ' \ - '--no-check-device --one-file-system ' \ - '--preserve-permissions --same-owner --seek ' \ - '--ignore-failed-read ' - expected = 'tar --create --one-file-system --preserve-permissions ' \ - '--same-owner --ignore-failed-read ' - - assert clean_tar_command(test_tar_command) == expected - - def test_add_gzip_to_command(self): - test_command = 'tar --create --one-file-system ' \ - '--preserve-permissions --same-owner ' \ - '--ignore-failed-read ' - expected = 'tar --create --one-file-system ' \ - '--preserve-permissions --same-owner ' \ - '--ignore-failed-read | gzip -7' - - assert add_gzip_to_command(test_command) == expected - # def test_start_sql_server(self): # backup_opt = BackupOpt1() # self.mock_process(FakeSubProcess())