Freezer Scheduler for Windows

Freezer now has support for scheduling jobs through windows service.

This windows service needs to be installed in user land rather than
service land, instructions are included in the README.rst

--no-api mode is not available on windows

Change-Id: I34db28ffc928703b01e6430a661214d66d81c518

ChangeLog

@@ -1,6 +1,23 @@
 CHANGES
 =======
 
+* Freezer Scheduler for Windows
+* Import pep3134daemon as local module
+* Freezer instructions for Windows
+* Fix metadata storage
+* Updated python-keystoneclient requirement
+* Updated requirements to match Liberty's one
+* Fix tests
+* Improvements for windows snapshots
+* Fix versions for Liberty
+
+1.1.3
+-----
+
+* Add backup metadata fields
+* Updated LICENSE file and headers
+* PBR version needs to be explicitly set on setup.py
+* Fix bug on setup.cfg after bin removal
 * Fixes for cinder backup
 * Removed freezer/bin directory from the repo
 * Add cygwin source of cygwin .dll

README.rst

@@ -45,25 +45,35 @@ Requirements
 Windows Requirements
 ====================
 
-  - Python 2.7
-  - GNU Tar binaries (we recommend to follow [this guide](https://github.com/memogarcia/freezer-windows-binaries#windows-binaries-for-freezer) to install them)
-  - [OpenSSL pre-compiled for windows](https://wiki.openssl.org/index.php/Binaries) or [direct download](https://indy.fulgan.com/SSL/openssl-1.0.1-i386-win32.zip)
-  - [Sync](https://technet.microsoft.com/en-us/sysinternals/bb897438.aspx)
-  - [Microsoft Visual C++ Compiler for Python 2.7](http://aka.ms/vcpython27)
-  - [PyWin32 for python 2.7](http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/)
+- Python 2.7
+- GNU Tar binaries (we recommend to follow [this guide](https://github.com/memogarcia/freezer-windows-binaries#windows-binaries-for-freezer) to install them)
+- [OpenSSL pre-compiled for windows](https://wiki.openssl.org/index.php/Binaries) or [direct download](https://indy.fulgan.com/SSL/openssl-1.0.1-i386-win32.zip)
+- [Sync](https://technet.microsoft.com/en-us/sysinternals/bb897438.aspx)
+- [Microsoft Visual C++ Compiler for Python 2.7](http://aka.ms/vcpython27)
+- [PyWin32 for python 2.7](http://sourceforge.net/projects/pywin32/files/pywin32/Build%20219/)
 
 Add binaries to Windows Path
 ----------------------------
 Go to **Control Panel\System and Security\System** and then **Advanced System Settings**, and click **Environment Variables** under **System Variables** edit **Path** and append in the end.
-  - ;C:\Sync
-  - ;C:\OpenSSL-Win64\bin
-  - ;C:\Python27;C:\Python27\Lib\site-packages\;C:\Python27\Scripts\
+- ;C:\Sync
+- ;C:\OpenSSL-Win64\bin
+- ;C:\Python27;C:\Python27\Lib\site-packages\;C:\Python27\Scripts\
 
 The following components support Windows OS Platform:
 
 - freezer-agent
 - freezer-scheduler
 
+Install Windows Scheduler
+-------------------------
+
+Freezer scheduler on windows run as a windows service and it needs to be installed as a user service.
+
+- open cmd as admin
+- whoami
+- cd C:\Python27\Lib\site-packages\freezer\scheduler
+- python win_service.py --username {whoami} --password {pc-password} install
+
 
 Installation & Env Setup
 ========================

freezer/scheduler/arguments.py

@@ -19,8 +19,12 @@ import argparse
 import os
 
 from freezer.apiclient import client as api_client
+from freezer import winutils
 
-DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d'
+if winutils.is_windows():
+    DEFAULT_FREEZER_SCHEDULER_CONF_D = r'C:\.freezer\scheduler\conf.d'
+else:
+    DEFAULT_FREEZER_SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d'
 
 
 def base_parser(parser):

freezer/scheduler/freezer_scheduler.py

@@ -30,7 +30,11 @@ import arguments
 import shell
 import utils
 
-from daemon import Daemon, NoDaemon
+from freezer import winutils
+if winutils.is_windows():
+    from win_daemon import Daemon, NoDaemon
+else:
+    from daemon import Daemon, NoDaemon
 from scheduler_job import Job
 
 
@@ -176,7 +180,7 @@ def main():
 
     if args.action is None:
         print ('No action')
-        return os.EX_DATAERR
+        return 65  # os.EX_DATAERR
 
     apiclient = None
 
@@ -184,13 +188,17 @@ def main():
         apiclient = client.Client(opts=args)
         if args.client_id:
             apiclient.client_id = args.client_id
+    else:
+        if winutils.is_windows():
+            print("--no-api mode is not available on windows")
+            return 69  # os.EX_UNAVAILABLE
 
     if args.action in doers:
         try:
             return doers[args.action](apiclient, args)
         except Exception as e:
             print ('ERROR {0}'.format(e))
-            return os.EX_SOFTWARE
+            return 70  # os.EX_SOFTWARE
 
     freezer_scheduler = FreezerScheduler(apiclient=apiclient,
                                          interval=int(args.interval),
@@ -200,7 +208,12 @@ def main():
         print ('Freezer Scheduler running in no-daemon mode')
         daemon = NoDaemon(daemonizable=freezer_scheduler)
     else:
-        daemon = Daemon(daemonizable=freezer_scheduler)
+        if winutils.is_windows():
+            daemon = Daemon(daemonizable=freezer_scheduler,
+                            interval=int(args.interval),
+                            job_path=args.jobs_dir)
+        else:
+            daemon = Daemon(daemonizable=freezer_scheduler)
 
     if args.action == 'start':
         daemon.start(log_file=args.log_file)
@@ -211,7 +224,9 @@ def main():
     elif args.action == 'status':
         daemon.status()
 
-    return os.EX_OK
+    # os.RETURN_CODES are only available to posix like systems, on windows
+    # we need to translate the code to an actual number which is the equivalent
+    return 0  # os.EX_OK
 
 
 if __name__ == '__main__':

freezer/scheduler/scheduler_job.py

@@ -25,6 +25,8 @@ import time
 
 from ConfigParser import ConfigParser
 
+from freezer import utils
+
 
 class StopState(object):
 
@@ -279,11 +281,12 @@ class Job(object):
         freezer_action = job_action.get('freezer_action', {})
         max_retries_interval = job_action.get('max_retries_interval', 60)
         action_name = freezer_action.get('action', '')
+        config_file_name = None
         while tries:
 
-            with tempfile.NamedTemporaryFile() as config_file:
+            with tempfile.NamedTemporaryFile(delete=False) as config_file:
                 self.save_action_to_file(freezer_action, config_file)
-
+                config_file_name = config_file.name
                 freezer_command = '{0} --metadata-out - --config {1}'.\
                     format(self.executable, config_file.name)
                 self.process = subprocess.Popen(freezer_command.split(),
@@ -291,6 +294,8 @@ class Job(object):
                                                 stderr=subprocess.PIPE,
                                                 env=os.environ.copy())
                 output, error = self.process.communicate()
+                # ensure the tempfile gets deleted
+                utils.delete_file(config_file_name)
 
             if error:
                 logging.error("[*] Freezer client error: {0}".format(error))
@@ -316,6 +321,7 @@ class Job(object):
                 return Job.SUCCESS_RESULT
         logging.error('[*] Job {0} action {1} failed after {2} tries'
                       .format(self.id, action_name, max_retries))
+
         return Job.FAIL_RESULT
 
     def execute(self):

freezer/scheduler/utils.py

@@ -35,8 +35,8 @@ def do_register(client, args=None):
         except freezer.apiclient.exceptions.ApiClientException as e:
             if e.status_code == 409:
                 print "Client already registered"
-            return os.EX_CANTCREAT
-        return os.EX_OK
+            return 73  # os.EX_CANTCREAT
+        return 0  # os.EX_OK
 
 
 def find_config_files(path):

freezer/scheduler/win_daemon.py

@@ -0,0 +1,155 @@
+# Copyright 2015 Hewlett-Packard
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import os
+import signal
+import subprocess
+
+import win32serviceutil
+
+from freezer.utils import shield
+from freezer.utils import create_subprocess
+from freezer.utils import create_dir
+from freezer.winutils import save_environment
+
+
+def setup_logging(log_file):
+
+    class NoLogFilter(logging.Filter):
+        def filter(self, record):
+            return False
+
+    def configure_logging(file_name):
+        expanded_file_name = os.path.expanduser(file_name)
+        expanded_dir_name = os.path.dirname(expanded_file_name)
+        create_dir(expanded_dir_name, do_log=False)
+        logging.basicConfig(
+            filename=expanded_file_name,
+            level=logging.INFO,
+            format=('%(asctime)s %(name)s %(levelname)s %(message)s'))
+        # filter out some annoying messages
+        # not the best position for this code
+        log_filter = NoLogFilter()
+        logging.getLogger("apscheduler.scheduler").\
+            addFilter(log_filter)
+        logging.getLogger("apscheduler.executors.default").\
+            addFilter(log_filter)
+        logging.getLogger("requests.packages.urllib3.connectionpool").\
+            addFilter(log_filter)
+        return expanded_file_name
+
+    log_file_paths = [log_file] if log_file else [
+        r'C:\.freezer\freezer-scheduler.log']
+    for file_name in log_file_paths:
+        try:
+            return configure_logging(file_name)
+        except IOError:
+            pass
+
+    raise Exception("Unable to write to log file")
+
+
+class Daemon(object):
+    """Daemon interface to start a windows service with a freezer-scheduler
+    instance
+    """
+    def __init__(self, daemonizable=None, interval=None, job_path=None):
+        self.service_name = 'FreezerService'
+        self.home = r'C:\.freezer'
+        # this is only need it in order to have the same interface as in linux
+        self.daemonizable = daemonizable
+        self.interval = interval or 60
+        self.job_path = job_path or r'C:\.freezer\scheduler\conf.d'
+
+    @shield
+    def start(self, log_file=None):
+        """Initialize freezer-scheduler instance inside a windows service
+        """
+        setup_logging(log_file)
+
+        create_dir(self.home)
+
+        # send arguments info to the windows service
+        os.environ['SERVICE_JOB_PATH'] = self.job_path
+        os.environ['SERVICE_INTERVAL'] = str(self.interval)
+        save_environment(self.home)
+
+        print('Freezer Service is starting')
+        win32serviceutil.StartService(self.service_name)
+
+    @shield
+    def reload(self):
+        """Reload the windows service
+        """
+        win32serviceutil.RestartService(self.service_name)
+
+    @shield
+    def stop(self):
+        """Stop the windows service by using sc queryex command, if we use
+        win32serviceutil.StoptService(self.service_name) it never gets stopped
+        becuase freezer_scheduler.start() blocks the windows service and
+        prevents any new signal to reach the service.
+        """
+        query = 'sc queryex {0}'.format(self.service_name)
+        out = create_subprocess(query)[0]
+        pid = None
+        for line in out.split('\n'):
+            if 'PID' in line:
+                pid = line.split(':')[1].strip()
+
+        command = 'taskkill /f /pid {0}'.format(pid)
+        create_subprocess(command)
+        print('Freezer Service has stopped')
+
+    @shield
+    def status(self):
+        """Return running status of Freezer Service
+        by querying win32serviceutil.QueryServiceStatus()
+        possible running status:
+            1 == stop
+            4 == running
+        """
+        if win32serviceutil.QueryServiceStatus(self.service_name)[1] == 4:
+            print("{0} is running normally".format(self.service_name))
+        else:
+            print("{0} is *NOT* running".format(self.service_name))
+
+
+class NoDaemon(object):
+    """A class that share the same interface as the Daemon class but doesn't
+    initialize a windows service to execute the scheduler, it runs in the
+    foreground
+    """
+    def __init__(self, daemonizable=None):
+        # this is only need it in order to have the same interface as in linux
+        self.daemonizable = daemonizable
+
+    @shield
+    def stop(self):
+        self.daemonizable.stop()
+
+    def status(self):
+        """Need it to have the same interface as Daemon
+        """
+        pass
+
+    def reload(self):
+        """Need it to have the same interface as Daemon
+        """
+        pass
+
+    @shield
+    def start(self, log_file=None):
+        self.daemonizable.start()

freezer/scheduler/win_service.py

@@ -0,0 +1,91 @@
+# Copyright 2014 Hewlett-Packard
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import servicemanager
+import win32service
+import win32serviceutil
+import win32event
+
+from freezer.winutils import set_environment
+
+
+class PySvc(win32serviceutil.ServiceFramework):
+    _svc_name_ = "FreezerService"
+    _svc_display_name_ = "Freezer Service"
+    _svc_description_ = "Freezer Service"
+
+    def __init__(self, args):
+        win32serviceutil.ServiceFramework.__init__(self, args)
+        # create an event to listen for stop requests on
+        self.hWaitStop = win32event.CreateEvent(None, 0, 0, None)
+        self.home = r'C:\.freezer'
+
+
+    def SvcDoRun(self):
+        """Run the windows service and start the scheduler in the background
+        """
+        rc = None
+
+        self.main()
+
+        # if the stop event hasn't been fired keep looping
+        while rc != win32event.WAIT_OBJECT_0:
+            # block for 5 seconds and listen for a stop event
+            rc = win32event.WaitForSingleObject(self.hWaitStop, 5000)
+
+    def SvcStop(self):
+        """Stop the windows service and stop the scheduler instance
+        """
+        # tell the SCM we're shutting down
+        self.ReportServiceStatus(win32service.SERVICE_STOP_PENDING)
+
+        # fire the stop event
+        servicemanager.LogInfoMsg("freezer-scheduler stopped")
+        win32event.SetEvent(self.hWaitStop)
+
+    def main(self):
+        import freezer.apiclient.client
+        from freezer.scheduler.freezer_scheduler import FreezerScheduler
+
+        servicemanager.LogMsg(
+            servicemanager.EVENTLOG_INFORMATION_TYPE,
+            servicemanager.PYS_SERVICE_STARTED,
+            (self._svc_name_, ''))
+
+        set_environment(self.home)
+
+        client = freezer.apiclient.client.Client(
+            version='2',
+            username=os.environ['OS_USERNAME'],
+            password=os.environ['OS_PASSWORD'],
+            tenant_name=os.environ['OS_TENANT_NAME'],
+            auth_url=os.environ['OS_AUTH_URL'],
+            endpoint=os.environ['OS_BACKUP_URL'])
+
+        scheduler = FreezerScheduler(
+            apiclient=client, interval=int(os.environ['SERVICE_INTERVAL']),
+            job_path=os.environ['SERVICE_JOB_PATH'])
+
+        scheduler.start()
+
+
+if __name__ == '__main__':
+    if len(sys.argv) == 1:
+        servicemanager.Initialize()
+        servicemanager.PrepareToHostSingle(PySvc)
+        servicemanager.StartServiceCtrlDispatcher()
+    else:
+        win32serviceutil.HandleCommandLine(PySvc)

freezer/utils.py

@@ -26,6 +26,8 @@ from ConfigParser import ConfigParser
 from distutils import spawn as distspawn
 import sys
 
+from functools import wraps
+
 
 class OpenstackOptions:
     """
@@ -420,3 +422,23 @@ def alter_proxy(proxy):
 
 def is_bsd():
     return 'darwin' in sys.platform or 'bsd' in sys.platform
+
+
+def shield(func):
+    """Remove try except boilerplate code from functions"""
+    @wraps(func)
+    def wrapper(*args, **kwargs):
+        try:
+            return func(*args, **kwargs)
+        except Exception as error:
+            logging.error(error)
+    return wrapper
+
+
+def delete_file(path_to_file):
+    """Delete a file from the file system
+    """
+    try:
+        os.remove(path_to_file)
+    except Exception:
+        logging.warning("Error deleting file {0}".format(path_to_file))

freezer/winutils.py

@@ -16,6 +16,8 @@
 import ctypes
 import logging
 import sys
+import os
+import json
 
 from freezer.utils import create_subprocess
 
@@ -57,21 +59,6 @@ def use_shadow(to_backup, windows_volume):
                              .format(windows_volume))
 
 
-def clean_tar_command(tar_cmd):
-    """ Delete tar arguments that are not supported by GnuWin32 tar"""
-    tar_cmd = tar_cmd.replace('--hard-dereference', '')
-    tar_cmd = tar_cmd.replace('--no-check-device', '')
-    tar_cmd = tar_cmd.replace('--warning=none', '')
-    tar_cmd = tar_cmd.replace('--seek', '')
-    tar_cmd = tar_cmd.replace('-z', '')
-    return tar_cmd
-
-
-def add_gzip_to_command(tar_cmd):
-    gzip_cmd = 'gzip -7'
-    return '{0} | {1}'.format(tar_cmd, gzip_cmd)
-
-
 def stop_sql_server(sql_server_instance):
     """ Stop a SQL Server instance to perform the backup of the db files """
 
@@ -95,3 +82,24 @@ def start_sql_server(sql_server_instance):
             raise Exception('[*] Error while starting SQL Server'
                             ', error {0}'.format(err))
         logging.info('[*] SQL Server back to normal')
+
+
+def save_environment(home):
+    """Read the environment from the terminal where the scheduler is
+    initialized and save the environment variables to be reused within the
+    windows service
+    """
+    env_path = os.path.join(home, 'env.json')
+    with open(env_path, 'wb') as tmp:
+        json.dump(os.environ.copy(), tmp)
+
+
+def set_environment(home):
+    """Read the environment variables saved by the win_daemon and restore it
+    here for the windows service
+    """
+    json_env = os.path.join(home, 'env.json')
+    with open(json_env, 'rb') as fp:
+        env = json.loads(fp.read())
+        for k, v in env.iteritems():
+            os.environ[str(k).strip()] = str(v).strip()

tests/test_winutils.py

@@ -14,8 +14,6 @@
 
 from freezer.winutils import is_windows
 from freezer.winutils import use_shadow
-from freezer.winutils import clean_tar_command
-from freezer.winutils import add_gzip_to_command
 from freezer.winutils import DisableFileSystemRedirection
 from freezer import winutils
 from commons import *
@@ -54,26 +52,6 @@ class TestWinutils(unittest.TestCase):
         # test if the volume format is incorrect
         self.assertRaises(Exception, use_shadow(path, test_volume))
 
-    def test_clean_tar_command(self):
-        test_tar_command = 'tar --create -z --warning=none ' \
-                           '--no-check-device --one-file-system ' \
-                           '--preserve-permissions --same-owner --seek ' \
-                           '--ignore-failed-read '
-        expected = 'tar --create    --one-file-system --preserve-permissions ' \
-                   '--same-owner  --ignore-failed-read '
-
-        assert clean_tar_command(test_tar_command) == expected
-
-    def test_add_gzip_to_command(self):
-        test_command = 'tar --create    --one-file-system ' \
-                       '--preserve-permissions --same-owner ' \
-                       '--ignore-failed-read '
-        expected = 'tar --create    --one-file-system ' \
-                   '--preserve-permissions --same-owner ' \
-                   '--ignore-failed-read  | gzip -7'
-
-        assert add_gzip_to_command(test_command) == expected
-
     # def test_start_sql_server(self):
     #     backup_opt = BackupOpt1()
     #     self.mock_process(FakeSubProcess())