From d4b9399e9b3051b03ae0e4206a65a0fc97e70d67 Mon Sep 17 00:00:00 2001 From: Fabrizio Vanni Date: Mon, 8 Jun 2015 12:05:08 +0100 Subject: [PATCH] freezer scheduler The freezer scheduler is to be executed as daemon process on the client machines It has the following responsibilities: * when using the api: - register -if necessary- as a client in the api - download the list of jobs from the api - schedule the jobs for execution - launch the freezer client at the scheduled time - collect metadata and exit codes and upload them to the api - periodically poll the api for new/updated jobs - if a job is part of a session (a coordinated group of jobs) it updates the session status when job starts/stops * when not using the api - load jobs configurations from files - schedule the jobs for execution - launch the freezer client at the scheduled time The freezer scheduler can also be used to manage jobs and sessions using the following positional parameters: job-list job-get job-create job-delete job-start job-stop session-list session-get session-create session-delete session-list-job session-add-job session-remove-job or to register the client in the api using the positional parameter: register Implements blueprint: freezer-scheduler-start Change-Id: I06ae202a0f464f7240c137744a5b54d1177cabd9 --- freezer/apiclient/actions.py | 80 ++ freezer/apiclient/client.py | 4 + freezer/apiclient/exceptions.py | 4 + freezer/apiclient/sessions.py | 162 ++++ freezer/arguments.py | 8 +- freezer/job.py | 24 + freezer/main.py | 7 +- freezer/scheduler/__init__.py | 0 freezer/scheduler/arguments.py | 170 ++++ freezer/scheduler/daemon.py | 160 ++++ freezer/scheduler/freezer_scheduler.py | 229 ++++++ freezer/scheduler/scheduler_job.py | 440 ++++++++++ freezer/scheduler/shell.py | 236 ++++++ freezer/scheduler/utils.py | 100 +++ freezer/swift.py | 3 - freezer_api/README.rst | 770 +++++++++++------- freezer_api/freezer_api/api/v1/__init__.py | 24 +- freezer_api/freezer_api/api/v1/actions.py | 101 +++ freezer_api/freezer_api/api/v1/sessions.py | 348 ++++++++ freezer_api/freezer_api/cmd/api.py | 7 +- freezer_api/freezer_api/common/exceptions.py | 15 +- .../freezer_api/common/json_schemas.py | 411 +++++++--- freezer_api/freezer_api/common/utils.py | 105 ++- freezer_api/freezer_api/storage/elastic.py | 224 ++++- freezer_api/requirements.txt | 6 + freezer_api/setup.cfg | 2 +- freezer_api/tests/common.py | 250 ++++-- freezer_api/tests/test_actions.py | 148 ++++ freezer_api/tests/test_elastic.py | 423 +++++++++- freezer_api/tests/test_homedoc.py | 2 +- freezer_api/tests/test_jobs.py | 22 + freezer_api/tests/test_sessions.py | 435 ++++++++++ freezer_api/tests/test_utils.py | 122 ++- freezer_api/tox.ini | 1 + requirements.txt | 2 + setup.py | 2 +- tests/test_apiclient_actions.py | 132 +++ tests/test_apiclient_registration.py | 7 + tests/test_apiclient_sessions.py | 227 ++++++ tests/test_scheduler_arguments.py | 84 ++ tests/test_scheduler_daemon.py | 56 ++ tox.ini | 2 + 42 files changed, 5011 insertions(+), 544 deletions(-) create mode 100644 freezer/apiclient/actions.py create mode 100644 freezer/apiclient/sessions.py create mode 100644 freezer/scheduler/__init__.py create mode 100644 freezer/scheduler/arguments.py create mode 100644 freezer/scheduler/daemon.py create mode 100755 freezer/scheduler/freezer_scheduler.py create mode 100644 freezer/scheduler/scheduler_job.py create mode 100644 freezer/scheduler/shell.py create mode 100644 freezer/scheduler/utils.py create mode 100644 freezer_api/freezer_api/api/v1/actions.py create mode 100644 freezer_api/freezer_api/api/v1/sessions.py create mode 100644 freezer_api/requirements.txt create mode 100644 freezer_api/tests/test_actions.py create mode 100644 freezer_api/tests/test_sessions.py create mode 100644 tests/test_apiclient_actions.py create mode 100644 tests/test_apiclient_sessions.py create mode 100644 tests/test_scheduler_arguments.py create mode 100644 tests/test_scheduler_daemon.py diff --git a/freezer/apiclient/actions.py b/freezer/apiclient/actions.py new file mode 100644 index 00000000..5d19fa2f --- /dev/null +++ b/freezer/apiclient/actions.py @@ -0,0 +1,80 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import requests + +import exceptions + + +class ActionManager(object): + + def __init__(self, client): + self.client = client + self.endpoint = self.client.endpoint + '/v1/actions/' + + @property + def headers(self): + return {'X-Auth-Token': self.client.auth_token} + + def create(self, doc, action_id=''): + action_id = action_id or doc.get('action_id', '') + endpoint = self.endpoint + action_id + r = requests.post(endpoint, + data=json.dumps(doc), + headers=self.headers) + if r.status_code != 201: + raise exceptions.ApiClientException(r) + action_id = r.json()['action_id'] + return action_id + + def delete(self, action_id): + endpoint = self.endpoint + action_id + r = requests.delete(endpoint, headers=self.headers) + if r.status_code != 204: + raise exceptions.ApiClientException(r) + + def list(self, limit=10, offset=0, search=None): + data = json.dumps(search) if search else None + query = {'limit': int(limit), 'offset': int(offset)} + r = requests.get(self.endpoint, headers=self.headers, + params=query, data=data) + if r.status_code != 200: + raise exceptions.ApiClientException(r) + return r.json()['actions'] + + def get(self, action_id): + endpoint = self.endpoint + action_id + r = requests.get(endpoint, headers=self.headers) + if r.status_code == 200: + return r.json() + if r.status_code == 404: + return None + raise exceptions.ApiClientException(r) + + def update(self, action_id, update_doc): + endpoint = self.endpoint + action_id + r = requests.patch(endpoint, + headers=self.headers, + data=json.dumps(update_doc)) + if r.status_code != 200: + raise exceptions.ApiClientException(r) + return r.json()['version'] diff --git a/freezer/apiclient/client.py b/freezer/apiclient/client.py index 74b57e91..e96edfb3 100644 --- a/freezer/apiclient/client.py +++ b/freezer/apiclient/client.py @@ -26,6 +26,8 @@ from openstackclient.identity import client as os_client from backups import BackupsManager from registration import RegistrationManager from jobs import JobManager +from actions import ActionManager +from sessions import SessionManager import exceptions @@ -64,6 +66,8 @@ class Client(object): self.backups = BackupsManager(self) self.registration = RegistrationManager(self) self.jobs = JobManager(self) + self.actions = ActionManager(self) + self.sessions = SessionManager(self) @cached_property def endpoint(self): diff --git a/freezer/apiclient/exceptions.py b/freezer/apiclient/exceptions.py index fea99307..c99a6b71 100644 --- a/freezer/apiclient/exceptions.py +++ b/freezer/apiclient/exceptions.py @@ -67,4 +67,8 @@ class ApiClientException(Exception): message = self.get_message_from_api_response(r) or \ self.get_message_from_response(r) or \ str(r) + try: + self.status_code = r.status_code + except: + self.status_code = None super(ApiClientException, self).__init__(message) diff --git a/freezer/apiclient/sessions.py b/freezer/apiclient/sessions.py new file mode 100644 index 00000000..4219be0e --- /dev/null +++ b/freezer/apiclient/sessions.py @@ -0,0 +1,162 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import requests + +import exceptions + + +class SessionManager(object): + + def __init__(self, client): + self.client = client + self.endpoint = self.client.endpoint + '/v1/sessions/' + + @property + def headers(self): + return {'X-Auth-Token': self.client.auth_token} + + def create(self, doc, session_id=''): + session_id = session_id or doc.get('session_id', '') + endpoint = self.endpoint + session_id + r = requests.post(endpoint, + data=json.dumps(doc), + headers=self.headers) + if r.status_code != 201: + raise exceptions.ApiClientException(r) + session_id = r.json()['session_id'] + return session_id + + def delete(self, session_id): + endpoint = self.endpoint + session_id + r = requests.delete(endpoint, headers=self.headers) + if r.status_code != 204: + raise exceptions.ApiClientException(r) + + def list_all(self, limit=10, offset=0, search=None): + data = json.dumps(search) if search else None + query = {'limit': int(limit), 'offset': int(offset)} + r = requests.get(self.endpoint, headers=self.headers, + params=query, data=data) + if r.status_code != 200: + raise exceptions.ApiClientException(r) + return r.json()['sessions'] + + def list(self, limit=10, offset=0, search={}): + new_search = search.copy() + new_search['match'] = search.get('match', []) + return self.list_all(limit, offset, new_search) + + def get(self, session_id): + endpoint = self.endpoint + session_id + r = requests.get(endpoint, headers=self.headers) + if r.status_code == 200: + return r.json() + if r.status_code == 404: + return None + raise exceptions.ApiClientException(r) + + def update(self, session_id, update_doc): + endpoint = self.endpoint + session_id + r = requests.patch(endpoint, + headers=self.headers, + data=json.dumps(update_doc)) + if r.status_code != 200: + raise exceptions.ApiClientException(r) + return r.json()['version'] + + def add_job(self, session_id, job_id): + # endpoint /v1/sessions/{sessions_id}/jobs/{job_id} + endpoint = '{0}{1}/jobs/{2}'.format(self.endpoint, session_id, job_id) + r = requests.put(endpoint, + headers=self.headers) + if r.status_code != 204: + raise exceptions.ApiClientException(r) + return + + def remove_job(self, session_id, job_id): + # endpoint /v1/sessions/{sessions_id}/jobs/{job_id} + endpoint = '{0}{1}/jobs/{2}'.format(self.endpoint, session_id, job_id) + retry = 5 + r = '' + while retry: + r = requests.delete(endpoint, + headers=self.headers) + if r.status_code == 204: + return + retry -= 1 + raise exceptions.ApiClientException(r) + + def start_session(self, session_id, job_id, session_tag): + """ + Informs the api that the client is starting the session + identified by the session_id and request the session_tag + to be incremented up to the requested value. + The returned session_id could be: + * current_tag + 1 if the session has started + * > current_tag + 1 if the action had already been started + by some other node and this node was out of sync + + :param session_id: + :param job_id: + :param session_tag: the new session_id + :return: the response obj: + { result: string 'running' or 'error', + 'session_tag': the new session_tag ) + """ + # endpoint /v1/sessions/{sessions_id}/action + endpoint = '{0}{1}/action'.format(self.endpoint, session_id) + doc = {"start": { + "job_id": job_id, + "current_tag": session_tag + }} + r = requests.post(endpoint, + headers=self.headers, + data=json.dumps(doc)) + if r.status_code != 202: + raise exceptions.ApiClientException(r) + return r.json() + + def end_session(self, session_id, job_id, session_tag, result): + """ + Informs the freezer service that the job has ended. + Privides information about the job's result and the session tag + + :param session_id: + :param job_id: + :param session_tag: + :param result: + :return: + """ + # endpoint /v1/sessions/{sessions_id}/action + endpoint = '{0}{1}/action'.format(self.endpoint, session_id) + doc = {"end": { + "job_id": job_id, + "current_tag": session_tag, + "result": result + }} + r = requests.post(endpoint, + headers=self.headers, + data=json.dumps(doc)) + if r.status_code != 202: + raise exceptions.ApiClientException(r) + return r.json() diff --git a/freezer/arguments.py b/freezer/arguments.py index 7bc01f0c..da4cbdb8 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -64,7 +64,7 @@ DEFAULT_PARAMS = { 'restore_abs_path': False, 'log_file': None, 'upload': True, 'mode': 'fs', 'action': 'backup', 'vssadmin': True, 'shadow': '', 'shadow_path': '', - 'windows_volume': '', 'command': None + 'windows_volume': '', 'command': None, 'metadata_out': False } @@ -268,6 +268,12 @@ def backup_arguments(args_dict={}): password = port = ''', dest='mysql_conf', default=False) + arg_parser.add_argument( + '--metadata-out', action='store', + help=('Set the filename to which write the metadata regarding ' + 'the backup metrics. Use "-" to output to standard output.'), + dest='metadata_out', default=False) + if is_windows(): arg_parser.add_argument( '--log-file', action='store', diff --git a/freezer/job.py b/freezer/job.py index cfb42763..0af7dbf2 100644 --- a/freezer/job.py +++ b/freezer/job.py @@ -19,6 +19,8 @@ Hudson (tjh@cryptsoft.com). ======================================================================== """ +import sys + from freezer import swift from freezer import utils from freezer import backup @@ -35,6 +37,9 @@ class Job: def execute(self): logging.info('[*] Action not implemented') + def get_metadata(self): + return None + @staticmethod def executemethod(func): def wrapper(self): @@ -115,6 +120,25 @@ class BackupJob(Job): else: raise ValueError('Please provide a valid backup mode') + def get_metadata(self): + metadata = { + 'current_level': self.conf.curr_backup_level, + 'fs_real_path': (self.conf.lvm_auto_snap or + self.conf.path_to_backup), + 'vol_snap_path': + self.conf.path_to_backup if self.conf.lvm_auto_snap else '', + 'client_os': sys.platform, + 'client_version': self.conf.__version__ + } + fields = ['action', 'always_level', 'backup_media', 'backup_name', + 'container', 'container_segments', 'curr_backup_level', + 'dry_run', 'hostname', 'path_to_backup', 'max_level', + 'mode', 'meta_data_file', 'backup_name', 'hostname', + 'time_stamp', 'curr_backup_level'] + for field_name in fields: + metadata[field_name] = self.conf.__dict__.get(field_name, '') + return metadata + class RestoreJob(Job): @Job.executemethod diff --git a/freezer/main.py b/freezer/main.py index d2bf5142..b233214a 100644 --- a/freezer/main.py +++ b/freezer/main.py @@ -31,7 +31,7 @@ import os import subprocess import logging import sys - +import json # Initialize backup options (backup_args, arg_parse) = backup_arguments() @@ -125,6 +125,11 @@ def freezer_main(args={}): freezer_job = job.create_job(backup_args) freezer_job.execute() + if backup_args.metadata_out == '-': + metadata = freezer_job.get_metadata() + if metadata: + sys.stdout.write(json.dumps(metadata)) + return backup_args diff --git a/freezer/scheduler/__init__.py b/freezer/scheduler/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer/scheduler/arguments.py b/freezer/scheduler/arguments.py new file mode 100644 index 00000000..2ae91882 --- /dev/null +++ b/freezer/scheduler/arguments.py @@ -0,0 +1,170 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import argparse +from prettytable import PrettyTable + +SCHEDULER_CONF_D = '/etc/freezer/scheduler/conf.d' + + +class OpenstackOptions(object): + def __init__(self, args, default_dict={}): + self.username = args.os_username or\ + default_dict.get('OS_USERNAME', None) + self.tenant_name = args.os_tenant_name or\ + default_dict.get('OS_TENANT_NAME', None) + self.auth_url = args.os_auth_url or\ + default_dict.get('OS_AUTH_URL', None) + self.password = args.os_password or\ + default_dict.get('OS_PASSWORD', None) + self.tenant_id = args.os_tenant_id or\ + default_dict.get('OS_TENANT_ID', None) + self.region_name = args.os_region_name or\ + default_dict.get('OS_REGION_NAME', None) + self.endpoint = args.os_endpoint or\ + default_dict.get('OS_SERVICE_ENDPOINT', None) + if not self.is_valid(): + raise Exception('ERROR: OS Options not valid: {0}'. + format(self.reason())) + + def __str__(self): + table = PrettyTable(["variable", "value"]) + table.add_row(['username', self.username]) + table.add_row(['tenant_name', self.tenant_name]) + table.add_row(['auth_url', self.auth_url]) + table.add_row(['password', self.password]) + table.add_row(['tenant_id', self.tenant_id]) + table.add_row(['region_name', self.region_name]) + table.add_row(['endpoint', self.endpoint]) + return table.__str__() + + def is_valid(self): + if self.reason(): + return False + return True + + def reason(self): + missing = [] + for attr in ['username', 'password', 'tenant_name', 'region_name']: + if not self.__getattribute__(attr): + missing.append(attr) + if missing: + return 'missing {0}'.format(', '.join(missing)) + return '' + + +def get_args(choices): + arg_parser = argparse.ArgumentParser() + arg_parser.add_argument( + 'action', action='store', default=None, choices=choices, help='') + arg_parser.add_argument( + '--debug', action='store_true', + help='Prints debugging output onto the console, this may include ' + 'OS environment variables, request and response calls. ' + 'Helpful for debugging and understanding the API calls.', + dest='debug', default=False) + arg_parser.add_argument( + '-j', '--job', action='store', + help=('name or ID of the job'), + dest='job', default=None) + arg_parser.add_argument( + '-s', '--session', action='store', + help=('name or ID of the session'), + dest='session', default=None) + arg_parser.add_argument( + '--file', action='store', + help=('Local file that contains the resource ' + 'to be uploaded/downloaded'), + dest='fname', default=None) + arg_parser.add_argument( + '--os-endpoint', action='store', + help=('Specify an endpoint to use instead of retrieving ' + 'one from the service catalog (via authentication). ' + 'Defaults to env[OS_SERVICE_ENDPOINT]'), + dest='os_endpoint', default=None) + arg_parser.add_argument( + '--os-username', action='store', + help=('Name used for authentication with the OpenStack ' + 'Identity service. Defaults to env[OS_USERNAME].'), + dest='os_username', default=None) + arg_parser.add_argument( + '--os-password', action='store', + help=('Password used for authentication with the OpenStack ' + 'Identity service. Defaults to env[OS_PASSWORD].'), + dest='os_password', default=None) + arg_parser.add_argument( + '--os-tenant-name', action='store', + help=('Tenant to request authorization on. Defaults to ' + 'env[OS_TENANT_NAME].'), + dest='os_tenant_name', default=None) + arg_parser.add_argument( + '--os-tenant-id', action='store', + help=('Tenant to request authorization on. Defaults to ' + 'env[OS_TENANT_ID].'), + dest='os_tenant_id', default=None) + arg_parser.add_argument( + '--os-auth-url', action='store', + help=('Specify the Identity endpoint to use for ' + 'authentication. Defaults to env[OS_AUTH_URL].'), + dest='os_auth_url', default=None) + arg_parser.add_argument( + '--os-region-name', action='store', + help=('Specify the region to use. Defaults to ' + 'env[OS_REGION_NAME].'), + dest='os_region_name', default=None) + arg_parser.add_argument( + '--os-token', action='store', + help=('Specify an existing token to use instead of retrieving' + ' one via authentication (e.g. with username & password). ' + 'Defaults to env[OS_SERVICE_TOKEN].'), + dest='os_token', default=None) + arg_parser.add_argument( + '-c', '--client-id', action='store', + help=('Specifies the client_id used when contacting the service.' + 'If not specified it will be automatically created' + 'using the tenant-id and the machine hostname.'), + dest='client_id', default=None) + + arg_parser.add_argument( + '-n', '--no-api', action='store_true', + help='Prevents the scheduler from using the api service', + dest='no_api', default=False) + arg_parser.add_argument( + '-a', '--active-only', action='store_true', + help='Filter only active jobs/session', + dest='active_only', default=False) + arg_parser.add_argument( + '-d', '--dir', action='store', + help=('Used to store/retrieve files on local storage, including ' + 'those exchanged with the api service. ' + 'Default value is {0}'.format(SCHEDULER_CONF_D)), + dest='jobs_dir', default=SCHEDULER_CONF_D) + arg_parser.add_argument( + '-i', '--interval', action='store', + help=('Specifies the api-polling interval in seconds.' + 'Defaults to 60 seconds'), + dest='interval', default=60) + + arg_parser.add_argument( + '-l', '--log-file', action='store', + help=('location of log file'), + dest='log_file', default=None) + return arg_parser.parse_args() diff --git a/freezer/scheduler/daemon.py b/freezer/scheduler/daemon.py new file mode 100644 index 00000000..7d60d484 --- /dev/null +++ b/freezer/scheduler/daemon.py @@ -0,0 +1,160 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import logging +import os +import signal +from tempfile import gettempdir +from time import sleep + +from pep3143daemon import DaemonContext, PidFile + +from freezer.utils import create_dir + + +class Daemon: + """ + A class to manage all the daemon-related stuff + + """ + + instance = None + exit_flag = False + + def __init__(self, daemonizable=None, pid_fname=None): + # daemonizable has to provide start/stop (and possibly reload) methods + Daemon.instance = self + self._pid_fname = pid_fname + self.daemonizable = daemonizable + + @staticmethod + def setup_logging(log_file): + + class NoLogFilter(logging.Filter): + def filter(self, record): + return False + + def configure_logging(file_name): + expanded_file_name = os.path.expanduser(file_name) + expanded_dir_name = os.path.dirname(expanded_file_name) + create_dir(expanded_dir_name, do_log=False) + logging.basicConfig( + filename=expanded_file_name, + level=logging.INFO, + format=('%(asctime)s %(name)s %(levelname)s %(message)s')) + # filter out some annoying messages + # not the best position for this code + log_filter = NoLogFilter() + logging.getLogger("apscheduler.scheduler").\ + addFilter(log_filter) + logging.getLogger("apscheduler.executors.default").\ + addFilter(log_filter) + logging.getLogger("requests.packages.urllib3.connectionpool").\ + addFilter(log_filter) + return expanded_file_name + + log_file_paths = [log_file] if log_file else [ + '/var/log/freezer-scheduler.log', + '~/.freezer/freezer-scheduler.log'] + for file_name in log_file_paths: + try: + return configure_logging(file_name) + except IOError: + pass + + raise Exception("Unable to write to log file") + + @staticmethod + def handle_program_exit(signum, frame): + Daemon.exit_flag = True + Daemon.instance.daemonizable.stop() + + @staticmethod + def handle_reload(signum, frame): + Daemon.instance.daemonizable.reload() + + @property + def signal_map(self): + return { + signal.SIGTERM: Daemon.handle_program_exit, + signal.SIGHUP: Daemon.handle_reload, + } + + @property + def pid_fname(self): + if not self._pid_fname: + fname = '{0}/freezer_sched_{1}.pid'.format( + gettempdir(), + os.path.split(os.path.expanduser('~'))[-1]) + self._pid_fname = os.path.normpath(fname) + return self._pid_fname + + @property + def pid(self): + if os.path.isfile(self.pid_fname): + with open(self.pid_fname, 'r') as f: + return int(f.read()) + return None + + @property + def jobs_file(self): + return '' + + @property + def no_api(self): + return False + + def start(self, log_file=None): + pidfile = PidFile(self.pid_fname) + with DaemonContext(pidfile=pidfile, signal_map=self.signal_map): + self.setup_logging(log_file) + while not Daemon.exit_flag: + try: + logging.info('[*] freezer daemon starting, pid: {0}'. + format(self.pid)) + self.daemonizable.start() + Daemon.exit_flag = True + except Exception as e: + logging.error('[*] Restarting daemonized procedure ' + 'after Fatal Error: {0}'.format(e)) + sleep(10) + logging.info('[*] freezer daemon done, pid: {0}'.format(self.pid)) + + def stop(self): + pid = self.pid + if pid: + os.kill(self.pid, signal.SIGTERM) + else: + print "Not Running" + + def status(self): + pid = self.pid + if pid: + print "Running with pid: {0}".format(pid) + else: + print "Not Running" + + def reload(self): + pid = self.pid + if pid: + os.kill(pid, signal.SIGHUP) + else: + print "Not Running" diff --git a/freezer/scheduler/freezer_scheduler.py b/freezer/scheduler/freezer_scheduler.py new file mode 100755 index 00000000..032070a7 --- /dev/null +++ b/freezer/scheduler/freezer_scheduler.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import logging +import os +import sys +import threading +from distutils import spawn + +from apscheduler.schedulers.blocking import BlockingScheduler + +from freezer.apiclient import client + +import arguments +import shell +import utils +from freezer.utils import create_dir + +from daemon import Daemon +from scheduler_job import Job + + +class FreezerScheduler(object): + def __init__(self, apiclient, interval, job_path): + # config_manager + self.client = apiclient + self.freezerc_executable = spawn.find_executable('freezerc') + self.job_path = job_path + self._client = None + self.lock = threading.Lock() + self.execution_lock = threading.Lock() + job_defaults = { + 'coalesce': True, + 'max_instances': 1 + } + self.scheduler = BlockingScheduler(job_defaults=job_defaults) + if self.client: + self.scheduler.add_job(self.poll, 'interval', + seconds=interval, id='api_poll') + + self.add_job = self.scheduler.add_job + self.remove_job = self.scheduler.remove_job + self.jobs = {} + + def get_jobs(self): + if self.client: + job_doc_list = [] + try: + job_doc_list = utils.get_active_jobs_from_api(self.client) + except Exception as e: + logging.error('Unable to get jobs from freezer api service. ' + '{0}'.format(e)) + try: + utils.save_jobs_to_disk(job_doc_list, self.job_path) + except Exception as e: + logging.error('Unable to save jobs to {0}. ' + '{1}'.format(self.job_path, e)) + return job_doc_list + else: + return utils.get_jobs_from_disk(self.job_path) + + def start_session(self, session_id, job_id, session_tag): + if self.client: + return self.client.sessions.start_session(session_id, + job_id, + session_tag) + else: + raise Exception("Unable to start session: api not in use.") + + def end_session(self, session_id, job_id, session_tag, result): + if self.client: + return self.client.sessions.end_session(session_id, + job_id, + session_tag, + result) + else: + raise Exception("Unable to end session: api not in use.") + + def upload_metadata(self, metadata_doc): + if self.client: + self.client.backups.create(metadata_doc) + + def start(self): + utils.do_register(self.client) + self.poll() + self.scheduler.start() + + def update_job(self, job_id, job_doc): + if self.client: + try: + return self.client.jobs.update(job_id, job_doc) + except Exception as e: + logging.error("[*] Job update error: {0}".format(e)) + + def update_job_status(self, job_id, status): + doc = {'job_schedule': {'status': status}} + self.update_job(job_id, doc) + + def is_scheduled(self, job_id): + return self.scheduler.get_job(job_id) is not None + + def create_job(self, job_doc): + job = Job.create(self, self.freezerc_executable, job_doc) + if job: + self.jobs[job.id] = job + logging.info("Created job {0}".format(job.id)) + return job + + def poll(self): + work_job_doc_list = [] + try: + work_job_doc_list = self.get_jobs() + except Exception as e: + logging.error("[*] Unable to get jobs: {0}".format(e)) + + work_job_id_list = [] + + # create job if necessary, then let it process its events + for job_doc in work_job_doc_list: + job_id = job_doc['job_id'] + work_job_id_list.append(job_id) + job = self.jobs.get(job_id, None) or self.create_job(job_doc) + if job: + job.process_event(job_doc) + + # request removal of any job that has been removed in the api + for job_id, job in self.jobs.iteritems(): + if job_id not in work_job_id_list: + job.remove() + + remove_list = [job_id for job_id, job in self.jobs.items() + if job.can_be_removed()] + + for k in remove_list: + self.jobs.pop(k) + + def stop(self): + try: + self.scheduler.shutdown(wait=False) + except: + pass + + def reload(self): + logging.warning("reload not supported") + + +def _get_doers(module): + doers = {} + for attr in (a for a in dir(module) if a.startswith('do_')): + command = attr[3:].replace('_', '-') + callback = getattr(module, attr) + doers[command] = callback + return doers + + +def main(): + doers = _get_doers(shell) + doers.update(_get_doers(utils)) + + possible_actions = doers.keys() + ['start', 'stop', 'status'] + + args = arguments.get_args(possible_actions) + + if args.action is None: + print "No action" + sys.exit(1) + + os_options = arguments.OpenstackOptions(args, os.environ) + if args.debug: + print os_options + + apiclient = None + if args.no_api is False: + apiclient = client.Client(username=os_options.username, + password=os_options.password, + tenant_name=os_options.tenant_name, + endpoint=os_options.endpoint, + auth_url=os_options.auth_url) + if args.client_id: + apiclient.client_id = args.client_id + + create_dir(args.jobs_dir, do_log=False) + + if args.action in doers: + try: + return doers[args.action](apiclient, args) + except Exception as e: + print ('ERROR {0}'.format(e)) + return 1 + + freezer_scheduler = FreezerScheduler(apiclient=apiclient, + interval=int(args.interval), + job_path=args.jobs_dir) + + daemon = Daemon(daemonizable=freezer_scheduler) + + if args.action == 'start': + daemon.start(log_file=args.log_file) + elif args.action == 'stop': + daemon.stop() + elif args.action == 'reload': + daemon.reload() + elif args.action == 'status': + daemon.status() + + return os.EX_OK + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/freezer/scheduler/scheduler_job.py b/freezer/scheduler/scheduler_job.py new file mode 100644 index 00000000..0f757582 --- /dev/null +++ b/freezer/scheduler/scheduler_job.py @@ -0,0 +1,440 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import logging +import subprocess +import tempfile +import datetime +import json +import time + +from ConfigParser import ConfigParser + + +class StopState(object): + + @staticmethod + def stop(job, doc): + job.job_doc = doc + job.event = Job.NO_EVENT + job.job_doc_status = Job.STOP_STATUS + job.scheduler.update_job(job.id, job.job_doc) + return Job.NO_EVENT + + @staticmethod + def abort(job, doc): + job.job_doc = doc + job.event = Job.NO_EVENT + job.job_doc_status = Job.STOP_STATUS + job.scheduler.update_job(job.id, job.job_doc) + return Job.NO_EVENT + + @staticmethod + def start(job, doc): + job.job_doc = doc + job.event = Job.NO_EVENT + job.job_doc_status = Job.STOP_STATUS + job.schedule() + job.scheduler.update_job(job.id, job.job_doc) + return Job.NO_EVENT + + @staticmethod + def remove(job): + job.unschedule() + job.job_doc_status = Job.REMOVED_STATUS + return Job.NO_EVENT + + +class ScheduledState(object): + + @staticmethod + def stop(job, doc): + job.unschedule() + job.scheduler.update_job(job.id, job.job_doc) + return Job.STOP_EVENT + + @staticmethod + def abort(job, doc): + return ScheduledState.stop(job, doc) + + @staticmethod + def start(job, doc): + job.event = Job.NO_EVENT + job.scheduler.update_job(job.id, job.job_doc) + return Job.NO_EVENT + + @staticmethod + def remove(job): + job.unschedule() + job.job_doc_status = Job.REMOVED_STATUS + return Job.NO_EVENT + + +class RunningState(object): + + @staticmethod + def stop(job, doc): + job.event = Job.STOP_EVENT + return Job.NO_EVENT + + @staticmethod + def abort(job, doc): + job.event = Job.ABORT_EVENT + return Job.NO_EVENT + + @staticmethod + def start(job, doc): + job.event = Job.NO_EVENT + job.scheduler.update_job(job.id, job.job_doc) + return Job.NO_EVENT + + @staticmethod + def remove(job): + job.event = Job.REMOVE_EVENT + return Job.NO_EVENT + + +class Job(object): + + NO_EVENT = '' + STOP_EVENT = 'stop' + START_EVENT = 'start' + ABORT_EVENT = 'abort' + REMOVE_EVENT = 'remove' + + STOP_STATUS = 'stop' + SCHEDULED_STATUS = 'scheduled' + RUNNING_STATUS = 'running' + REMOVED_STATUS = 'removed' + COMPLETED_STATUS = 'completed' + + FAIL_RESULT = 'fail' + SUCCESS_RESULT = 'success' + ABORTED_RESULT = 'aborted' + + @staticmethod + def create(scheduler, executable, job_doc): + job = Job(scheduler, executable, job_doc) + if job.job_doc_status in ['running', 'scheduled']: + logging.warning('Job {0} already has {1} status, ' + 'skipping'.format(job.id, job.job_doc_status)) + return None + if not job.event and not job.job_doc_status: + logging.warning('Autostart Job {0}'.format(job.id)) + job.event = Job.START_EVENT + return job + + def __init__(self, scheduler, executable, job_doc): + self.scheduler = scheduler + self.executable = executable + self.job_doc = job_doc + self.process = None + self.state = StopState + + def remove(self): + with self.scheduler.lock: + # delegate to state object + logging.info('REMOVE job {0}'.format(self.id)) + self.state.remove(self) + + @property + def id(self): + return self.job_doc['job_id'] + + @property + def session_id(self): + return self.job_doc.get('session_id', '') + + @session_id.setter + def session_id(self, value): + self.job_doc['session_id'] = value + + @property + def session_tag(self): + return self.job_doc.get('session_tag', 0) + + @session_tag.setter + def session_tag(self, value): + self.job_doc['session_tag'] = value + + @property + def event(self): + return self.job_doc['job_schedule'].get('event', '') + + @event.setter + def event(self, value): + self.job_doc['job_schedule']['event'] = value + + @property + def job_doc_status(self): + return self.job_doc['job_schedule'].get('status', '') + + @job_doc_status.setter + def job_doc_status(self, value): + self.job_doc['job_schedule']['status'] = value + + @property + def result(self): + return self.job_doc['job_schedule'].get('result', '') + + @result.setter + def result(self, value): + self.job_doc['job_schedule']['result'] = value + + def can_be_removed(self): + return self.job_doc_status == Job.REMOVED_STATUS + + @staticmethod + def save_action_to_file(action, f): + parser = ConfigParser() + parser.add_section('action') + for action_k, action_v in action.items(): + parser.set('action', action_k, action_v) + parser.write(f) + f.seek(0) + + @property + def schedule_date(self): + return self.job_doc['job_schedule'].get('schedule_date', '') + + @property + def schedule_interval(self): + return self.job_doc['job_schedule'].get('schedule_interval', '') + + @property + def schedule_cron_fields(self): + cron_fields = ['year', 'month', 'day', 'week', 'day_of_week', + 'hour', 'minute', 'second'] + return {key: value + for key, value in self.job_doc['job_schedule'].items() + if key in cron_fields} + + @property + def scheduled(self): + return self.scheduler.is_scheduled(self.id) + + def get_schedule_args(self): + if self.schedule_date: + return {'trigger': 'date', + 'run_date': self.schedule_date} + elif self.schedule_interval: + kwargs = {'trigger': 'interval'} + if self.schedule_interval == 'continuous': + kwargs.update({'seconds': 1}) + else: + val, unit = self.schedule_interval.split(' ') + kwargs.update({unit: int(val)}) + return kwargs + else: + cron_fields = self.schedule_cron_fields + if cron_fields: + return {'trigger': 'cron'}.update(cron_fields) + # no scheduling information, schedule to start within a few seconds + return {'trigger': 'date', + 'run_date': datetime.datetime.now() + + datetime.timedelta(0, 2, 0)} + + def process_event(self, job_doc): + with self.scheduler.lock: + next_event = job_doc['job_schedule'].get('event', '') + while next_event: + if next_event == Job.STOP_EVENT: + logging.info('JOB {0} event: STOP'.format(self.id)) + next_event = self.state.stop(self, job_doc) + elif next_event == Job.START_EVENT: + logging.info('JOB {0} event: START'.format(self.id)) + next_event = self.state.start(self, job_doc) + elif next_event == Job.ABORT_EVENT: + logging.info('JOB {0} event: ABORT'.format(self.id)) + next_event = self.state.abort(self, job_doc) + + def upload_metadata(self, metadata_string): + try: + metadata = json.loads(metadata_string) + if metadata: + self.scheduler.upload_metadata(metadata) + except Exception as e: + logging.error('[*] metrics upload error: {0}'.format(e)) + logging.info("[*] Job {0}, freezer action metadata uploaded") + + def execute_job_action(self, job_action): + max_retries = job_action.get('max_retries', 1) + tries = max_retries + freezer_action = job_action.get('freezer_action', {}) + max_retries_interval = job_action.get('max_retries_interval', 60) + action_name = freezer_action.get('action', '') + while tries: + + with tempfile.NamedTemporaryFile() as config_file: + self.save_action_to_file(freezer_action, config_file) + + freezer_command = 'python {0} --metadata-out - --config {1}'.\ + format(self.executable, config_file.name) + self.process = subprocess.Popen(freezer_command.split(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + output, error = self.process.communicate() + + if error: + logging.error("[*] Freezer client error: {0}".format(error)) + elif output: + self.upload_metadata(output) + + if self.process.returncode: + # ERROR + tries -= 1 + if tries: + logging.warning('[*] Job {0} failed {1} action,' + ' retrying in {2} seconds' + .format(self.id, action_name, + max_retries_interval)) + # sleeping with the bloody lock, but we don't want other + # actions to mess with our stuff like fs snapshots, do we ? + time.sleep(max_retries_interval) + else: + # SUCCESS + logging.info('[*] Job {0} action {1}' + ' returned success exit code'. + format(self.id, action_name)) + return Job.SUCCESS_RESULT + logging.error('[*] Job {0} action {1} failed after {2} tries' + .format(self.id, action_name, max_retries)) + return Job.FAIL_RESULT + + def execute(self): + result = Job.SUCCESS_RESULT + with self.scheduler.execution_lock: + with self.scheduler.lock: + logging.info('job {0} running'.format(self.id)) + self.state = RunningState + self.job_doc_status = Job.RUNNING_STATUS + self.scheduler.update_job_status(self.id, self.job_doc_status) + + self.start_session() + + for job_action in self.job_doc.get('job_actions', []): + if job_action.get('mandatory', False) or\ + (result == Job.SUCCESS_RESULT): + result = self.execute_job_action(job_action) + else: + freezer_action = job_action.get('freezer_action', {}) + action_name = freezer_action.get('action', '') + logging.warning("[*]skipping {0} action". + format(action_name)) + self.result = result + self.finish() + + def finish(self): + self.end_session(self.result) + with self.scheduler.lock: + if self.event == Job.REMOVE_EVENT: + self.unschedule() + self.job_doc_status = Job.REMOVED_STATUS + return + + if not self.scheduled: + self.job_doc_status = Job.COMPLETED_STATUS + self.state = StopState + self.scheduler.update_job(self.id, self.job_doc) + return + + if self.event in [Job.STOP_EVENT, Job.ABORT_EVENT]: + self.unschedule() + self.job_doc_status = Job.COMPLETED_STATUS + self.scheduler.update_job(self.id, self.job_doc) + else: + self.job_doc_status = Job.SCHEDULED_STATUS + self.state = ScheduledState + self.scheduler.update_job_status(self.id, self.job_doc_status) + + def start_session(self): + if not self.session_id: + return + retry = 5 + while retry: + try: + resp = self.scheduler.start_session(self.session_id, + self.id, + self.session_tag) + if resp['result'] == 'success': + self.session_tag = resp['session_tag'] + return + except Exception as e: + logging.error('[*]Error while starting session {0}. {1}'. + format(self.session_id, e)) + logging.warning('[*]Retrying to start session {0}'. + format(self.session_id)) + retry -= 1 + logging.error('[*]Unable to start session {0}'.format(self.session_id)) + + def end_session(self, result): + if not self.session_id: + return + retry = 5 + while retry: + try: + resp = self.scheduler.end_session(self.session_id, + self.id, + self.session_tag, + result) + if resp['result'] == 'success': + return + except Exception as e: + logging.error('[*]Error while ending session {0}. {1}'. + format(self.session_id, e)) + logging.warning('[*]Retrying to end session {0}'. + format(self.session_id)) + retry -= 1 + logging.error('[*]Unable to end session {0}'.format(self.session_id)) + + def schedule(self): + try: + kwargs = self.get_schedule_args() + self.scheduler.add_job(self.execute, id=self.id, **kwargs) + except Exception as e: + logging.error("[*] Unable to schedule job {0}: {1}". + format(self.id, e)) + + if self.scheduled: + self.job_doc_status = Job.SCHEDULED_STATUS + self.state = ScheduledState + else: + # job not scheduled or already started and waiting for lock + self.job_doc_status = Job.COMPLETED_STATUS + self.state = StopState + + def unschedule(self): + try: + # already executing job are not present in the apscheduler list + self.scheduler.remove_job(job_id=self.id) + except: + pass + self.event = Job.NO_EVENT + self.job_doc_status = Job.STOP_STATUS + self.state = StopState + + def terminate(self): + if self.process: + self.process.terminate() + + def kill(self): + if self.process: + self.process.kill() diff --git a/freezer/scheduler/shell.py b/freezer/scheduler/shell.py new file mode 100644 index 00000000..2213c254 --- /dev/null +++ b/freezer/scheduler/shell.py @@ -0,0 +1,236 @@ +#!/usr/bin/env python +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import os +import utils +from freezer.utils import create_dir + +from prettytable import PrettyTable + +try: + from betterprint import pprint +except: + def pprint(doc): + print json.dumps(doc, indent=4) + + +def do_session_remove_job(client, args): + """ + remove the job from the session + """ + if not args.session_id: + raise Exception("Parameter --session required") + if not args.job: + raise Exception("Parameter --job required") + client.sessions.remove_job(args.session_id, args.job) + + +def do_session_add_job(client, args): + """ + adds a job to the session + """ + if not args.session_id: + raise Exception("Parameter --session required") + if not args.job: + raise Exception("Parameter --job required") + for job_id in args.job.split(','): + try: + client.sessions.add_job(args.session_id, job_id) + except Exception as e: + print "Error processin job {0}. {1}".format(job_id, e) + + +def do_session_list_job(client, args): + """ + prints a list of jobs assigned to the specific session + + :return: None + """ + if not args.session_id: + raise Exception("Parameter --session required") + session_doc = client.sessions.get(args.session_id) + jobs = session_doc.get('jobs', {}) + table = PrettyTable(["job_id", "status", "result", "client_id"]) + for job_id, job_data in jobs.iteritems(): + table.add_row([job_id, + job_data['status'], + job_data['result'], + job_data['client_id'] + ]) + print table + + +def do_session_delete(client, args): + if not args.session_id: + raise Exception("Parameter --session required") + client.sessions.delete(args.session_id) + + +def do_session_create(client, args): + """ + creates a session object loading it from disk + + :return: None + """ + if not args.fname: + raise Exception("Parameter --file required") + session_doc = utils.load_doc_from_json_file(args.fname) + session_id = client.sessions.create(session_doc) + print "Created session {0}".format(session_id) + + +def do_session_get(client, args): + """ + gets a specific session object and saves it to file. If file is not + specified the session obj is printed. + :return: None + """ + if not args.session_id: + raise Exception("Parameter --session required") + session_doc = client.sessions.get(args.session_id) + if args.fname: + utils.save_doc_to_json_file(session_doc, args.fname) + else: + pprint(session_doc) + + +def do_session_list(client, args): + """ + print a list of all jobs + + :return: None + """ + table = PrettyTable(["session_id", "tag", "status", + "description", "jobs", "last_start"]) + l = client.sessions.list() + while l: + offset = len(l) + for doc in l: + table.add_row([doc['session_id'], + doc['session_tag'], + doc['status'], + doc.get('description', ''), + len(doc.get('jobs', [])), + doc['last_start']]) + l = client.sessions.list(offset=offset) + print table + + +def do_job_create(client, args): + if not args.fname: + raise Exception("Parameter --file required") + job_doc = utils.load_doc_from_json_file(args.fname) + job_id = client.jobs.create(job_doc) + print "Created job {0}".format(job_id) + + +def do_job_delete(client, args): + if not args.job: + raise Exception("Parameter --job required") + client.jobs.delete(args.job) + + +def do_job_get(client, args): + if not args.job: + raise Exception("Parameter --job required") + job_doc = client.jobs.get(args.job) + if args.fname: + args.save_doc_to_file(job_doc, args.fname) + else: + pprint(job_doc) + + +def do_job_start(client, args): + if not args.job: + raise Exception("Parameter --job required") + client.jobs.start_job(args.job) + print "Job {0} started".format(args.job) + + +def do_job_stop(client, args): + if not args.job: + raise Exception("Parameter --job required") + client.jobs.stop_job(args.job) + print "Job {0} stopped".format(args.job) + + +def do_job_download(client, args): + create_dir(args.jobs_dir, do_log=True) + for doc in _job_list(client, args): + fname = os.path.normpath('{0}/job_{1}.conf'. + format(args.jobs_dir, doc['job_id'])) + try: + utils.save_doc_to_json_file(doc, fname) + except: + print "Unable to write to file {0}".format(fname) + + +def do_job_upload(client, args): + for job_doc in utils.get_jobs_from_disk(args.jobs_dir): + job_id = client.jobs.create(job_doc) + print "Uploaded job {0}".format(job_id) + + +def _job_list(client, args): + search = {} + if args.active_only: + search = {"match_not": [{"status": "completed"}]} + l = client.jobs.list(search=search) + while l: + offset = len(l) + for doc in l: + yield doc + l = client.jobs.list(offset=offset, search=search) + raise StopIteration + + +def do_job_list(client, args): + table = PrettyTable(["job_id", "description", "# actions", + "status", "event", "result", "session_id"]) + for doc in _job_list(client, args): + job_scheduling = doc.get('job_schedule', {}) + job_status = job_scheduling.get('status', '') + job_event = job_scheduling.get('event', '') + job_result = job_scheduling.get('result', '') + table.add_row([doc['job_id'], + doc.get('description', ''), + len(doc.get('job_actions', [])), + job_status, + job_event, + job_result, + doc.get('session_id', '')]) + print table + + +def do_client_list(client, args): + table = PrettyTable(["client_id", "hostname", "description"]) + l = client.registration.list() + while l: + offset = len(l) + for doc in l: + client_doc = doc['client'] + table.add_row([client_doc['client_id'], + client_doc.get('hostname', ''), + client_doc.get('description', '')]) + l = client.registration.list(offset=offset) + print table diff --git a/freezer/scheduler/utils.py b/freezer/scheduler/utils.py new file mode 100644 index 00000000..dc2a6624 --- /dev/null +++ b/freezer/scheduler/utils.py @@ -0,0 +1,100 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import os +import socket +import freezer.apiclient.exceptions + +CONFIG_FILE_EXT = '.conf' + + +def do_register(client, args=None): + if client: + client_info = { + "client_id": client.client_id, + "hostname": socket.gethostname() + } + try: + client.registration.create(client_info) + except freezer.apiclient.exceptions.ApiClientException as e: + if e.status_code == 409: + print "Client already registered" + return os.EX_CANTCREAT + return os.EX_OK + + +def find_config_files(path): + expanded_path = os.path.expanduser(path) + if os.path.isfile(expanded_path): + return [expanded_path] + file_list = [] + if os.path.isdir(expanded_path): + for fname in os.walk(expanded_path).next()[2]: + if CONFIG_FILE_EXT.upper() == os.path.splitext(fname)[1].upper(): + file_list.append('{0}/{1}'.format(expanded_path, fname)) + return file_list + raise Exception("unable to find job files at the provided path " + "{0}".format(path)) + + +def load_doc_from_json_file(fname, debug=False): + with open(fname, 'rb') as fd: + try: + doc = json.load(fd) + except Exception as e: + raise Exception("Unable to load conf file. {0}".format(e)) + if debug: + print "File {0} loaded: ".format(fname) + return doc + + +def save_doc_to_json_file(doc, fname, debug=False): + with open(fname, 'w') as fd: + json.dump(doc, fd, indent=4) + if debug: + print 'Saved doc to file: {0}'.format(fname) + + +def get_jobs_from_disk(path): + job_doc_list = [ + load_doc_from_json_file(f) for f in find_config_files(path)] + return [x for x in job_doc_list if x] + + +def save_jobs_to_disk(job_doc_list, path): + for doc in job_doc_list: + fname = os.path.normpath('{0}/job_{1}.conf'. + format(path, doc['job_id'])) + save_doc_to_json_file(doc, fname) + + +def get_active_jobs_from_api(client): + # might raise + search = {"match_not": [{"status": "completed"}]} + job_list, offset = [], 0 + while True: + jobs = client.jobs.list(limit=10, offset=offset, search=search) + job_list.extend(jobs) + if len(jobs) < 10: + break + offset += len(jobs) + return job_list diff --git a/freezer/swift.py b/freezer/swift.py index e8fceedb..0f476fb8 100644 --- a/freezer/swift.py +++ b/freezer/swift.py @@ -451,12 +451,9 @@ def get_newest_backup(hostname, backup_name, remote_match_backup): raise Exception("remote match backups are empty") sorted_backups_list = sort_backup_list(remote_match_backup) - print sorted_backups_list - for remote_obj in sorted_backups_list: obj_name_match = re.search(r'^{0}_({1})_(\d+)_\d+?$'.format( hostname, backup_name), remote_obj, re.I) - print obj_name_match if not obj_name_match: continue tar_metadata_obj = 'tar_metadata_{0}'.format(remote_obj) diff --git a/freezer_api/README.rst b/freezer_api/README.rst index 4dff7d86..3e4a16c4 100644 --- a/freezer_api/README.rst +++ b/freezer_api/README.rst @@ -3,44 +3,56 @@ Freezer API =========== -Installation -============ +1. Installation +=============== + +1.1 Install required packages +----------------------------- +:: -Install required packages -------------------------- # pip install keystonemiddleware falcon -Elasticsearch support:: +Elasticsearch support +:: + # pip install elasticsearch -Install freezer_api -------------------- +1.2 Install freezer_api +----------------------- +:: + # git clone https://github.com/stackforge/freezer.git # cd freezer/freezer_api && sudo python setup.py install this will install into /usr/local -edit config file ----------------- +1.3 edit config file +-------------------- +:: + # sudo vi /etc/freezer-api.conf -run simple instance -------------------- +1.4 run simple instance +----------------------- +:: + # freezer-api -examples running using uwsgi ----------------------------- +1.5 examples running using uwsgi +-------------------------------- +:: + # uwsgi --http :9090 --need-app --master --module freezer_api.cmd.api:application # uwsgi --https :9090,foobar.crt,foobar.key --need-app --master --module freezer_api.cmd.api:application -Concepts and definitions -======================== +2. Concepts and definitions +=========================== *hostname* is _probably_ going to be the host fqdn. @@ -52,145 +64,176 @@ identifies a backup defined as "container_hostname_backupname" identifies a group of related backups which share the same container,hostname and backupname -*backup_session* -is a group of backups which share container,hostname and backupname, but -are also related by dependency. -*backup_session_id* -utilizes the timestamp of the first (level 0) backup in the session -It is identified by (container, hostname, backupname, timestamp-of-level-0) +3. API registration +=================== +:: + + keystone user-create --name freezer --pass FREEZER_PWD + keystone user-role-add --user freezer --tenant service --role admin + + keystone service-create --name freezer --type backup \ + --description "Freezer Backup Service" + + keystone endpoint-create \ + --service-id $(keystone service-list | awk '/ backup / {print $2}') \ + --publicurl http://freezer_api_publicurl:port \ + --internalurl http://freezer_api_internalurl:port \ + --adminurl http://freezer_api_adminurl:port \ + --region regionOne -API registration -================ -keystone user-create --name freezer --pass FREEZER_PWD -keystone user-role-add --user freezer --tenant service --role admin - -keystone service-create --name freezer --type backup \ - --description "Freezer Backup Service" - -keystone endpoint-create \ - --service-id $(keystone service-list | awk '/ backup / {print $2}') \ - --publicurl http://freezer_api_publicurl:port \ - --internalurl http://freezer_api_internalurl:port \ - --adminurl http://freezer_api_adminurl:port \ - --region regionOne - - -API routes -========== +4. API routes +============= General ------- -GET / List API version -GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03 +:: + + GET / List API version + GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03 Backup metadata --------------- -GET /v1/backups(?limit,marker) Lists backups -POST /v1/backups Creates backup entry +:: -GET /v1/backups/{backup_id} Get backup details -UPDATE /v1/backups/{backup_id} Updates the specified backup -DELETE /v1/backups/{backup_id} Deletes the specified backup + GET /v1/backups(?limit,marker) Lists backups + POST /v1/backups Creates backup entry + + GET /v1/backups/{backup_id} Get backup details + UPDATE /v1/backups/{backup_id} Updates the specified backup + DELETE /v1/backups/{backup_id} Deletes the specified backup Freezer clients management -------------------------- -GET /v1/clients(?limit,offset) Lists registered clients -POST /v1/clients Creates client entry +:: -GET /v1/clients/{freezerc_id} Get client details -UPDATE /v1/clients/{freezerc_id} Updates the specified client information -DELETE /v1/clients/{freezerc_id} Deletes the specified client information + GET /v1/clients(?limit,offset) Lists registered clients + POST /v1/clients Creates client entry + + GET /v1/clients/{freezerc_id} Get client details + UPDATE /v1/clients/{freezerc_id} Updates the specified client information + DELETE /v1/clients/{freezerc_id} Deletes the specified client information Freezer jobs management ----------------------- -GET /v1/jobs(?limit,offset) Lists registered jobs -POST /v1/jobs Creates job entry +:: -GET /v1/jobs/{jobs_id} Get job details -POST /v1/jobs/{jobs_id} creates or replaces a job entry using the specified job_id -UPDATE /v1/jobs/{jobs_id} Updates the existing job information -DELETE /v1/jobs/{jobs_id} Deletes the specified job information -PATCH /v1/jobs/{jobs_id} Updates part of the document + GET /v1/jobs(?limit,offset) Lists registered jobs + POST /v1/jobs Creates job entry -Data Structures -=============== + GET /v1/jobs/{jobs_id} Get job details + POST /v1/jobs/{jobs_id} creates or replaces a job entry using the specified job_id + DELETE /v1/jobs/{jobs_id} Deletes the specified job information + PATCH /v1/jobs/{jobs_id} Updates part of the document -Backup metadata structure -------------------------- +Freezer actions management +-------------------------- +:: + + GET /v1/actions(?limit,offset) Lists registered action + POST /v1/actions Creates action entry + + GET /v1/actions/{actions_id} Get action details + POST /v1/actions/{actions_id} creates or replaces a action entry using the specified action_id + DELETE /v1/actions/{actions_id} Deletes the specified action information + PATCH /v1/actions/{actions_id} Updates part of the action document + +Freezer sessions management +--------------------------- +:: + + GET /v1/sessions(?limit,offset) Lists registered session + POST /v1/sessions Creates session entry + + GET /v1/sessions/{sessions_id} Get session details + POST /v1/sessions/{sessions_id} creates or replaces a session entry using the specified session_id + DELETE /v1/sessions/{sessions_id} Deletes the specified session information + PATCH /v1/sessions/{sessions_id} Updates part of the session document + + POST /v1/sessions/{sessions_id}/action requests actions (e.g. start/end) upon a specific session + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + +5. Backup metadata structure +============================ NOTE: sizes are in MB +:: -backup_metadata:= -{ - "container": string, - "host_name": string, # fqdn, client has to provide consistent information here ! - "backup_name": string, - "timestamp": int, - "level": int, - "backup_session": int, - "max_level": int, - "mode" : string, (fs mongo mysql) - "fs_real_path": string, - "vol_snap_path": string, - "total_broken_links" : int, - "total_fs_files" : int, - "total_directories" : int, - "backup_size_uncompressed" : int, - "backup_size_compressed" : int, - "total_backup_session_size" : int, - "compression_alg": string, (gzip bzip xz) - "encrypted": bool, - "client_os": string - "broken_links" : [string, string, string], - "excluded_files" : [string, string, string] - "cli": string, equivalent cli used when executing the backup ? - "version": string -} + backup_metadata:= + { + "container": string, + "host_name": string, # fqdn, client has to provide consistent information here ! + "backup_name": string, + "timestamp": int, + "level": int, + "max_level": int, + "mode" : string, (fs mongo mysql) + "fs_real_path": string, + "vol_snap_path": string, + "total_broken_links" : int, + "total_fs_files" : int, + "total_directories" : int, + "backup_size_uncompressed" : int, + "backup_size_compressed" : int, + "compression_alg": string, (gzip bzip xz) + "encrypted": bool, + "client_os": string + "broken_links" : [string, string, string], + "excluded_files" : [string, string, string] + "cli": string, equivalent cli used when executing the backup ? + "version": string + } The api wraps backup_metadata dictionary with some additional information. It stores and returns the information provided in this form: -{ - "backup_id": string # container_hostname_backupname_timestamp_level - "user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided) - "user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided) +:: - "backup_metadata": { #--- actual backup_metadata provided - "container": string, - "host_name": string, - "backup_name": string, - "timestamp": int, - ... - } -} + { + "backup_id": string # container_hostname_backupname_timestamp_level + "user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided) + "user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided) + + "backup_metadata": { #--- actual backup_metadata provided + "container": string, + "host_name": string, + "backup_name": string, + "timestamp": int, + ... + } + } -Freezer Client document structure ---------------------------------- +6. Freezer Client document structure +==================================== + Identifies a freezer client for the purpose of sending action -# client_info document contains information relevant for client identification -client_info:= -{ - "client_id": string actually a concatenation "tenant-id_hostname" - "hostname": string - "description": string - "config_id": string # configuration in use by the client -} +client_info document contains information relevant for client identification:: + + client_info:= + { + "client_id": string actually a concatenation "tenant-id_hostname" + "hostname": string + "description": string + "uuid": + } -# client_type document embeds the client_info and adds user_id -client_type := -{ - "client" : client_info document, - "user_id": string, # owner of the information (OS X-User-Id, keystone provided, added by api) -} +client_type document embeds the client_info and adds user_id:: + + client_type := + { + "client" : client_info document, + "user_id": string, # owner of the information (OS X-User-Id, keystone provided, added by api) + } -Jobs ----- +7. Jobs +======= A job describes a single action to be executed by a freezer client, for example a backup, or a restore. It contains the necessary information as if they were provided on the command line. @@ -199,54 +242,75 @@ job_id, user_id, client_id, status, scheduling information etc Scheduling information enables future/recurrent execution of jobs -+---------------------+ -| Job | -| | job_action +-------------------+ -| +job_id +------------------>| job action dict | -| +client_id | +-------------------+ -| +user_id | -| +description | job_schedule -| +---------------+ -| | | +-------------------+ -+---------------------+ +-->| job schedule dict | - +-------------------+ +:: + + +---------------------+ + | Job | + +---------------------+ job_actions +--------------+ + | +---------------->| job_action | + | +job_id | 0..* +--------------+ freezer_action + | +client_id | | +mandatory |-------------+ + | +user_id | | +retries | | +----------------+ + | +description | job_schedule +--------------+ +->| freezer_action | + | +---------------+ +----------------+ + | | | +-------------------+ + +---------------------+ +-->| job schedule dict | + +-------------------+ -job document structure: +job document structure:: -"job": { - "job_action": { parameters for freezer to execute a specific action } - "job_schedule": { scheduling information } - "job_id": string - "client_id": string - "user_id": string - "description": string -} + "job": { + "job_action": { parameters for freezer to execute a specific action } + "job_schedule": { scheduling information } + "job_id": string + "client_id": string + "user_id": string + "description": string + } -"job_action": { -{ - "action" : string - "mode" : string - "src_file" : string - "backup_name" : string - "container" : string - ... -} + "job_actions": + [ + { + "freezer_action" : + { + "action" : string + "mode" : string + "src_file" : string + "backup_name" : string + "container" : string + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 + }, + { + "freezer_action" : + { + ... + }, + "mandatory": False, + "max_retries": 3, + "max_retry_interval": 60 -"job_schedule": { - "time_created": int (timestamp) - "time_started": int (timestamp) - "time_ended": int (timestamp) - "status": string ["stop", "scheduled", "running", "aborting", "removed"] - "event": string ["", "stop", "start", "abort", "remove"] - "result": string ["", "success", "fail", "aborted"] + } + ] - SCHEDULING TIME INFORMATION -} + "job_schedule": { + "time_created": int (timestamp) + "time_started": int (timestamp) + "time_ended": int (timestamp) + "status": string ["stop", "scheduled", "running", "aborting", "removed"] + "event": string ["", "stop", "start", "abort", "remove"] + "result": string ["", "success", "fail", "aborted"] + + SCHEDULING TIME INFORMATION + } -Scheduling Time Information ---------------------------- +7.1 Scheduling Time Information +------------------------------- Three types of scheduling can be identified: * date - used for single run jobs @@ -255,20 +319,24 @@ Three types of scheduling can be identified: Each type has specific parameters which can be given. -date scheduling ---------------- +7.1.1 date scheduling +--------------------- +:: "schedule_date": : datetime isoformat -interval scheduling -------------------- +7.1.2 interval scheduling +------------------------- +:: + "schedule_interval" : "continuous", "N weeks" / "N days" / "N hours" / "N minutes" / "N seconds" "schedule_start_date" : datetime isoformat "schedule_end_date" : datetime isoformat -cron-like scheduling --------------------- +7.1.3 cron-like scheduling +-------------------------- +:: "schedule_year" : 4 digit year "schedule_month" : 1-12 @@ -282,158 +350,260 @@ cron-like scheduling "schedule_start_date" : datetime isoformat "schedule_end_date" : datetime isoformat +7.2 Job examples +---------------- +example backup freezer_action:: -example backup job_action -"job_action": { - "action" : "backup" - "mode" : "fs" - "src_file" : "/home/tylerdurden/project_mayhem" - "backup_name" : "project_mayhem_backup" - "container" : "my_backup_container" - "max_backup_level" : int - "always_backup_level": int - "restart_always_backup": int - "no_incremental" : bool - "encrypt_pass_file" : private_key_file - "log_file" : "/var/log/freezer.log" - "hostname" : false - "max_cpu_priority" : false -} - -example restore job_action -"job_action": { - "action": "restore" - "restore-abs-path": "/home/tylerdurden/project_mayhem" - "container" : "my_backup_container" - "backup-name": "project_mayhem_backup" - "restore-from-host": "another_host" - "max_cpu_priority": true -} - - -example scheduled backup job -job will be executed once at the provided datetime - -"job": { - "job_action": { - "action" : "backup", - "mode" : "fs", - "src_file" : "/home/tylerdurden/project_mayhem", - "backup_name" : "project_mayhem_backup", - "container" : "my_backup_container", - } - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 0, - "status": "scheduled", - "schedule_date": "2015-06-02T16:20:00" - } - "job_id": "blabla", - "client_id": "blabla", - "user_id": "blabla", - "description": "scheduled one shot", -} - - -new job, in stop status, with pending start request -job will be executed daily at the provided hour:min:sec -while year,month,day are ignored, if provided - -"job": { - "job_action": { + "freezer_action": { "action" : "backup" "mode" : "fs" "src_file" : "/home/tylerdurden/project_mayhem" "backup_name" : "project_mayhem_backup" "container" : "my_backup_container" - }, - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 0, - "status": "stop", - "event": "start" - "schedule_period" : "daily" - "schedule_time": "2015-06-02T16:20:00" - }, - "job_id": "blabla", - "client_id": "blabla", - "user_id": "blabla", - "description": "daily backup", -} + "max_backup_level" : int + "always_backup_level": int + "restart_always_backup": int + "no_incremental" : bool + "encrypt_pass_file" : private_key_file + "log_file" : "/var/log/freezer.log" + "hostname" : false + "max_cpu_priority" : false + } +example restore freezer_action:: -multiple scheduling choices allowed -"job": { - "job_action": { - "action" : "backup" - "mode" : "fs" - "src_file" : "/home/tylerdurden/project_mayhem" - "backup_name" : "project_mayhem_backup" + "freezer_action": { + "action": "restore" + "restore-abs-path": "/home/tylerdurden/project_mayhem" "container" : "my_backup_container" + "backup-name": "project_mayhem_backup" + "restore-from-host": "another_host" + "max_cpu_priority": true + } + + +example scheduled backup job. +job will be executed once at the provided datetime:: + + "job": { + "job_actions": + [ + { + "freezer_action": + { + "action" : "backup", + "mode" : "fs", + "src_file" : "/home/tylerdurden/project_mayhem", + "backup_name" : "project_mayhem_backup", + "container" : "my_backup_container", + } + "exit_status": "fail|success" + "max_retries": int, + "max_retries_interval": secs, + "mandatory": bool + }, + { + action + ... + }, + { + action + ... + } + ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "stop | scheduled | running", + "schedule_date": "2015-06-02T16:20:00", + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "scheduled one shot", + } + + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + + "status": "stop", + "event": "start" + "schedule_interval" : "1 day" + "schedule_start_date" : "2015-06-02T16:20:00" + }, + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "daily backup", + } + + +multiple scheduling choices allowed:: + + "job": { + "job_actions": + [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "scheduled" + "schedule_month" : "1-6, 9-12" + "schedule_day" : "mon, wed, fri" + "schedule_hour": "03" + "schedule_minute": "25" + } + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "daily backup", + } + + +Finished job with result:: + + "job": { + "job_actions": [ ... ], + "job_schedule": + { + "time_created": 1234, + "time_started": 1234, + "time_ended": 4321, + "status": "stop", + "event": "", + "result": "success", + "schedule_time": "2015-06-02T16:20:00" + }, + "job_id": "blabla", + "client_id": "blabla", + "user_id": "blabla", + "description": "one shot job", + } + + + +8 Actions +========= +Actions are stored only to facilitate the assembling of different actions into jobs in the web UI. +They are not directly used by the scheduler. +They are stored in this structure: +:: + + { + "freezer_action": { + "action": string, + "backup_name": string, + .... + }, + "mandatory": bool, + "max_retries": int, + "max_retries_interval": int + + "action_id": string, + "user_id": string } - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 0, - "status": "scheduled" - "schedule_month" : "1-6, 9-12" - "schedule_day" : "mon, wed, fri" - "schedule_hour": "03" - "schedule_minute": "25" + + +9. Sessions +=========== +A session is a group of jobs which share the same scheduling time. A session is identified +by its **session_id** and has a numeric tag (**session_tag**) which is incremented each time that a new session +is started. +The purpose of the *session_tag* is that of identifying a group of jobs which have been executed +together and which therefore represent a snapshot of a distributed system. + +When a job is added to a session, the scheduling time of the session is copied into the +job data structure, so that any job belonging to the same session will start at the same time. + + +9.1 Session Data Structure +-------------------------- +:: + + session = + { + "session_id": string, + "session_tag": int, + "description": string, + "hold_off": int (seconds), + "schedule": { scheduling information, same as jobs }, + "jobs": { 'job_id_1': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + }, + 'job_id_2': { + "client_id": string, + "status": string, + "result": string + "time_started": int (timestamp), + "time_ended": int (timestamp), + } + } + "time_start": int timestam, + "time_end": int timestam, + "time_started": int (timestamp), + "time_ended": int (timestamp), + "status": string "completed" "running", + "result": string "success" "fail", + "user_id": string } - "job_id": "blabla", - "client_id": "blabla", - "user_id": "blabla", - "description": "daily backup", -} +9.2 Session actions +------------------- +When the freezer scheduler running on a node wants to start a session, +it sends a POST request to the following endpoint: :: -Finished job with result: -"job": { - "job_action": { - "action" : "backup" - "mode" : "fs" - "src_file" : "/home/tylerdurden/project_mayhem" - "backup_name" : "project_mayhem_backup" - "container" : "my_backup_container" - }, - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 4321, - "status": "stop", - "event": "", - "result": "success", - "schedule_time": "2015-06-02T16:20:00" - }, - "job_id": "blabla", - "client_id": "blabla", - "user_id": "blabla", - "description": "one shot job", -} + POST /v1/sessions/{sessions_id}/action +The body of the request bears the action and parameters -Ini version: +9.2.1 Session START action +-------------------------- +:: -[job] -job_id = 12344321 -client_id = 12344321 -user_id = qwerty -description = scheduled one shot + { + "start": { + "job_id": "JOB_ID_HERE", + "current_tag": 22 + } + } -[job_action] -action = backup -mode = fs -src_file = /home/tylerdurden/project_mayhem -backup_name = project_mayhem_backup -container = my_backup_container +Example of a succesfull response: :: + + { + 'result': 'success', + 'session_tag': 23 + } + +8.2.2 Session STOP action +------------------------- +:: + + { + "end": { + "job_id": "JOB_ID_HERE", + "current_tag": 23, + "result": "success|fail" + } + } + +8.3 Session-Job association +--------------------------- + + PUT /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session + DELETE /v1/sessions/{sessions_id}/jobs/{job_id} adds the job to the session -[job_schedule] -time_created = 1234 -time_started = 1234 -time_ended = -status = scheduled -schedule_time = 2015-06-02T16:20:00 diff --git a/freezer_api/freezer_api/api/v1/__init__.py b/freezer_api/freezer_api/api/v1/__init__.py index 73ccdc6b..2538d5c9 100644 --- a/freezer_api/freezer_api/api/v1/__init__.py +++ b/freezer_api/freezer_api/api/v1/__init__.py @@ -19,10 +19,13 @@ Hudson (tjh@cryptsoft.com). ======================================================================== """ +from freezer_api.api.v1 import actions from freezer_api.api.v1 import backups from freezer_api.api.v1 import clients -from freezer_api.api.v1 import jobs from freezer_api.api.v1 import homedoc +from freezer_api.api.v1 import jobs +from freezer_api.api.v1 import sessions + VERSION = { 'id': '1', @@ -59,4 +62,23 @@ def public_endpoints(storage_driver): ('/jobs/{job_id}', jobs.JobsResource(storage_driver)), + + ('/actions', + actions.ActionsCollectionResource(storage_driver)), + + ('/actions/{action_id}', + actions.ActionsResource(storage_driver)), + + ('/sessions', + sessions.SessionsCollectionResource(storage_driver)), + + ('/sessions/{session_id}', + sessions.SessionsResource(storage_driver)), + + ('/sessions/{session_id}/action', + sessions.SessionsAction(storage_driver)), + + ('/sessions/{session_id}/jobs/{job_id}', + sessions.SessionsJob(storage_driver)), + ] diff --git a/freezer_api/freezer_api/api/v1/actions.py b/freezer_api/freezer_api/api/v1/actions.py new file mode 100644 index 00000000..aad954fb --- /dev/null +++ b/freezer_api/freezer_api/api/v1/actions.py @@ -0,0 +1,101 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import falcon +from freezer_api.common import exceptions + + +class ActionsCollectionResource(object): + """ + Handler for endpoint: /v1/actions + """ + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp): + # GET /v1/actions(?limit,offset) Lists actions + user_id = req.get_header('X-User-ID') + offset = req.get_param_as_int('offset') or 0 + limit = req.get_param_as_int('limit') or 10 + search = req.context.get('doc', {}) + obj_list = self.db.search_action(user_id=user_id, offset=offset, + limit=limit, search=search) + req.context['result'] = {'actions': obj_list} + + def on_post(self, req, resp): + # POST /v1/actions Creates action entry + try: + doc = req.context['doc'] + except KeyError: + raise exceptions.BadDataFormat( + message='Missing request body') + + user_id = req.get_header('X-User-ID') + action_id = self.db.add_action(user_id=user_id, doc=doc) + resp.status = falcon.HTTP_201 + req.context['result'] = {'action_id': action_id} + + +class ActionsResource(object): + """ + Handler for endpoint: /v1/actions/{action_id} + """ + + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp, action_id): + # GET /v1/actions/{action_id} retrieves the specified action + # search in body + user_id = req.get_header('X-User-ID') or '' + obj = self.db.get_action(user_id=user_id, action_id=action_id) + if obj: + req.context['result'] = obj + else: + resp.status = falcon.HTTP_404 + + def on_delete(self, req, resp, action_id): + # DELETE /v1/actions/{action_id} Deletes the specified action + user_id = req.get_header('X-User-ID') + self.db.delete_action(user_id=user_id, action_id=action_id) + req.context['result'] = {'action_id': action_id} + resp.status = falcon.HTTP_204 + + def on_patch(self, req, resp, action_id): + # PATCH /v1/actions/{action_id} updates the specified action + user_id = req.get_header('X-User-ID') or '' + doc = req.context.get('doc', {}) + new_version = self.db.update_action(user_id=user_id, + action_id=action_id, + patch_doc=doc) + req.context['result'] = {'action_id': action_id, + 'version': new_version} + + def on_post(self, req, resp, action_id): + # PUT /v1/actions/{job_id} creates/replaces the specified action + user_id = req.get_header('X-User-ID') or '' + doc = req.context.get('doc', {}) + new_version = self.db.replace_action(user_id=user_id, + action_id=action_id, + doc=doc) + resp.status = falcon.HTTP_201 + req.context['result'] = {'action_id': action_id, + 'version': new_version} diff --git a/freezer_api/freezer_api/api/v1/sessions.py b/freezer_api/freezer_api/api/v1/sessions.py new file mode 100644 index 00000000..fa6155b9 --- /dev/null +++ b/freezer_api/freezer_api/api/v1/sessions.py @@ -0,0 +1,348 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import time + +import falcon + +from freezer_api.common import exceptions + + +class SessionsCollectionResource(object): + """ + Handler for endpoint: /v1/sessions + """ + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp): + # GET /v1/sessions(?limit,offset) Lists sessions + user_id = req.get_header('X-User-ID') + offset = req.get_param_as_int('offset') or 0 + limit = req.get_param_as_int('limit') or 10 + search = req.context.get('doc', {}) + obj_list = self.db.search_session(user_id=user_id, offset=offset, + limit=limit, search=search) + req.context['result'] = {'sessions': obj_list} + + def on_post(self, req, resp): + # POST /v1/sessions Creates session entry + try: + doc = req.context['doc'] + except KeyError: + raise exceptions.BadDataFormat( + message='Missing request body') + + user_id = req.get_header('X-User-ID') + session_id = self.db.add_session(user_id=user_id, doc=doc) + resp.status = falcon.HTTP_201 + req.context['result'] = {'session_id': session_id} + + +class SessionsResource(object): + """ + Handler for endpoint: /v1/sessions/{session_id} + """ + + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp, session_id): + # GET /v1/sessions/{session_id} retrieves the specified session + # search in body + user_id = req.get_header('X-User-ID') or '' + obj = self.db.get_session(user_id=user_id, session_id=session_id) + if obj: + req.context['result'] = obj + else: + resp.status = falcon.HTTP_404 + + def on_delete(self, req, resp, session_id): + # DELETE /v1/sessions/{session_id} Deletes the specified session + user_id = req.get_header('X-User-ID') + self.db.delete_session(user_id=user_id, session_id=session_id) + req.context['result'] = {'session_id': session_id} + resp.status = falcon.HTTP_204 + + def on_patch(self, req, resp, session_id): + # PATCH /v1/sessions/{session_id} updates the specified session + user_id = req.get_header('X-User-ID') or '' + doc = req.context.get('doc', {}) + new_version = self.db.update_session(user_id=user_id, + session_id=session_id, + patch_doc=doc) + req.context['result'] = {'session_id': session_id, + 'version': new_version} + + def on_post(self, req, resp, session_id): + # PUT /v1/sessions/{session_id} creates/replaces the specified session + user_id = req.get_header('X-User-ID') or '' + doc = req.context.get('doc', {}) + new_version = self.db.replace_session(user_id=user_id, + session_id=session_id, + doc=doc) + resp.status = falcon.HTTP_201 + req.context['result'] = {'session_id': session_id, + 'version': new_version} + + +class SessionsAction(object): + """ + Handler for endpoint: /v1/sessions/{session_id}/action + """ + + def __init__(self, storage_driver): + self.db = storage_driver + + def on_post(self, req, resp, session_id): + # POST /v1/sessions/{session_id}/action + # executes an action on the specified session + + user_id = req.get_header('X-User-ID') or '' + doc = req.context.get('doc', {}) + + try: + action, params = next(doc.iteritems()) + except: + raise exceptions.BadDataFormat("Bad action request format") + + session_doc = self.db.get_session(user_id=user_id, + session_id=session_id) + session = Session(session_doc) + session.execute_action(action, params) + + if session.need_update: + self.db.update_session(user_id=user_id, + session_id=session_id, + patch_doc=session.doc) + resp.status = falcon.HTTP_202 + req.context['result'] = {'result': session.action_result, + 'session_tag': session.session_tag} + + +class Session(object): + """ + A class to manage the actions that can be taken upon a + Session data structure. + It modifies information contained in its document + in accordance to the requested action + """ + def __init__(self, doc): + self.doc = doc + self.action_result = '' + self.need_update = False + + @property + def session_tag(self): + return self.doc.get('session_tag', 0) + + @session_tag.setter + def session_tag(self, value): + self.doc['session_tag'] = value + + def execute_action(self, action, params): + if action == 'start': + try: + self.start(params['job_id'], params['current_tag']) + except exceptions.BadDataFormat: + raise + except Exception as e: + raise exceptions.FreezerAPIException(e) + elif action == 'end': + try: + self.end(params['job_id'], params['result']) + except exceptions.BadDataFormat: + raise + except Exception as e: + raise exceptions.FreezerAPIException(e) + else: + raise exceptions.MethodNotImplemented("Bad Action Method") + + def end(self, job_id, result): + """ + Apply the 'end' action to the session object + If the request can be accepted it modifies the relevant fields + and sets the need_update member to notify that the stored + document needs to be updated + """ + now = int(time.time()) + self.set_job_end(job_id, result, now) + new_result = self.get_job_overall_result() + if self.doc.get('status', '') != 'completed': + if new_result in ['fail', 'success']: + self.doc['time_end'] = now + self.doc['result'] = new_result + self.doc['status'] = 'completed' + self.action_result = 'success' + self.need_update = True + + def start(self, job_id, job_tag): + """ + Apply the 'start' action to the session object + If the request can be accepted it modifies the relevant fields + and sets the need_update member to notify that the stored + document needs to be updated + """ + now = int(time.time()) + time_since_last_start = now - self.doc.get('time_start', 0) + + if job_tag > self.session_tag: + raise exceptions.BadDataFormat('requested tag value too high') + + if time_since_last_start <= self.doc.get('hold_off', 60): + # session has been started not so long ago + # tag increments are not allowed during hold_off + if job_tag < self.session_tag: + self.action_result = 'success' + self.set_job_start(job_id, now) + self.need_update = True + else: + self.action_result = 'hold-off' + self.need_update = False + elif time_since_last_start > self.doc.get('hold_off', 60): + # out of hold_off window: + # - ok to trigger new action start (job_tag == session_tag) + # if job_tag < session_tag client is probably out-of-sync + if self.session_tag == job_tag: + self.session_tag += 1 + self.doc['time_start'] = now + self.doc['status'] = 'running' + self.doc['result'] = '' + self.action_result = 'success' + self.set_job_start(job_id, now) + self.need_update = True + else: + self.action_result = 'out-of-sync' + self.need_update = False + + def get_job_overall_result(self): + """ + check the status of all the jobs and return the overall session result + """ + for job in self.doc['jobs'].itervalues(): + if job['status'] != 'completed': + return 'running' + if job['result'] != 'success': + return 'fail' + return 'success' + + def set_job_end(self, job_id, result, timestamp): + try: + job = self.doc['jobs'][job_id] + except: + raise exceptions.BadDataFormat('job_id not found in session') + job['status'] = 'completed' + job['result'] = result + job['time_ended'] = timestamp + + def set_job_start(self, job_id, timestamp): + try: + job = self.doc['jobs'][job_id] + except: + raise exceptions.BadDataFormat('job_id not found in session') + job['status'] = 'running' + job['result'] = '' + job['time_started'] = timestamp + + +class SessionsJob(object): + """ + Handler for endpoint: /v1/sessions/{session_id}/jobs/{job_id} + """ + + def __init__(self, storage_driver): + self.db = storage_driver + + def on_put(self, req, resp, session_id, job_id): + """ + add a job to a session + + :param req: + :param resp: + :param session_id: + :param job_id: + :return: + """ + + user_id = req.get_header('X-User-ID', '') + + # --- update session object + job_doc = self.db.get_job(user_id=user_id, job_id=job_id) + + job_schedule = job_doc.get('job_schedule', {}) + session_update_doc = { + 'jobs': { + job_id: { + 'client_id': job_doc['client_id'], + 'status': job_schedule.get('status', ''), + 'result': job_schedule.get('result', ''), + 'time_started': job_schedule.get('time_started', ''), + 'time_ended': job_schedule.get('time_ended', '') + } + } + } + + self.db.update_session(user_id=user_id, + session_id=session_id, + patch_doc=session_update_doc) + # --- update job object + session_doc = self.db.get_session(user_id=user_id, + session_id=session_id) + job_update_doc = { + 'session_id': session_id, + 'session_tag': session_doc['session_tag'], + 'job_schedule': session_doc['schedule'] + } + self.db.update_job(user_id=user_id, + job_id=job_id, + patch_doc=job_update_doc) + resp.status = falcon.HTTP_204 + + def on_delete(self, req, resp, session_id, job_id): + """ + remove a job from the session + + :param req: + :param resp: + :param session_id: + :param job_id: + :return: + """ + + user_id = req.get_header('X-User-ID') or '' + + session_doc = self.db.get_session(user_id=user_id, + session_id=session_id) + session_doc['jobs'].pop(job_id, None) + + # when replacing, db might raise a VersionConflictEngineException + self.db.replace_session(user_id=user_id, + session_id=session_id, + doc=session_doc) + job_update_doc = { + 'session_id': '', + 'session_tag': '', + 'job_event': 'stop' + } + self.db.update_job(user_id=user_id, + job_id=job_id, + patch_doc=job_update_doc) + resp.status = falcon.HTTP_204 diff --git a/freezer_api/freezer_api/cmd/api.py b/freezer_api/freezer_api/cmd/api.py index 19d6ea1c..543d8b0a 100644 --- a/freezer_api/freezer_api/cmd/api.py +++ b/freezer_api/freezer_api/cmd/api.py @@ -77,8 +77,13 @@ except Exception as err: def main(): + # quick simple server for testing purposes or simple scenarios ip, port = '127.0.0.1', 9090 - httpd = simple_server.make_server(ip, port, application) + if len(sys.argv) > 1: + ip = sys.argv[1] + if ':' in ip: + ip, port = ip.split(':') + httpd = simple_server.make_server(ip, int(port), application) message = 'Server listening on {0}:{1}'.format(ip, port) print message logging.info(message) diff --git a/freezer_api/freezer_api/common/exceptions.py b/freezer_api/freezer_api/common/exceptions.py index a1343964..2a9d5f97 100644 --- a/freezer_api/freezer_api/common/exceptions.py +++ b/freezer_api/freezer_api/common/exceptions.py @@ -32,7 +32,7 @@ class FreezerAPIException(falcon.HTTPError): def __init__(self, message=''): if message: - self.message = message + self.message = str(message) logging.error(message) Exception.__init__(self, message) @@ -74,6 +74,7 @@ class DocumentNotFound(FreezerAPIException): title="Not Found", description=ex.message) + class AccessForbidden(FreezerAPIException): @staticmethod def handle(ex, req, resp, params): @@ -81,10 +82,20 @@ class AccessForbidden(FreezerAPIException): title="Access Forbidden", description=ex.message) + +class MethodNotImplemented(FreezerAPIException): + @staticmethod + def handle(ex, req, resp, params): + raise falcon.HTTPMethodNotAllowed( + title="Bad Method", + description=ex.message) + + exception_handlers_catalog = [ BadDataFormat, DocumentExists, StorageEngineError, DocumentNotFound, - AccessForbidden + AccessForbidden, + MethodNotImplemented ] diff --git a/freezer_api/freezer_api/common/json_schemas.py b/freezer_api/freezer_api/common/json_schemas.py index e1e1c15c..aebbd53b 100644 --- a/freezer_api/freezer_api/common/json_schemas.py +++ b/freezer_api/freezer_api/common/json_schemas.py @@ -19,124 +19,163 @@ Hudson (tjh@cryptsoft.com). ======================================================================== """ -job_action_properties = { - "action": { - "id": "action", - "type": "string" - }, - "mode": { - "id": "mode", - "type": "string" - }, - "src_file": { - "id": "src_file", - "type": "string" - }, - "backup_name": { - "id": "backup_name", - "type": "string" - }, - "container": { - "id": "container", - "type": "string" - } - } +freezer_action_properties = { + "action": { + "id": "action", + "pattern": "^[\w-]+$", + "type": "string" + }, + "mode": { + "id": "mode", + "pattern": "^[\w-]+$", + "type": "string" + }, + "src_file": { + "id": "src_file", + "type": "string" + }, + "backup_name": { + "id": "backup_name", + "type": "string" + }, + "container": { + "id": "container", + "type": "string" + }, + "restore_abs_path": { + "id": "restore_abs_path", + "type": "string" + }, +} job_schedule_properties = { - "time_created": { - "id": "time_created", - "type": "integer" - }, - "time_started": { - "id": "time_started", - "type": "integer" - }, - "time_ended": { - "id": "time_ended", - "type": "integer" - }, - "event": { - "id": "event", - "type": "string", - "enum": ["", "stop", "start", "abort", "remove"] - }, - "status": { - "id": "status", - "type": "string", - "enum": ["completed", "stop", "scheduled", "running", "aborting", "removed"] - }, - "result": { - "id": "result", - "type": "string", - "enum": ["", "success", "fail", "aborted"] - }, - "schedule_date": { - "id": "schedule_date", - "type": "string", - "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$" - }, - "schedule_interval": { - "id": "schedule_interval", - "type": "string", - "pattern": "^(continuous|(\d+ +(weeks|weeks|days|hours|minutes|seconds)))$" - }, - "schedule_start_date": { - "id": "schedule_start_date", - "type": "string", - "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$" - }, - "schedule_end_date": { - "id": "schedule_end_date", - "type": "string", - "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-](?:2[0-3]|[01][0-9]):[0-5][0-9])?$" - }, - "schedule_year": { - "id": "schedule_year", - "type": "string", - "pattern": "^\d{4}$" - }, - "schedule_month": { - "id": "schedule_month", - "type": "string" - }, - "schedule_day": { - "id": "schedule_day", - "type": "string" - }, - "schedule_week": { - "id": "schedule_week", - "type": "string" - }, - "schedule_day_of_week": { - "id": "schedule_day_of_week", - "type": "string" - }, - "schedule_hour": { - "id": "schedule_hour", - "type": "string" - }, - "schedule_minute": { - "id": "schedule_minute", - "type": "string" - }, - "schedule_second": { - "id": "schedule_second", - "type": "string" - }, - } + "time_created": { + "id": "time_created", + "type": "integer" + }, + "time_started": { + "id": "time_started", + "type": "integer" + }, + "time_ended": { + "id": "time_ended", + "type": "integer" + }, + "event": { + "id": "event", + "type": "string", + "enum": ["", "stop", "start", "abort", "remove"] + }, + "status": { + "id": "status", + "type": "string", + "enum": ["completed", "stop", "scheduled", + "running", "aborting", "removed"] + }, + "result": { + "id": "result", + "type": "string", + "enum": ["", "success", "fail", "aborted"] + }, + "schedule_date": { + "id": "schedule_date", + "type": "string", + "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])" + "-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9])" + ":([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-]" + "(?:2[0-3]|[01][0-9]):[0-5][0-9])?$" + }, + "schedule_interval": { + "id": "schedule_interval", + "type": "string", + "pattern": "^(continuous|(\d+ +(weeks|weeks|days|" + "hours|minutes|seconds)))$" + }, + "schedule_start_date": { + "id": "schedule_start_date", + "type": "string", + "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])" + "-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9]):" + "([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-]" + "(?:2[0-3]|[01][0-9]):[0-5][0-9])?$" + }, + "schedule_end_date": { + "id": "schedule_end_date", + "type": "string", + "pattern": "^(-?(?:[1-9][0-9]*)?[0-9]{4})-(1[0-2]|0[1-9])" + "-(3[01]|0[1-9]|[12][0-9])T(2[0-3]|[01][0-9])" + ":([0-5][0-9]):([0-5][0-9])(\.[0-9]+)?(Z|[+-]" + "(?:2[0-3]|[01][0-9]):[0-5][0-9])?$" + }, + "schedule_year": { + "id": "schedule_year", + "type": "string", + "pattern": "^\d{4}$" + }, + "schedule_month": { + "id": "schedule_month", + "type": "string" + }, + "schedule_day": { + "id": "schedule_day", + "type": "string" + }, + "schedule_week": { + "id": "schedule_week", + "type": "string" + }, + "schedule_day_of_week": { + "id": "schedule_day_of_week", + "type": "string" + }, + "schedule_hour": { + "id": "schedule_hour", + "type": "string" + }, + "schedule_minute": { + "id": "schedule_minute", + "type": "string" + }, + "schedule_second": { + "id": "schedule_second", + "type": "string" + }, +} job_schema = { "id": "/", "type": "object", - "properties": { + "definitions": { + "freezer_action": { + "properties": freezer_action_properties, + "additionalProperties": True + }, "job_action": { - "id": "job_action", - "type": "object", - "properties": job_action_properties, - "additionalProperties": True, - "required": [ - "action" - ] + "properties": { + "freezer_action": { + "$ref": "#/definitions/freezer_action" + }, + "max_retries": { + "type": "integer" + }, + "max_retries_interval": { + "type": "integer" + }, + "mandatory": { + "type": "boolean" + } + }, + "additionalProperties": True + }, + "job_action_list": { + "items": { + "$ref": "#/definitions/job_action" + } + } + }, + "properties": { + "job_actions": { + "$ref": "#/definitions/job_action_list" }, "job_schedule": { "id": "job_schedule", @@ -162,11 +201,15 @@ job_schema = { "description": { "id": "description", "type": "string" + }, + "_version": { + "id": "_version", + "type": "integer" } }, "additionalProperties": False, "required": [ - "job_action", + "job_actions", "job_schedule", "job_id", "client_id", @@ -174,17 +217,41 @@ job_schema = { ] } - job_patch_schema = { "id": "/", "type": "object", - "properties": { - "job_action": { - "id": "job_action", - "type": "object", - "properties": job_action_properties, + "definitions": { + "freezer_action": { + "properties": freezer_action_properties, "additionalProperties": True }, + "job_action": { + "properties": { + "freezer_action": { + "$ref": "#/definitions/freezer_action" + }, + "max_retries": { + "type": "integer" + }, + "max_retries_interval": { + "type": "integer" + }, + "mandatory": { + "type": "boolean" + } + }, + "additionalProperties": True + }, + "job_action_list": { + "items": { + "$ref": "#/definitions/job_action" + } + } + }, + "properties": { + "job_actions": { + "$ref": "#/definitions/job_action_list" + }, "job_schedule": { "id": "job_schedule", "type": "object", @@ -193,20 +260,128 @@ job_patch_schema = { }, "job_id": { "id": "job_id", + "pattern": "^[\w-]+$", "type": "string" }, "client_id": { "id": "client_id", + "pattern": "^[\w-]+$", "type": "string" }, "user_id": { "id": "user_id", + "pattern": "^[\w-]+$", "type": "string" }, "description": { "id": "description", "type": "string" + }, + "_version": { + "id": "_version", + "type": "integer" } }, "additionalProperties": False } + + +additional_action_properties = { + "action_id": { + "id": "action_id", + "pattern": "^[\w-]+$", + "type": "string" + }, + "user_id": { + "id": "user_id", + "pattern": "^[\w-]+$", + "type": "string" + }, +} + + +action_schema = { + "id": "/", + "type": "object", + "properties": dict(freezer_action_properties.items() + + additional_action_properties.items()), + "additionalProperties": True, + "required": [ + "action_id", + "user_id" + ] +} + + +action_patch_schema = { + "id": "/", + "type": "object", + "properties": dict(freezer_action_properties.items() + + additional_action_properties.items()), + "additionalProperties": True +} + +session_schema = { + "id": "/", + "type": "object", + "properties": { + "session_id": { + "id": "session_id", + "pattern": "^[\w-]+$", + "type": "string" + }, + "user_id": { + "id": "user_id", + "pattern": "^[\w-]+$", + "type": "string" + }, + "session_tag": { + "id": "session_tag", + "type": "integer" + }, + "time_started": { + "id": "session_tag", + "type": "integer" + }, + "time_ended": { + "id": "session_tag", + "type": "integer" + }, + }, + "additionalProperties": True, + "required": [ + "session_id", + "session_tag", + "user_id" + ] +} + +session_patch_schema = { + "id": "/", + "type": "object", + "properties": { + "session_id": { + "id": "session_id", + "pattern": "^[\w-]+$", + "type": "string" + }, + "user_id": { + "id": "user_id", + "pattern": "^[\w-]+$", + "type": "string" + }, + "session_tag": { + "id": "session_tag", + "type": "integer" + }, + "time_started": { + "id": "session_tag", + "type": "integer" + }, + "time_ended": { + "id": "session_tag", + "type": "integer" + }, + }, + "additionalProperties": True +} diff --git a/freezer_api/freezer_api/common/utils.py b/freezer_api/freezer_api/common/utils.py index 5848e394..ad22eb8d 100644 --- a/freezer_api/freezer_api/common/utils.py +++ b/freezer_api/freezer_api/common/utils.py @@ -55,7 +55,7 @@ class BackupMetadataDoc: def backup_set_id(self): return '{0}_{1}_{2}'.format( self.data['container'], - self.data['host_name'], + self.data['hostname'], self.data['backup_name'] ) @@ -63,8 +63,8 @@ class BackupMetadataDoc: def backup_id(self): return '{0}_{1}_{2}'.format( self.backup_set_id, - self.data['timestamp'], - self.data['level'] + self.data['time_stamp'], + self.data['curr_backup_level'] ) @@ -120,3 +120,102 @@ class JobDoc: }) JobDoc.validate(doc) return doc + + +class ActionDoc: + action_doc_validator = jsonschema.Draft4Validator( + schema=json_schemas.action_schema) + action_patch_validator = jsonschema.Draft4Validator( + schema=json_schemas.action_patch_schema) + + @staticmethod + def validate(doc): + try: + ActionDoc.action_doc_validator.validate(doc) + except Exception as e: + raise exceptions.BadDataFormat(str(e).splitlines()[0]) + + @staticmethod + def validate_patch(doc): + try: + ActionDoc.action_patch_validator.validate(doc) + except Exception as e: + raise exceptions.BadDataFormat(str(e).splitlines()[0]) + + @staticmethod + def create_patch(doc): + # changes in user_id or action_id are not allowed + doc.pop('user_id', None) + doc.pop('action_id', None) + ActionDoc.validate_patch(doc) + return doc + + @staticmethod + def create(doc, user_id): + doc.update({ + 'user_id': user_id, + 'action_id': uuid.uuid4().hex, + }) + ActionDoc.validate(doc) + return doc + + @staticmethod + def update(doc, user_id, action_id): + doc.update({ + 'user_id': user_id, + 'action_id': action_id, + }) + ActionDoc.validate(doc) + return doc + + +class SessionDoc: + session_doc_validator = jsonschema.Draft4Validator( + schema=json_schemas.session_schema) + session_patch_validator = jsonschema.Draft4Validator( + schema=json_schemas.session_patch_schema) + + @staticmethod + def validate(doc): + try: + SessionDoc.session_doc_validator.validate(doc) + except Exception as e: + raise exceptions.BadDataFormat(str(e).splitlines()[0]) + + @staticmethod + def validate_patch(doc): + try: + SessionDoc.session_patch_validator.validate(doc) + except Exception as e: + raise exceptions.BadDataFormat(str(e).splitlines()[0]) + + @staticmethod + def create_patch(doc): + # changes in user_id or session_id are not allowed + doc.pop('user_id', None) + doc.pop('session_id', None) + SessionDoc.validate_patch(doc) + return doc + + @staticmethod + def create(doc, user_id, hold_off=30): + doc.update({ + 'user_id': user_id, + 'session_id': uuid.uuid4().hex, + 'session_tag': 0, + 'status': 'active', + 'last_start': '', + 'jobs': [] + }) + doc['hold_off'] = doc.get('hold_off', hold_off) + SessionDoc.validate(doc) + return doc + + @staticmethod + def update(doc, user_id, session_id): + doc.update({ + 'user_id': user_id, + 'session_id': session_id, + }) + SessionDoc.validate(doc) + return doc diff --git a/freezer_api/freezer_api/storage/elastic.py b/freezer_api/freezer_api/storage/elastic.py index c3b763b1..055b257a 100644 --- a/freezer_api/freezer_api/storage/elastic.py +++ b/freezer_api/freezer_api/storage/elastic.py @@ -23,6 +23,8 @@ import elasticsearch import logging from freezer_api.common.utils import BackupMetadataDoc from freezer_api.common.utils import JobDoc +from freezer_api.common.utils import ActionDoc +from freezer_api.common.utils import SessionDoc from freezer_api.common import exceptions @@ -33,16 +35,19 @@ class TypeManager: self.doc_type = doc_type @staticmethod - def get_base_search_filter(user_id, search={}): + def get_base_search_filter(user_id, search=None): + search = search or {} user_id_filter = {"term": {"user_id": user_id}} base_filter = [user_id_filter] match_list = [{"match": m} for m in search.get('match', [])] match_not_list = [{"match": m} for m in search.get('match_not', [])] - base_filter.append({"query": {"bool": {"must": match_list, "must_not": match_not_list}}}) + base_filter.append({"query": {"bool": {"must": match_list, + "must_not": match_not_list}}}) return base_filter @staticmethod - def get_search_query(user_id, doc_id, search={}): + def get_search_query(user_id, doc_id, search=None): + search = search or {} try: base_filter = TypeManager.get_base_search_filter(user_id, search) query_filter = {"filter": {"bool": {"must": base_filter}}} @@ -65,16 +70,19 @@ class TypeManager: message='Get operation failed: {0}'.format(e)) if doc['user_id'] != user_id: raise exceptions.AccessForbidden("Document access forbidden") + if '_version' in res: + doc['_version'] = res['_version'] return doc - def search(self, user_id, doc_id=None, search={}, offset=0, limit=10): + def search(self, user_id, doc_id=None, search=None, offset=0, limit=10): + search = search or {} query_dsl = self.get_search_query(user_id, doc_id, search) try: res = self.es.search(index=self.index, doc_type=self.doc_type, size=limit, from_=offset, body=query_dsl) except elasticsearch.ConnectionError: raise exceptions.StorageEngineError( - message='unable to connecto to db server') + message='unable to connect to db server') except Exception as e: raise exceptions.StorageEngineError( message='search operation failed: {0}'.format(e)) @@ -82,11 +90,17 @@ class TypeManager: return [x['_source'] for x in hit_list] def insert(self, doc, doc_id=None): + version = doc.pop('_version', 0) try: res = self.es.index(index=self.index, doc_type=self.doc_type, - body=doc, id=doc_id) + body=doc, id=doc_id, version=version) created = res['created'] version = res['_version'] + except elasticsearch.TransportError as e: + if e.status_code == 409: + raise exceptions.DocumentExists(message=e.error) + raise exceptions.StorageEngineError( + message='index operation failed {0}'.format(e)) except Exception as e: raise exceptions.StorageEngineError( message='index operation failed {0}'.format(e)) @@ -109,7 +123,8 @@ class BackupTypeManager(TypeManager): TypeManager.__init__(self, es, doc_type, index=index) @staticmethod - def get_search_query(user_id, doc_id, search={}): + def get_search_query(user_id, doc_id, search=None): + search = search or {} base_filter = TypeManager.get_base_search_filter(user_id, search) if doc_id is not None: base_filter.append({"term": {"backup_id": doc_id}}) @@ -132,7 +147,8 @@ class ClientTypeManager(TypeManager): TypeManager.__init__(self, es, doc_type, index=index) @staticmethod - def get_search_query(user_id, doc_id, search={}): + def get_search_query(user_id, doc_id, search=None): + search = search or {} base_filter = TypeManager.get_base_search_filter(user_id, search) if doc_id is not None: base_filter.append({"term": {"client_id": doc_id}}) @@ -145,7 +161,8 @@ class JobTypeManager(TypeManager): TypeManager.__init__(self, es, doc_type, index=index) @staticmethod - def get_search_query(user_id, doc_id, search={}): + def get_search_query(user_id, doc_id, search=None): + search = search or {} base_filter = TypeManager.get_base_search_filter(user_id, search) if doc_id is not None: base_filter.append({"term": {"job_id": doc_id}}) @@ -153,21 +170,93 @@ class JobTypeManager(TypeManager): return {'query': {'filtered': query_filter}} def update(self, job_id, job_update_doc): + version = job_update_doc.pop('_version', 0) update_doc = {"doc": job_update_doc} try: res = self.es.update(index=self.index, doc_type=self.doc_type, - id=job_id, body=update_doc) + id=job_id, body=update_doc, version=version) version = res['_version'] - except elasticsearch.TransportError: + except elasticsearch.TransportError as e: + if e.status_code == 409: + raise exceptions.DocumentExists(message=e.error) raise exceptions.DocumentNotFound( message='Unable to find job to update ' - 'with id {0} '.format(job_id)) + 'with id {0}. {1}'.format(job_id, e)) except Exception: raise exceptions.StorageEngineError( message='Unable to update job with id {0}'.format(job_id)) return version +class ActionTypeManager(TypeManager): + def __init__(self, es, doc_type, index='freezer'): + TypeManager.__init__(self, es, doc_type, index=index) + + @staticmethod + def get_search_query(user_id, doc_id, search=None): + search = search or {} + base_filter = TypeManager.get_base_search_filter(user_id, search) + if doc_id is not None: + base_filter.append({"term": {"action_id": doc_id}}) + query_filter = {"filter": {"bool": {"must": base_filter}}} + return {'query': {'filtered': query_filter}} + + def update(self, action_id, action_update_doc): + version = action_update_doc.pop('_version', 0) + update_doc = {"doc": action_update_doc} + try: + res = self.es.update(index=self.index, doc_type=self.doc_type, + id=action_id, body=update_doc, + version=version) + version = res['_version'] + except elasticsearch.TransportError as e: + if e.status_code == 409: + raise exceptions.DocumentExists(message=e.error) + raise exceptions.DocumentNotFound( + message='Unable to find action to update ' + 'with id {0} '.format(action_id)) + except Exception: + raise exceptions.StorageEngineError( + message='Unable to update action with' + ' id {0}'.format(action_id)) + return version + + +class SessionTypeManager(TypeManager): + def __init__(self, es, doc_type, index='freezer'): + TypeManager.__init__(self, es, doc_type, index=index) + + @staticmethod + def get_search_query(user_id, doc_id, search=None): + search = search or {} + base_filter = TypeManager.get_base_search_filter(user_id, search) + if doc_id is not None: + base_filter.append({"term": {"session_id": doc_id}}) + query_filter = {"filter": {"bool": {"must": base_filter}}} + return {'query': {'filtered': query_filter}} + + def update(self, session_id, session_update_doc): + version = session_update_doc.pop('_version', 0) + update_doc = {"doc": session_update_doc} + try: + res = self.es.update(index=self.index, doc_type=self.doc_type, + id=session_id, body=update_doc, + version=version) + version = res['_version'] + except elasticsearch.TransportError as e: + if e.status_code == 409: + raise exceptions.DocumentExists(message=e.error) + raise exceptions.DocumentNotFound( + message='Unable to update session {0}. ' + '{1}'.format(session_id, e)) + + except Exception: + raise exceptions.StorageEngineError( + message='Unable to update session with ' + 'id {0}'.format(session_id)) + return version + + class ElasticSearchEngine(object): def __init__(self, hosts, index='freezer'): @@ -177,9 +266,12 @@ class ElasticSearchEngine(object): self.backup_manager = BackupTypeManager(self.es, 'backups') self.client_manager = ClientTypeManager(self.es, 'clients') self.job_manager = JobTypeManager(self.es, 'jobs') + self.action_manager = ActionTypeManager(self.es, 'actions') + self.session_manager = SessionTypeManager(self.es, 'sessions') def get_backup(self, user_id, backup_id=None, - offset=0, limit=10, search={}): + offset=0, limit=10, search=None): + search = search or {} return self.backup_manager.search(user_id, backup_id, search=search, @@ -204,7 +296,8 @@ class ElasticSearchEngine(object): return self.backup_manager.delete(user_id, backup_id) def get_client(self, user_id, client_id=None, - offset=0, limit=10, search={}): + offset=0, limit=10, search=None): + search = search or {} return self.client_manager.search(user_id, client_id, search=search, @@ -233,7 +326,8 @@ class ElasticSearchEngine(object): def get_job(self, user_id, job_id): return self.job_manager.get(user_id, job_id) - def search_job(self, user_id, offset=0, limit=10, search={}): + def search_job(self, user_id, offset=0, limit=10, search=None): + search = search or {} return self.job_manager.search(user_id, search=search, offset=offset, @@ -278,3 +372,103 @@ class ElasticSearchEngine(object): logging.info('Job {0} replaced with version {1}'. format(job_id, version)) return version + + def get_action(self, user_id, action_id): + return self.action_manager.get(user_id, action_id) + + def search_action(self, user_id, offset=0, limit=10, search=None): + search = search or {} + return self.action_manager.search(user_id, + search=search, + offset=offset, + limit=limit) + + def add_action(self, user_id, doc): + actiondoc = ActionDoc.create(doc, user_id) + action_id = actiondoc['action_id'] + self.action_manager.insert(actiondoc, action_id) + logging.info('Action registered, action id: {0}'. + format(action_id)) + return action_id + + def delete_action(self, user_id, action_id): + return self.action_manager.delete(user_id, action_id) + + def update_action(self, user_id, action_id, patch_doc): + valid_patch = ActionDoc.create_patch(patch_doc) + + # check that document exists + assert (self.action_manager.get(user_id, action_id)) + + version = self.action_manager.update(action_id, valid_patch) + logging.info('Action {0} updated to version {1}'. + format(action_id, version)) + return version + + def replace_action(self, user_id, action_id, doc): + # check that no document exists with + # same action_id and different user_id + try: + self.action_manager.get(user_id, action_id) + except exceptions.DocumentNotFound: + pass + + valid_doc = ActionDoc.update(doc, user_id, action_id) + + (created, version) = self.action_manager.insert(valid_doc, action_id) + if created: + logging.info('Action {0} created'.format(action_id, version)) + else: + logging.info('Action {0} replaced with version {1}'. + format(action_id, version)) + return version + + def get_session(self, user_id, session_id): + return self.session_manager.get(user_id, session_id) + + def search_session(self, user_id, offset=0, limit=10, search=None): + search = search or {} + return self.session_manager.search(user_id, + search=search, + offset=offset, + limit=limit) + + def add_session(self, user_id, doc): + session_doc = SessionDoc.create(doc, user_id) + session_id = session_doc['session_id'] + self.session_manager.insert(session_doc, session_id) + logging.info('Session registered, session id: {0}'. + format(session_id)) + return session_id + + def delete_session(self, user_id, session_id): + return self.session_manager.delete(user_id, session_id) + + def update_session(self, user_id, session_id, patch_doc): + valid_patch = SessionDoc.create_patch(patch_doc) + + # check that document exists + assert (self.session_manager.get(user_id, session_id)) + + version = self.session_manager.update(session_id, valid_patch) + logging.info('Session {0} updated to version {1}'. + format(session_id, version)) + return version + + def replace_session(self, user_id, session_id, doc): + # check that no document exists with + # same session_id and different user_id + try: + self.session_manager.get(user_id, session_id) + except exceptions.DocumentNotFound: + pass + + valid_doc = SessionDoc.update(doc, user_id, session_id) + + (created, version) = self.session_manager.insert(valid_doc, session_id) + if created: + logging.info('Session {0} created'.format(session_id)) + else: + logging.info('Session {0} replaced with version {1}'. + format(session_id, version)) + return version diff --git a/freezer_api/requirements.txt b/freezer_api/requirements.txt new file mode 100644 index 00000000..a66f96b7 --- /dev/null +++ b/freezer_api/requirements.txt @@ -0,0 +1,6 @@ +oslo.config +oslo.i18n +keystonemiddleware +jsonschema +elasticsearch +falcon>=0.2.0 diff --git a/freezer_api/setup.cfg b/freezer_api/setup.cfg index d8fef976..eb48c8ef 100644 --- a/freezer_api/setup.cfg +++ b/freezer_api/setup.cfg @@ -7,7 +7,7 @@ summary = OpenStack Backup and Restore Service description-file = README.rst -author = Fausto Marzi, Fabrizio Fresco, Fabrizio Vanni', +author = Fausto Marzi, Fabrizio Fresco, Fabrizio Vanni, author_email = fausto.marzi@hp.com, fabrizio.vanni@hp.com, fabrizio.fresco@hp.com home-page = https://github.com/stackforge/freezer diff --git a/freezer_api/tests/common.py b/freezer_api/tests/common.py index ef9f3211..3fdac8ff 100644 --- a/freezer_api/tests/common.py +++ b/freezer_api/tests/common.py @@ -34,10 +34,10 @@ fake_data_0_wrapped_backup_metadata = { 'user_name': 'asdffdsa', 'backup_metadata': { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 8475903425, - "level": 0, + "time_stamp": 8475903425, + "curr_backup_level": 0, "backup_session": 8475903425, "max_level": 5, "mode" : "fs", @@ -60,10 +60,10 @@ fake_data_0_wrapped_backup_metadata = { fake_data_0_backup_metadata = { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 8475903425, - "level": 0, + "time_stamp": 8475903425, + "curr_backup_level": 0, "backup_session": 8475903425, "max_level": 5, "mode": "fs", @@ -84,10 +84,10 @@ fake_data_0_backup_metadata = { } fake_malformed_data_0_backup_metadata = { - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 8475903425, - "level": 0, + "time_stamp": 8475903425, + "curr_backup_level": 0, "backup_session": 8475903425, "max_level": 5, "mode": "fs", @@ -123,10 +123,10 @@ fake_data_0_elasticsearch_hit = { "_type": "backups", "_source": { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 8475903425, - "level": 0, + "time_stamp": 8475903425, + "curr_backup_level": 0, "backup_session": 8475903425, "max_level": 5, "mode" : "fs", @@ -180,33 +180,57 @@ fake_job_0_elasticsearch_not_found = { "found": False } + fake_job_0 = { - "job_action": { - "action": "backup", - "mode": "fs", - "src_file": "/home/tylerdurden/project_mayhem", - "backup_name": "project_mayhem_backup", - "container": "my_backup_container" - }, - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 1234, - "status": "stop", - "schedule_date": "2015-06-02T16:20:00", - "schedule_interval": "2 days" - }, - "job_id": "e7181e5e-2c75-43f8-92c0-c037ae5f11e4", - "client_id": "mytenantid_myhostname", - "user_id": "f4db4da085f043059441565720b217c7", - "description": "test action 4" + "job_actions": [ + { + "freezer_action": { + "action": "backup", + "mode": "fs", + "src_file": "/home/tylerdurden/project_mayhem", + "backup_name": "project_mayhem_backup", + "container": "my_backup_container" + }, + "max_retries": 3, + "max_retries_interval": 60, + "mandatory": False + }, + { + "freezer_action": { + "action": "restore", + "mode": "fs", + "restore_abs_path": "/home/tylerdurden/project_mayhem", + "restore_from_host": "node_on_which_backup_was_made", + "backup_name": "project_mayhem_backup", + "container": "my_backup_container" + }, + "max_retries": 3, + "max_retries_interval": 60, + "mandatory": True + } + ], + "job_schedule": { + "time_created": 1234, + "time_started": 1234, + "time_ended": 1234, + "status": "stop", + "result": "success", + "schedule_date": "2015-06-02T16:20:00", + "schedule_interval": "2 days" + }, + "job_id": "e7181e5e-2c75-43f8-92c0-c037ae5f11e4", + "client_id": "mytenantid_myhostname", + "user_id": "f4db4da085f043059441565720b217c7", + "description": "test action 4" } def get_fake_job_0(): return copy.deepcopy(fake_job_0) def get_fake_job_1(): - return copy.deepcopy(fake_job_1) + job = copy.deepcopy(fake_job_0) + job["job_id"] = 'pqoqurioew' + return job fake_job_0_elasticsearch_found = { "_id": "e7181e5e-2c75-43f8-92c0-c037ae5f11e4", @@ -218,61 +242,16 @@ fake_job_0_elasticsearch_found = { } -fake_job_1 = { - "job_action": { - "action": "backup", - "mode": "fs", - "src_file": "/home/tylerdurden/project_mayhem", - "backup_name": "project_mayhem_backup", - "container": "my_backup_container", - }, - "job_schedule": { - "time_created": 1234, - "time_started": 1234, - "time_ended": 0, - "status": "invalid", - "schedule_time": "2015-06-02T16:20:00" - }, - "job_id": "1b05e367-7832-42df-850e-bc48eabee04e", - "client_id": "mytenantid_myhostname", - "user_id": "f4db4da085f043059441565720b217c7", - "description": "test action 4" -} - -# fake_action_1 = { -# "action_id": "1b05e367-7832-42df-850e-bc48eabee04e", -# "client_id": "mytenantid_myhostname", -# "description": "test action 4", -# "job": { -# "action": "restore", -# "backup-name": "project_mayhem_backup", -# "container": "my_backup_container", -# "max_cpu_priority": True, -# "restore-abs-path": "/home/tylerdurden/project_mayhem", -# "restore-from-host": "another_host" -# }, -# "status": "pending", -# "time_created": 1431100962, -# "time_end": 0, -# "time_start": 0 -# } -# -# fake_action_1_doc = { -# "action": fake_action_1, -# "user_id": "f4db4da085f043059441565720b217c7" -# } -# -# fake_data_1_wrapped_backup_metadata = { 'backup_id': 'freezer_container_alpha_important_data_backup_125235431_1', 'user_id': 'qwerty1234', 'user_name': 'asdffdsa', 'backup_metadata': { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 125235431, - "level": 1, + "time_stamp": 125235431, + "curr_backup_level": 1, "backup_session": 8475903425, "max_level": 5, "mode" : "fs", @@ -316,6 +295,117 @@ fake_client_entry_1 = { } +fake_action_0 = { + "freezer_action": + { + "action": "backup", + "mode": "fs", + "src_file": "/home/tylerdurden/project_mayhem", + "backup_name": "project_mayhem_backup", + "container": "my_backup_container", + }, + "exit_status": "success", + "max_retries": 3, + "max_retries_interval": 60, + "mandatory": True, + + "action_id": "qwerqwerqwerrewq", + "user_id": "user_id-is-provided-by-keystone" +} + + +fake_action_1 = { + "freezer_action": + { + "action": "backup", + "mode": "fs", + "src_file": "/home/tylerdurden/project_mayhem", + "backup_name": "project_mayhem_backup", + "container": "my_backup_container", + }, + "exit_status": "success", + "max_retries": 3, + "max_retries_interval": 60, + "mandatory": True, + + "action_id": "jk4lkjbn4r3k", + "user_id": "user_id-is-provided-by-keystone" +} + + +def get_fake_action_0(): + return copy.deepcopy(fake_action_0) + + +def get_fake_action_1(): + return copy.deepcopy(fake_action_1) + + +fake_session_0 = { + "session_id": 'turistidellademocrazia', + "session_tag": 5, + "description": 'some text here', + "hold_off": 60, + "schedule": { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "invalid", + "schedule_time": "2015-06-02T16:20:00" + }, + "jobs": { + 'venerescollataincorpodalolita': { + "client_id": 'bruco', + "status": 'running', + "start_time": 12344321, + }, + 'job_id_2': { + "client_id": "cocktail", + "status": 'completed', + "result": 'success', + "start_time": 123321, + "end_time": 123325, + } + }, + "time_start": 123412344, + "time_end": 432234432, + "status": "running", + "user_id": "califfo" +} + +fake_session_1 = { + "session_id": 'turistidellademocrazia', + "session_tag": 5, + "description": 'some text here', + "hold_off": 60, + "schedule": { + "time_created": 1234, + "time_started": 1234, + "time_ended": 0, + "status": "invalid", + "schedule_time": "2015-06-02T16:20:00" + }, + "jobs": { + 'venerescollataincorpodalolita': { + "client_id": 'bruco', + "status": 'running', + "start_time": 12344321, + } + }, + "time_start": 123412344, + "time_end": 432234432, + "status": "running", + "user_id": "califfo" +} + + +def get_fake_session_0(): + return copy.deepcopy(fake_session_0) + +def get_fake_session_1(): + return copy.deepcopy(fake_session_1) + + class FakeReqResp: def __init__(self, method='GET', body=''): diff --git a/freezer_api/tests/test_actions.py b/freezer_api/tests/test_actions.py new file mode 100644 index 00000000..a4158a5a --- /dev/null +++ b/freezer_api/tests/test_actions.py @@ -0,0 +1,148 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import unittest +from mock import Mock, patch + +import random +import falcon + +from common import * +from freezer_api.common.exceptions import * + +from freezer_api.api.v1 import actions as v1_actions + + +class TestActionsCollectionResource(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_action_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_actions.ActionsCollectionResource(self.mock_db) + + def test_on_get_return_empty_list(self): + self.mock_db.search_action.return_value = [] + expected_result = {'actions': []} + self.resource.on_get(self.mock_req, self.mock_req) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_get_return_correct_list(self): + self.mock_db.search_action.return_value = [get_fake_action_0(), get_fake_action_1()] + expected_result = {'actions': [get_fake_action_0(), get_fake_action_1()]} + self.resource.on_get(self.mock_req, self.mock_req) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_post_raises_when_missing_body(self): + self.mock_db.add_action.return_value = fake_action_0['action_id'] + self.assertRaises(BadDataFormat, self.resource.on_post, self.mock_req, self.mock_req) + + def test_on_post_inserts_correct_data(self): + action = get_fake_action_0() + self.mock_req.context['doc'] = action + self.mock_db.add_action.return_value = 'pjiofrdslaikfunr' + expected_result = {'action_id': 'pjiofrdslaikfunr'} + self.resource.on_post(self.mock_req, self.mock_req) + self.assertEqual(self.mock_req.status, falcon.HTTP_201) + self.assertEqual(self.mock_req.context['result'], expected_result) + # assigned_action_id = self.mock_req.context['doc']['action_id'] + # self.assertNotEqual(assigned_action_id, fake_action_0['action_id']) + +class TestActionsResource(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_action_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_actions.ActionsResource(self.mock_db) + + def test_create_resource(self): + self.assertIsInstance(self.resource, v1_actions.ActionsResource) + + def test_on_get_return_no_result_and_404_when_not_found(self): + self.mock_db.get_action.return_value = None + self.resource.on_get(self.mock_req, self.mock_req, fake_action_0['action_id']) + self.assertNotIn('result', self.mock_req.context) + self.assertEqual(self.mock_req.status, falcon.HTTP_404) + + def test_on_get_return_correct_data(self): + self.mock_db.get_action.return_value = get_fake_action_0() + self.resource.on_get(self.mock_req, self.mock_req, fake_action_0['action_id']) + result = self.mock_req.context['result'] + self.assertEqual(result, get_fake_action_0()) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_delete_removes_proper_data(self): + self.resource.on_delete(self.mock_req, self.mock_req, fake_action_0['action_id']) + result = self.mock_req.context['result'] + expected_result = {'action_id': fake_action_0['action_id']} + self.assertEquals(self.mock_req.status, falcon.HTTP_204) + self.assertEqual(result, expected_result) + + def test_on_patch_ok_with_some_fields(self): + new_version = random.randint(0, 99) + self.mock_db.update_action.return_value = new_version + patch_doc = {'some_field': 'some_value', + 'because': 'size_matters'} + self.mock_req.context['doc'] = patch_doc + + expected_patch = patch_doc.copy() + + expected_result = {'action_id': fake_action_0['action_id'], + 'version': new_version} + + self.resource.on_patch(self.mock_req, self.mock_req, fake_action_0['action_id']) + self.mock_db.update_action.assert_called_with( + user_id=fake_action_0['user_id'], + action_id=fake_action_0['action_id'], + patch_doc=patch_doc) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + + def test_on_post_ok(self): + new_version = random.randint(0, 99) + self.mock_db.replace_action.return_value = new_version + action = get_fake_action_0() + self.mock_req.context['doc'] = action + expected_result = {'action_id': fake_action_0['action_id'], + 'version': new_version} + + self.resource.on_post(self.mock_req, self.mock_req, fake_action_0['action_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_201) + self.assertEqual(self.mock_req.context['result'], expected_result) + + def test_on_post_raises_when_db_replace_action_raises(self): + self.mock_db.replace_action.side_effect = AccessForbidden('regular test failure') + action = get_fake_action_0() + self.mock_req.context['doc'] = action + self.assertRaises(AccessForbidden, self.resource.on_post, + self.mock_req, + self.mock_req, + fake_action_0['action_id']) diff --git a/freezer_api/tests/test_elastic.py b/freezer_api/tests/test_elastic.py index 6e5944f0..84cb861b 100644 --- a/freezer_api/tests/test_elastic.py +++ b/freezer_api/tests/test_elastic.py @@ -51,8 +51,6 @@ class TypeManager(unittest.TestCase): ]}}}] self.assertEqual(q, expected_q) - - def test_get_ok(self): self.mock_es.get.return_value = fake_job_0_elasticsearch_found res = self.type_manager.get(user_id=fake_job_0_user_id, @@ -102,16 +100,28 @@ class TypeManager(unittest.TestCase): def test_insert_ok(self): self.mock_es.index.return_value = {'created': True, '_version': 15} - test_doc = {'test_key_412': 'test_value_412'} + test_doc = {'test_key_412': 'test_value_412', '_version': 5} res = self.type_manager.insert(doc=test_doc) self.assertEqual(res, (True, 15)) - self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None) + self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None, version=5) - def test_insert_fails(self): + def test_insert_raise_StorageEngineError_on_ES_Exception(self): self.mock_es.index.side_effect = Exception('regular test failure') - test_doc = {'test_key_412': 'test_value_412'} + test_doc = {'test_key_412': 'test_value_412', '_version': 5} self.assertRaises(StorageEngineError, self.type_manager.insert, doc=test_doc) - self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None) + self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None, version=5) + + def test_insert_raise_StorageEngineError_on_ES_TransportError_exception(self): + self.mock_es.index.side_effect = TransportError(500, 'regular test failure') + test_doc = {'test_key_412': 'test_value_412', '_version': 5} + self.assertRaises(StorageEngineError, self.type_manager.insert, doc=test_doc) + self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None, version=5) + + def test_insert_raise_DocumentExists_on_ES_TransportError409_exception(self): + self.mock_es.index.side_effect = TransportError(409, 'regular test failure') + test_doc = {'test_key_412': 'test_value_412', '_version': 5} + self.assertRaises(DocumentExists, self.type_manager.insert, doc=test_doc) + self.mock_es.index.assert_called_with(index='freezer', doc_type='base_doc_type', body=test_doc, id=None, version=5) def test_delete(self): doc_id='mydocid345' @@ -213,19 +223,26 @@ class JobTypeManager(unittest.TestCase): u'_version': 3 } res = self.job_manager.update(job_id=fake_job_0_job_id, - job_update_doc={'status': 'sleepy'}) + job_update_doc={'status': 'sleepy', '_version': 12}) self.assertEqual(res, 3) self.mock_es.update.assert_called_with(index=self.job_manager.index, doc_type=self.job_manager.doc_type, id=fake_job_0_job_id, - body={"doc": {'status': 'sleepy'}}) + body={"doc": {'status': 'sleepy'}}, + version=12) def test_update_raise_DocumentNotFound_when_not_found(self): - self.mock_es.update.side_effect = TransportError('regular test failure') + self.mock_es.update.side_effect = TransportError('regular test failure', 1) self.assertRaises(DocumentNotFound, self.job_manager.update, job_id=fake_job_0_job_id, job_update_doc={'status': 'sleepy'}) + def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): + self.mock_es.update.side_effect = TransportError(409, 'regular test failure') + self.assertRaises(DocumentExists, self.job_manager.update, + job_id=fake_job_0_job_id, + job_update_doc={'status': 'sleepy'}) + def test_update_raise_StorageEngineError_when_db_raises(self): self.mock_es.update.side_effect = Exception('regular test failure') self.assertRaises(StorageEngineError, self.job_manager.update, @@ -233,6 +250,125 @@ class JobTypeManager(unittest.TestCase): job_update_doc={'status': 'sleepy'}) +class ActionTypeManager(unittest.TestCase): + + def setUp(self): + self.mock_es = Mock() + self.action_manager = elastic.ActionTypeManager(self.mock_es, 'actions') + + def test_get_search_query(self): + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + q = self.action_manager.get_search_query('my_user_id', 'my_doc_id', search=my_search) + expected_q = {'query': {'filtered': {'filter': + {'bool': + {'must': + [{'term': {'user_id': 'my_user_id'}}, + {'query': + {'bool': + {'must_not': + [], + 'must': + [{'match': {'some_field': 'some text'}}, + {'match': {'description': 'some other text'}}]}}}, + {'term': {'action_id': 'my_doc_id'}} + ]}}}}} + self.assertEqual(q, expected_q) + + def test_update_ok(self): + self.mock_es.update.return_value = { + u'_id': u'd6c1e00d-b9c1-4eb3-8219-1e83c02af101', + u'_index': u'freezer', + u'_type': u'actions', + u'_version': 3 + } + res = self.action_manager.update(action_id='poiuuiop7890', + action_update_doc={'status': 'sleepy', '_version': 12}) + self.assertEqual(res, 3) + self.mock_es.update.assert_called_with(index=self.action_manager.index, + doc_type=self.action_manager.doc_type, + id='poiuuiop7890', + body={"doc": {'status': 'sleepy'}}, + version=12) + + def test_update_raise_DocumentNotFound_when_not_found(self): + self.mock_es.update.side_effect = TransportError('regular test failure', 1) + self.assertRaises(DocumentNotFound, self.action_manager.update, + action_id='asdfsadf', + action_update_doc={'status': 'sleepy'}) + + def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): + self.mock_es.update.side_effect = TransportError(409, 'regular test failure') + self.assertRaises(DocumentExists, self.action_manager.update, + action_id='pepepepepe2321', + action_update_doc={'status': 'sleepy'}) + + def test_update_raise_StorageEngineError_when_db_raises(self): + self.mock_es.update.side_effect = Exception('regular test failure') + self.assertRaises(StorageEngineError, self.action_manager.update, + action_id='pepepepepe2321', + action_update_doc={'status': 'sleepy'}) + + +class SessionTypeManager(unittest.TestCase): + + def setUp(self): + self.mock_es = Mock() + self.session_manager = elastic.SessionTypeManager(self.mock_es, 'sessions') + + def test_get_search_query(self): + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + q = self.session_manager.get_search_query('my_user_id', 'my_doc_id', search=my_search) + expected_q = {'query': {'filtered': {'filter': + {'bool': + {'must': + [{'term': {'user_id': 'my_user_id'}}, + {'query': + {'bool': + {'must_not': + [], + 'must': + [{'match': {'some_field': 'some text'}}, + {'match': {'description': 'some other text'}}]}}}, + {'term': {'session_id': 'my_doc_id'}} + ]}}}}} + self.assertEqual(q, expected_q) + + def test_update_ok(self): + self.mock_es.update.return_value = { + u'_id': u'd6c1e00d-b9c1-4eb3-8219-1e83c02af101', + u'_index': u'freezer', + u'_type': u'actions', + u'_version': 3 + } + res = self.session_manager.update(session_id='poiuuiop7890', + session_update_doc={'status': 'sleepy', '_version': 12}) + self.assertEqual(res, 3) + self.mock_es.update.assert_called_with(index=self.session_manager.index, + doc_type=self.session_manager.doc_type, + id='poiuuiop7890', + body={"doc": {'status': 'sleepy'}}, + version=12) + + def test_update_raise_DocumentNotFound_when_not_found(self): + self.mock_es.update.side_effect = TransportError('regular test failure', 1) + self.assertRaises(DocumentNotFound, self.session_manager.update, + session_id='asdfsadf', + session_update_doc={'status': 'sleepy'}) + + def test_update_raise_DocumentExists_when_elasticsearch_returns_409(self): + self.mock_es.update.side_effect = TransportError(409, 'regular test failure') + self.assertRaises(DocumentExists, self.session_manager.update, + session_id='pepepepepe2321', + session_update_doc={'status': 'sleepy'}) + + def test_update_raise_StorageEngineError_when_db_raises(self): + self.mock_es.update.side_effect = Exception('regular test failure') + self.assertRaises(StorageEngineError, self.session_manager.update, + session_id='pepepepepe2321', + session_update_doc={'status': 'sleepy'}) + class TestElasticSearchEngine_backup(unittest.TestCase): @@ -569,3 +705,270 @@ class TestElasticSearchEngine_job(unittest.TestCase): job_id=fake_job_0_job_id, doc=get_fake_job_0()) self.assertEqual(res, 3) + + +class TestElasticSearchEngine_action(unittest.TestCase): + + @patch('freezer_api.storage.elastic.logging') + @patch('freezer_api.storage.elastic.elasticsearch') + def setUp(self, mock_elasticsearch, mock_logging): + mock_elasticsearch.Elasticsearch.return_value = Mock() + self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + self.eng.action_manager = Mock() + + def test_get_action_userid_and_action_id_return_doc(self): + self.eng.action_manager.get.return_value = get_fake_action_0() + res = self.eng.get_action(user_id=fake_action_0['user_id'], + action_id=fake_action_0['action_id']) + self.assertEqual(res, fake_action_0) + self.eng.action_manager.get.assert_called_with( + fake_action_0['user_id'], + fake_action_0['action_id']) + + def test_get_action_userid_and_action_id_return_none(self): + self.eng.action_manager.get.return_value = None + res = self.eng.get_action(user_id=fake_action_0['user_id'], + action_id=fake_action_0['action_id']) + self.assertEqual(res, None) + self.eng.action_manager.get.assert_called_with( + fake_action_0['user_id'], + fake_action_0['action_id']) + + def test_get_action_with_userid_and_search_return_list(self): + self.eng.action_manager.search.return_value = \ + [fake_action_0, fake_action_0] + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + res = self.eng.search_action(user_id=fake_action_0['user_id'], + offset=6, limit=15, + search=my_search) + self.assertEqual(res, [fake_action_0, fake_action_0]) + self.eng.action_manager.search.assert_called_with( + fake_action_0['user_id'], + search=my_search, + limit=15, offset=6) + + def test_get_action_with_userid_and_search_return_empty_list(self): + self.eng.action_manager.search.return_value = [] + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + res = self.eng.search_action(user_id=fake_action_0['user_id'], + offset=6, limit=15, + search=my_search) + self.assertEqual(res, []) + self.eng.action_manager.search.assert_called_with( + fake_action_0['user_id'], + search=my_search, + limit=15, offset=6) + + @patch('freezer_api.storage.elastic.ActionDoc') + def test_add_action_ok(self, mock_actiondoc): + mock_actiondoc.create.return_value = get_fake_action_0() + self.eng.action_manager.insert.return_value = (True, 1) + res = self.eng.add_action(user_id=fake_action_0['user_id'], + doc=get_fake_action_0()) + self.assertEqual(res, fake_action_0['action_id']) + self.eng.action_manager.insert.assert_called_with(fake_action_0, + fake_action_0['action_id']) + + def test_add_action_raises_StorageEngineError_when_manager_insert_raises(self): + self.eng.action_manager.get.return_value = None + self.eng.action_manager.insert.side_effect = StorageEngineError('regular test failure') + self.assertRaises(StorageEngineError, self.eng.add_action, + user_id=fake_action_0['user_id'], + doc=get_fake_action_0()) + + def test_delete_action_ok(self): + self.eng.action_manager.delete.return_value = fake_action_0['action_id'] + res = self.eng.delete_action(user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id']) + self.assertEqual(res, fake_action_0['action_id']) + + def test_delete_client_raises_StorageEngineError_when_es_delete_raises(self): + self.eng.action_manager.delete.side_effect = StorageEngineError() + self.assertRaises(StorageEngineError, self.eng.delete_action, + user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id']) + + def test_update_action_raises_DocumentNotFound_when_doc_not_exists(self): + self.eng.action_manager.get.side_effect = DocumentNotFound('regular test failure') + patch = {'action_id': 'black_milk'} + self.assertRaises(DocumentNotFound, self.eng.update_action, + user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id'], + patch_doc=patch) + + def test_update_action_raises_DocumentNotFound_when_update_raises_DocumentNotFound(self): + self.eng.action_manager.get.return_value = get_fake_action_0() + patch = {'action_id': 'black_milk'} + self.eng.action_manager.update.side_effect = DocumentNotFound('regular test failure') + self.assertRaises(DocumentNotFound, self.eng.update_action, + user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id'], + patch_doc=patch) + + def test_update_action_returns_new_doc_version(self): + self.eng.action_manager.get.return_value = get_fake_action_0() + patch = {'action_id': 'group_four'} + self.eng.action_manager.update.return_value = 11 + res = self.eng.update_action(user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id'], + patch_doc=patch) + self.assertEqual(res, 11) + + def test_replace_action_raises_AccessForbidden_when_action_manager_raises_AccessForbidden(self): + self.eng.action_manager.get.side_effect = AccessForbidden('regular test failure') + self.eng.action_manager.insert.return_value = (True, 3) + self.assertRaises(AccessForbidden, self.eng.replace_action, + user_id=fake_action_0['action_id'], + action_id=fake_action_0['action_id'], + doc=get_fake_action_0()) + + # def test_replace_action_returns_ok_when_doc_is_new(self): + # self.eng.action_manager.get.side_effect = DocumentNotFound('regular test failure') + # self.eng.action_manager.insert.return_value = (True, 1) + # res = self.eng.replace_action(user_id=fake_action_0['action_id'], + # action_id=fake_action_0['action_id'], + # doc=get_fake_action_0()) + # self.assertEqual(res, 1) + + # def test_replace_action_returns_version_1_when_doc_is_overwritten(self): + # self.eng.action_manager.get.return_value = get_fake_action_0() + # self.eng.action_manager.insert.return_value = (False, 3) + # res = self.eng.replace_action(user_id=fake_action_0['action_id'], + # action_id=fake_action_0['action_id'], + # doc=get_fake_action_0()) + # self.assertEqual(res, 3) + + +class TestElasticSearchEngine_session(unittest.TestCase): + + @patch('freezer_api.storage.elastic.logging') + @patch('freezer_api.storage.elastic.elasticsearch') + def setUp(self, mock_elasticsearch, mock_logging): + mock_elasticsearch.Elasticsearch.return_value = Mock() + self.eng = elastic.ElasticSearchEngine('http://elasticservaddr:1997') + self.eng.session_manager = Mock() + + def test_get_session_userid_and_session_id_return_doc(self): + self.eng.session_manager.get.return_value = get_fake_session_0() + res = self.eng.get_session(user_id=fake_session_0['user_id'], + session_id=fake_session_0['session_id']) + self.assertEqual(res, fake_session_0) + self.eng.session_manager.get.assert_called_with(fake_session_0['user_id'], + fake_session_0['session_id']) + + def test_get_session_userid_and_session_id_return_none(self): + self.eng.session_manager.get.return_value = None + res = self.eng.get_session(user_id=fake_session_0['user_id'], + session_id=fake_session_0['session_id']) + self.assertEqual(res, None) + self.eng.session_manager.get.assert_called_with( + fake_session_0['user_id'], + fake_session_0['session_id']) + + def test_get_session_with_userid_and_search_return_list(self): + self.eng.session_manager.search.return_value = \ + [fake_session_0, fake_session_0] + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + res = self.eng.search_session(user_id=fake_session_0['user_id'], + offset=6, limit=15, + search=my_search) + self.assertEqual(res, [fake_session_0, fake_session_0]) + self.eng.session_manager.search.assert_called_with( + fake_session_0['user_id'], + search=my_search, + limit=15, offset=6) + + def test_get_session_with_userid_and_search_return_empty_list(self): + self.eng.session_manager.search.return_value = [] + my_search = {'match': [{'some_field': 'some text'}, + {'description': 'some other text'}]} + res = self.eng.search_session(user_id=fake_session_0['user_id'], + offset=6, limit=15, + search=my_search) + self.assertEqual(res, []) + self.eng.session_manager.search.assert_called_with( + fake_session_0['user_id'], + search=my_search, + limit=15, offset=6) + + @patch('freezer_api.storage.elastic.SessionDoc') + def test_add_session_ok(self, mock_sessiondoc): + mock_sessiondoc.create.return_value = get_fake_session_0() + self.eng.session_manager.insert.return_value = (True, 1) + res = self.eng.add_session(user_id=fake_session_0['user_id'], + doc=get_fake_session_0()) + self.assertEqual(res, fake_session_0['session_id']) + self.eng.session_manager.insert.assert_called_with(fake_session_0, + fake_session_0['session_id']) + + def test_add_session_raises_StorageEngineError_when_manager_insert_raises(self): + self.eng.session_manager.get.return_value = None + self.eng.session_manager.insert.side_effect = StorageEngineError('regular test failure') + self.assertRaises(StorageEngineError, self.eng.add_session, + user_id=fake_session_0['user_id'], + doc=get_fake_session_0()) + + def test_delete_session_ok(self): + self.eng.session_manager.delete.return_value = fake_session_0['session_id'] + res = self.eng.delete_session(user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id']) + self.assertEqual(res, fake_session_0['session_id']) + + def test_delete_client_raises_StorageEngineError_when_es_delete_raises(self): + self.eng.session_manager.delete.side_effect = StorageEngineError() + self.assertRaises(StorageEngineError, self.eng.delete_session, + user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id']) + + def test_update_session_raises_DocumentNotFound_when_doc_not_exists(self): + self.eng.session_manager.get.side_effect = DocumentNotFound('regular test failure') + patch = {'session_id': 'black_milk'} + self.assertRaises(DocumentNotFound, self.eng.update_session, + user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + patch_doc=patch) + + def test_update_session_raises_DocumentNotFound_when_update_raises_DocumentNotFound(self): + self.eng.session_manager.get.return_value = get_fake_session_0() + patch = {'session_id': 'black_milk'} + self.eng.session_manager.update.side_effect = DocumentNotFound('regular test failure') + self.assertRaises(DocumentNotFound, self.eng.update_session, + user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + patch_doc=patch) + + def test_update_session_returns_new_doc_version(self): + self.eng.session_manager.get.return_value = get_fake_session_0() + patch = {'session_id': 'group_four'} + self.eng.session_manager.update.return_value = 11 + res = self.eng.update_session(user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + patch_doc=patch) + self.assertEqual(res, 11) + + def test_replace_session_raises_AccessForbidden_when_session_manager_raises_AccessForbidden(self): + self.eng.session_manager.get.side_effect = AccessForbidden('regular test failure') + self.eng.session_manager.insert.return_value = (True, 3) + self.assertRaises(AccessForbidden, self.eng.replace_session, + user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + doc=get_fake_session_0()) + + def test_replace_session_returns_ok_when_doc_is_new(self): + self.eng.session_manager.get.side_effect = DocumentNotFound('regular test failure') + self.eng.session_manager.insert.return_value = (True, 1) + res = self.eng.replace_session(user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + doc=get_fake_session_0()) + self.assertEqual(res, 1) + + def test_replace_session_returns_version_1_when_doc_is_overwritten(self): + self.eng.session_manager.get.return_value = get_fake_session_0() + self.eng.session_manager.insert.return_value = (False, 3) + res = self.eng.replace_session(user_id=fake_session_0['session_id'], + session_id=fake_session_0['session_id'], + doc=get_fake_session_0()) + self.assertEqual(res, 3) diff --git a/freezer_api/tests/test_homedoc.py b/freezer_api/tests/test_homedoc.py index 89821ec0..d86d51fc 100644 --- a/freezer_api/tests/test_homedoc.py +++ b/freezer_api/tests/test_homedoc.py @@ -1,6 +1,6 @@ """Freezer swift.py related tests -Copyright 2014 Hewlett-Packard +Copyright 2015 Hewlett-Packard Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/freezer_api/tests/test_jobs.py b/freezer_api/tests/test_jobs.py index 46888512..c7f796ad 100644 --- a/freezer_api/tests/test_jobs.py +++ b/freezer_api/tests/test_jobs.py @@ -1,3 +1,25 @@ +"""Freezer swift.py related tests + +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + import unittest from mock import Mock, patch diff --git a/freezer_api/tests/test_sessions.py b/freezer_api/tests/test_sessions.py new file mode 100644 index 00000000..19dcc454 --- /dev/null +++ b/freezer_api/tests/test_sessions.py @@ -0,0 +1,435 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import unittest +from mock import Mock, patch + +import random +import falcon + +from common import * +from freezer_api.common.exceptions import * + +from freezer_api.api.v1 import sessions as v1_sessions + + +class TestSessionsCollectionResource(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_session_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_sessions.SessionsCollectionResource(self.mock_db) + + def test_on_get_return_empty_list(self): + self.mock_db.search_session.return_value = [] + expected_result = {'sessions': []} + self.resource.on_get(self.mock_req, self.mock_req) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_get_return_correct_list(self): + self.mock_db.search_session.return_value = [get_fake_session_0(), get_fake_session_1()] + expected_result = {'sessions': [get_fake_session_0(), get_fake_session_1()]} + self.resource.on_get(self.mock_req, self.mock_req) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_post_raises_when_missing_body(self): + self.mock_db.add_session.return_value = fake_session_0['session_id'] + self.assertRaises(BadDataFormat, self.resource.on_post, self.mock_req, self.mock_req) + + def test_on_post_inserts_correct_data(self): + session = get_fake_session_0() + self.mock_req.context['doc'] = session + self.mock_db.add_session.return_value = 'pjiofrdslaikfunr' + expected_result = {'session_id': 'pjiofrdslaikfunr'} + self.resource.on_post(self.mock_req, self.mock_req) + self.assertEqual(self.mock_req.status, falcon.HTTP_201) + self.assertEqual(self.mock_req.context['result'], expected_result) + # assigned_session_id = self.mock_req.context['doc']['session_id'] + # self.assertNotEqual(assigned_session_id, fake_session_0['session_id']) + +class TestSessionsResource(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_session_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_sessions.SessionsResource(self.mock_db) + + def test_create_resource(self): + self.assertIsInstance(self.resource, v1_sessions.SessionsResource) + + def test_on_get_return_no_result_and_404_when_not_found(self): + self.mock_db.get_session.return_value = None + self.resource.on_get(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertNotIn('result', self.mock_req.context) + self.assertEqual(self.mock_req.status, falcon.HTTP_404) + + def test_on_get_return_correct_data(self): + self.mock_db.get_session.return_value = get_fake_session_0() + self.resource.on_get(self.mock_req, self.mock_req, fake_session_0['session_id']) + result = self.mock_req.context['result'] + self.assertEqual(result, get_fake_session_0()) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + + def test_on_delete_removes_proper_data(self): + self.resource.on_delete(self.mock_req, self.mock_req, fake_session_0['session_id']) + result = self.mock_req.context['result'] + expected_result = {'session_id': fake_session_0['session_id']} + self.assertEquals(self.mock_req.status, falcon.HTTP_204) + self.assertEqual(result, expected_result) + + def test_on_patch_ok_with_some_fields(self): + new_version = random.randint(0, 99) + self.mock_db.update_session.return_value = new_version + patch_doc = {'some_field': 'some_value', + 'because': 'size_matters'} + self.mock_req.context['doc'] = patch_doc + + expected_patch = patch_doc.copy() + + expected_result = {'session_id': fake_session_0['session_id'], + 'version': new_version} + + self.resource.on_patch(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.mock_db.update_session.assert_called_with( + user_id=fake_session_0['user_id'], + session_id=fake_session_0['session_id'], + patch_doc=patch_doc) + self.assertEqual(self.mock_req.status, falcon.HTTP_200) + result = self.mock_req.context['result'] + self.assertEqual(result, expected_result) + + def test_on_post_ok(self): + new_version = random.randint(0, 99) + self.mock_db.replace_session.return_value = new_version + session = get_fake_session_0() + self.mock_req.context['doc'] = session + expected_result = {'session_id': fake_session_0['session_id'], + 'version': new_version} + + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_201) + self.assertEqual(self.mock_req.context['result'], expected_result) + + def test_on_post_raises_when_db_replace_session_raises(self): + self.mock_db.replace_session.side_effect = AccessForbidden('regular test failure') + session = get_fake_session_0() + self.mock_req.context['doc'] = session + self.assertRaises(AccessForbidden, self.resource.on_post, + self.mock_req, + self.mock_req, + fake_session_0['session_id']) + + +class TestSessionsAction(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_session_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_sessions.SessionsAction(self.mock_db) + + def test_create_resource(self): + self.assertIsInstance(self.resource, v1_sessions.SessionsAction) + + def test_on_post_raises_when_unable_to_read_action_from_body(self): + self.mock_req.context['doc'] = {} + self.assertRaises(BadDataFormat, self.resource.on_post, + self.mock_req, + self.mock_req, + fake_session_0['session_id']) + + def test_on_post_start_action_ok(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + + action = {"start": { + "job_id": 'job_id_2', + "current_tag": 5 + }} + self.mock_req.context['doc'] = action + expected_result = {'result': 'success', + 'session_tag': 6} + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_202) + self.assertEqual(self.mock_req.context['result'], expected_result) + + def test_on_post_start_action_raises_BadDataFormat_when_job_not_in_session(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + action = {"start": { + "job_id": 'missedme', + "current_tag": 5 + }} + self.mock_req.context['doc'] = action + self.assertRaises(BadDataFormat, self.resource.on_post, self.mock_req, + self.mock_req, fake_session_0['session_id']) + + def test_on_post_start_action_raises_BadDataFormat_when_current_tag_too_high(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + action = {"start": { + "job_id": 'missedme', + "current_tag": 6 + }} + self.mock_req.context['doc'] = action + self.assertRaises(BadDataFormat, self.resource.on_post, self.mock_req, + self.mock_req, fake_session_0['session_id']) + + def test_on_post_end_action_ok(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + action = {"end": { + "job_id": 'job_id_2', + "current_tag": 5, + "result": "success" + }} + self.mock_req.context['doc'] = action + expected_result = {'result': 'success', + 'session_tag': 5} + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_202) + self.assertEqual(self.mock_req.context['result'], expected_result) + + def test_on_post_end_action_raises_BadDataFormat_when_job_not_in_session(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + action = {"end": { + "job_id": 'ahahahahah', + "current_tag": 5, + "result": "success" + }} + self.mock_req.context['doc'] = action + self.assertRaises(BadDataFormat, self.resource.on_post, self.mock_req, + self.mock_req, fake_session_0['session_id']) + + def test_on_post_raises_MethodNotImplemented_when_methon_not_implemented(self): + new_version = random.randint(0, 99) + self.mock_db.get_session.return_value = get_fake_session_0() + self.mock_db.update_session.return_value = new_version + action = {"method_not_implemented": { + "job_id": 'ahahahahah', + "current_tag": 5, + "result": "success" + }} + self.mock_req.context['doc'] = action + self.assertRaises(MethodNotImplemented, self.resource.on_post, self.mock_req, + self.mock_req, fake_session_0['session_id']) + + @patch('freezer_api.api.v1.sessions.time') + def test_on_post_start_succeeds_in_holdoff_if_tag_needs_not_increment(self, mock_time): + mock_time.time.return_value = 1000 + new_version = random.randint(0, 99) + session_doc = get_fake_session_0() + session_doc['time_start'] = 999 + self.mock_db.get_session.return_value = session_doc + self.mock_db.update_session.return_value = new_version + action = {"start": { + "job_id": 'job_id_2', + "current_tag": 4 + }} + self.mock_req.context['doc'] = action + expected_result = {'result': 'success', + 'session_tag': 5} + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_202) + self.assertEqual(self.mock_req.context['result'], expected_result) + + @patch('freezer_api.api.v1.sessions.time') + def test_on_post_start_replies_holdoff_if_tag_would_increment(self, mock_time): + mock_time.time.return_value = 1000 + new_version = random.randint(0, 99) + session_doc = get_fake_session_0() + session_doc['time_start'] = 999 + self.mock_db.get_session.return_value = session_doc + self.mock_db.update_session.return_value = new_version + action = {"start": { + "job_id": 'job_id_2', + "current_tag": 5 + }} + self.mock_req.context['doc'] = action + expected_result = {'result': 'hold-off', + 'session_tag': 5} + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_202) + self.assertEqual(self.mock_req.context['result'], expected_result) + + @patch('freezer_api.api.v1.sessions.time') + def test_on_post_start_outofholdoff_replies_outofsync_when_tag_too_low(self, mock_time): + mock_time.time.return_value = 2000 + new_version = random.randint(0, 99) + session_doc = get_fake_session_0() + session_doc['time_start'] = 999 + self.mock_db.get_session.return_value = session_doc + self.mock_db.update_session.return_value = new_version + action = {"start": { + "job_id": 'job_id_2', + "current_tag": 2 + }} + self.mock_req.context['doc'] = action + expected_result = {'result': 'out-of-sync', + 'session_tag': 5} + self.resource.on_post(self.mock_req, self.mock_req, fake_session_0['session_id']) + self.assertEqual(self.mock_req.status, falcon.HTTP_202) + self.assertEqual(self.mock_req.context['result'], expected_result) + + +class TestSessions(unittest.TestCase): + + def setUp(self): + self.session_doc = {} + self.session = v1_sessions.Session(self.session_doc) + + def test_create_resource(self): + self.assertIsInstance(self.session, v1_sessions.Session) + + def test_overall_result_running(self): + self.session_doc['jobs'] = {'job1': {'status': 'completed', + 'result': 'success'}, + 'job2': {'status': 'running', + 'result': ''}} + res = self.session.get_job_overall_result() + self.assertEquals(res, 'running') + + def test_overall_result_fail(self): + self.session_doc['jobs'] = {'job1': {'status': 'completed', + 'result': 'success'}, + 'job2': {'status': 'completed', + 'result': 'fail'}} + res = self.session.get_job_overall_result() + self.assertEquals(res, 'fail') + + def test_overall_result_success(self): + self.session_doc['jobs'] = {'job1': {'status': 'completed', + 'result': 'success'}, + 'job2': {'status': 'completed', + 'result': 'success'}} + res = self.session.get_job_overall_result() + self.assertEquals(res, 'success') + + + + +class TestSessionsJobs(unittest.TestCase): + + def setUp(self): + self.mock_db = Mock() + self.mock_req = Mock() + self.mock_req.get_header.return_value = fake_session_0['user_id'] + self.mock_req.context = {} + self.mock_req.status = falcon.HTTP_200 + self.resource = v1_sessions.SessionsJob(self.mock_db) + + def test_create_resource(self): + self.assertIsInstance(self.resource, v1_sessions.SessionsJob) + + def test_on_put_adds_job_to_session_jobs(self): + session = get_fake_session_0() + job = get_fake_job_0() + job_info = {job['job_id']: {'client_id': job['client_id'], + 'status': job['job_schedule']['status'], + 'result': job['job_schedule']['result'], + 'time_started': job['job_schedule']['time_started'], + 'time_ended': job['job_schedule']['time_ended']}} + session_update_doc = {'jobs': job_info} + + self.mock_db.get_session.return_value = session + self.mock_db.get_job.return_value = job + + self.resource.on_put(self.mock_req, self.mock_req, + session['session_id'], + job['job_id']) + self.mock_db.update_session.assert_called_with(user_id=session['user_id'], + session_id=session['session_id'], + patch_doc=session_update_doc) + + def test_on_put_updates_job_with_session_info(self): + session = get_fake_session_0() + job = get_fake_job_0() + + self.mock_db.get_session.return_value = session + self.mock_db.get_job.return_value = job + + job_update_doc = { + 'session_id': session['session_id'], + 'session_tag': session['session_tag'], + 'job_schedule': session['schedule'] + } + + self.resource.on_put(self.mock_req, self.mock_req, + session['session_id'], + job['job_id']) + self.mock_db.update_job.assert_called_with(user_id=session['user_id'], + job_id=job['job_id'], + patch_doc=job_update_doc) + + def test_on_delete_removes_job_from_session_jobs(self): + session = get_fake_session_0() + updated_session = get_fake_session_1() + job = get_fake_job_0() + + self.mock_db.get_session.return_value = session + self.mock_db.get_job.return_value = job + + self.resource.on_delete(self.mock_req, self.mock_req, + session['session_id'], + 'job_id_2') + + self.mock_db.replace_session.assert_called_with(user_id=session['user_id'], + session_id=session['session_id'], + doc=updated_session) + + def test_on_delete_removes_session_info_from_job_and_stops_job(self): + session = get_fake_session_0() + job = get_fake_job_0() + + self.mock_db.get_session.return_value = session + self.mock_db.get_job.return_value = job + + job_update_doc = { + 'session_id': '', + 'session_tag': '', + 'job_event': 'stop' + } + + self.resource.on_delete(self.mock_req, self.mock_req, + session['session_id'], + job['job_id']) + + self.mock_db.update_job.assert_called_with(user_id=session['user_id'], + job_id=job['job_id'], + patch_doc=job_update_doc) diff --git a/freezer_api/tests/test_utils.py b/freezer_api/tests/test_utils.py index a69722ef..7325c573 100644 --- a/freezer_api/tests/test_utils.py +++ b/freezer_api/tests/test_utils.py @@ -30,10 +30,10 @@ from common import * DATA_backup_metadata = { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 12341234, - "level": 0, + "time_stamp": 12341234, + "curr_backup_level": 0, "backup_session": 12341234, "max_level": 5, "mode" : "fs", @@ -65,10 +65,10 @@ DATA_wrapped_backup_metadata = { 'backup_id': DATA_backup_id, 'backup_medatada': { "container": "freezer_container", - "host_name": "alpha", + "hostname": "alpha", "backup_name": "important_data_backup", - "timestamp": 12341234, - "level": 0, + "time_stamp": 12341234, + "curr_backup_level": 0, "backup_session": 12341234, "max_level": 5, "mode": "fs", @@ -184,3 +184,113 @@ class TestJobDoc(unittest.TestCase): self.assertRaises(BadDataFormat, utils.JobDoc.create, job_doc, 'dude') +class TestActionDoc(unittest.TestCase): + + def test_validate_ok_when_data_ok(self): + action_doc = get_fake_action_0() + res = utils.ActionDoc.validate(action_doc) + self.assertIsNone(res) + + def test_validate_raises_BadDataFormat_when_doc_has_no_actionid(self): + action_doc = get_fake_action_0() + action_doc.pop('action_id') + self.assertRaises(BadDataFormat, utils.ActionDoc.validate, action_doc) + + def test_validate_raises_BadDataFormat_when_doc_has_no_userid(self): + action_doc = get_fake_action_0() + action_doc.pop('user_id') + self.assertRaises(BadDataFormat, utils.ActionDoc.validate, action_doc) + + def test_validate_raises_BadDataFormat_when_doc_has_invalid_field(self): + action_doc = get_fake_action_0() + action_doc['action_id'] = 44 + self.assertRaises(BadDataFormat, utils.ActionDoc.validate, action_doc) + + def test_validate_patch_raises_when_doc_has_invalid_field(self): + action_doc = get_fake_action_0() + action_doc['action_id'] = 44 + self.assertRaises(BadDataFormat, utils.ActionDoc.validate_patch, action_doc) + + def test_createpatch_pops_actionid_and_userid(self): + action_doc = get_fake_action_0() + res_doc = utils.ActionDoc.create_patch(action_doc) + self.assertFalse('action_id' in res_doc) + self.assertFalse('user_id' in res_doc) + + def test_createpatch_raises_BadDataFormat_when_patch_has_invalid_field(self): + action_doc = get_fake_action_0() + action_doc['action'] = 44 + self.assertRaises(BadDataFormat, utils.ActionDoc.create_patch, action_doc) + + @patch('freezer_api.common.utils.uuid') + def test_create_inserts_correct_uuid(self, mock_uuid): + mock_uuid.uuid4.return_value = mock_uuid + mock_uuid.hex = 'hotforteacher' + action_doc = get_fake_action_0() + res_doc = utils.ActionDoc.create(action_doc, 'dude') + self.assertEqual(res_doc['user_id'], 'dude') + self.assertEqual(res_doc['action_id'], 'hotforteacher') + + @patch('freezer_api.common.utils.uuid') + def test_create_raises_BadDataFormat_when_isvalid_fails(self, mock_uuid): + mock_uuid.uuid4.return_value = mock_uuid + mock_uuid.hex = 'hotforteacher' + action_doc = get_fake_action_0() + action_doc['action'] = 44 + self.assertRaises(BadDataFormat, utils.ActionDoc.create, action_doc, 'dude') + + +class TestSessionDoc(unittest.TestCase): + + def test_validate_ok_when_data_ok(self): + session_doc = get_fake_session_0() + res = utils.SessionDoc.validate(session_doc) + self.assertIsNone(res) + + def test_validate_raises_BadDataFormat_when_doc_has_no_sessionid(self): + session_doc = get_fake_session_0() + session_doc.pop('session_id') + self.assertRaises(BadDataFormat, utils.SessionDoc.validate, session_doc) + + def test_validate_raises_BadDataFormat_when_doc_has_no_userid(self): + session_doc = get_fake_session_0() + session_doc.pop('user_id') + self.assertRaises(BadDataFormat, utils.SessionDoc.validate, session_doc) + + def test_validate_raises_BadDataFormat_when_doc_has_invalid_field(self): + session_doc = get_fake_session_0() + session_doc['session_id'] = 44 + self.assertRaises(BadDataFormat, utils.SessionDoc.validate, session_doc) + + def test_validate_patch_raises_when_doc_has_invalid_field(self): + session_doc = get_fake_session_0() + session_doc['session_id'] = 44 + self.assertRaises(BadDataFormat, utils.SessionDoc.validate_patch, session_doc) + + def test_createpatch_pops_sessionid_and_userid(self): + session_doc = get_fake_session_0() + res_doc = utils.SessionDoc.create_patch(session_doc) + self.assertFalse('session_id' in res_doc) + self.assertFalse('user_id' in res_doc) + + def test_createpatch_raises_BadDataFormat_when_patch_has_invalid_field(self): + session_doc = get_fake_session_0() + session_doc['session_tag'] = 'ouch' + self.assertRaises(BadDataFormat, utils.SessionDoc.create_patch, session_doc) + + @patch('freezer_api.common.utils.uuid') + def test_create_inserts_correct_uuid(self, mock_uuid): + mock_uuid.uuid4.return_value = mock_uuid + mock_uuid.hex = 'hotforteacher' + session_doc = get_fake_session_0() + res_doc = utils.SessionDoc.create(session_doc, 'dude') + self.assertEqual(res_doc['user_id'], 'dude') + self.assertEqual(res_doc['session_id'], 'hotforteacher') + + @patch('freezer_api.common.utils.uuid') + def test_create_raises_BadDataFormat_when_isvalid_fails(self, mock_uuid): + mock_uuid.uuid4.return_value = mock_uuid + mock_uuid.hex = 'hotforteacher' + session_doc = get_fake_session_0() + session_doc['time_started'] = 'ouch' + self.assertRaises(BadDataFormat, utils.SessionDoc.create, session_doc, 'dude') diff --git a/freezer_api/tox.ini b/freezer_api/tox.ini index 8c57e03c..ba298c40 100644 --- a/freezer_api/tox.ini +++ b/freezer_api/tox.ini @@ -14,6 +14,7 @@ deps = falcon keystonemiddleware elasticsearch + jsonschema install_command = pip install -U {opts} {packages} setenv = VIRTUAL_ENV={envdir} diff --git a/requirements.txt b/requirements.txt index 201797e2..00373a5d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,8 @@ python-novaclient>=2.21.0 docutils>=0.8.1 pymysql pymongo +apscheduler +pep3143daemon [testing] pytest diff --git a/setup.py b/setup.py index ebcb29f8..cc0ba3e8 100644 --- a/setup.py +++ b/setup.py @@ -92,7 +92,7 @@ setup( }, entry_points={ 'console_scripts': [ - 'freezerc=freezer.main:freezer_main' + 'freezer-scheduler=freezer.scheduler.freezer_scheduler:main' ] }, data_files=[('freezer/scripts', ['freezer/scripts/vss.ps1']), diff --git a/tests/test_apiclient_actions.py b/tests/test_apiclient_actions.py new file mode 100644 index 00000000..ed93e3a9 --- /dev/null +++ b/tests/test_apiclient_actions.py @@ -0,0 +1,132 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import unittest +from mock import Mock, patch + +from freezer.apiclient import exceptions +from freezer.apiclient import actions + + +class TestActionManager(unittest.TestCase): + + def setUp(self): + self.mock_client = Mock() + self.mock_response = Mock() + self.mock_client.endpoint = 'http://testendpoint:9999' + self.mock_client.auth_token = 'testtoken' + self.mock_client.client_id = 'test_client_id_78900987' + self.action_manager = actions.ActionManager(self.mock_client) + + @patch('freezer.apiclient.actions.requests') + def test_create(self, mock_requests): + self.assertEqual(self.action_manager.endpoint, 'http://testendpoint:9999/v1/actions/') + self.assertEqual(self.action_manager.headers, {'X-Auth-Token': 'testtoken'}) + + @patch('freezer.apiclient.actions.requests') + def test_create_ok(self, mock_requests): + self.mock_response.status_code = 201 + self.mock_response.json.return_value = {'action_id': 'qwerqwer'} + mock_requests.post.return_value = self.mock_response + retval = self.action_manager.create({'action': 'metadata'}) + self.assertEqual(retval, 'qwerqwer') + + @patch('freezer.apiclient.actions.requests') + def test_create_fail_when_api_return_error_code(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.post.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.action_manager.create, {'action': 'metadata'}) + + @patch('freezer.apiclient.actions.requests') + def test_delete_ok(self, mock_requests): + self.mock_response.status_code = 204 + mock_requests.delete.return_value = self.mock_response + retval = self.action_manager.delete('test_action_id') + self.assertIsNone(retval) + + @patch('freezer.apiclient.actions.requests') + def test_delete_fail(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.delete.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.action_manager.delete, 'test_action_id') + + @patch('freezer.apiclient.actions.requests') + def test_get_ok(self, mock_requests): + self.mock_response.status_code = 200 + self.mock_response.json.return_value = {'action_id': 'qwerqwer'} + mock_requests.get.return_value = self.mock_response + retval = self.action_manager.get('test_action_id') + self.assertEqual(retval, {'action_id': 'qwerqwer'}) + + @patch('freezer.apiclient.actions.requests') + def test_get_fails_on_error_different_from_404(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.get.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.action_manager.get, 'test_action_id') + + @patch('freezer.apiclient.actions.requests') + def test_get_none(self, mock_requests): + self.mock_response.status_code = 404 + mock_requests.get.return_value = self.mock_response + retval = self.action_manager.get('test_action_id') + self.assertIsNone(retval) + + @patch('freezer.apiclient.actions.requests') + def test_list_ok(self, mock_requests): + self.mock_response.status_code = 200 + action_list = [{'action_id_0': 'bomboloid'}, {'action_id_1': 'asdfasdf'}] + self.mock_response.json.return_value = {'actions': action_list} + mock_requests.get.return_value = self.mock_response + retval = self.action_manager.list() + self.assertEqual(retval, action_list) + + @patch('freezer.apiclient.actions.requests') + def test_list_error(self, mock_requests): + self.mock_response.status_code = 404 + action_list = [{'action_id_0': 'bomboloid'}, {'action_id_1': 'asdfasdf'}] + self.mock_response.json.return_value = {'clients': action_list} + mock_requests.get.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.action_manager.list) + + @patch('freezer.apiclient.actions.requests') + def test_update_ok(self, mock_requests): + self.mock_response.status_code = 200 + self.mock_response.json.return_value = { + "patch": {"status": "bamboozled"}, + "version": 12, + "action_id": "d454beec-1f3c-4d11-aa1a-404116a40502" + } + mock_requests.patch.return_value = self.mock_response + retval = self.action_manager.update('d454beec-1f3c-4d11-aa1a-404116a40502', {'status': 'bamboozled'}) + self.assertEqual(retval, 12) + + @patch('freezer.apiclient.actions.requests') + def test_update_raise_MetadataUpdateFailure_when_api_return_error_code(self, mock_requests): + self.mock_response.json.return_value = { + "patch": {"status": "bamboozled"}, + "version": 12, + "action_id": "d454beec-1f3c-4d11-aa1a-404116a40502" + } + self.mock_response.status_code = 404 + self.mock_response.text = '{"title": "Not Found","description":"No document found with ID d454beec-1f3c-4d11-aa1a-404116a40502x"}' + mock_requests.patch.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.action_manager.update, + 'd454beec-1f3c-4d11-aa1a-404116a40502', {'status': 'bamboozled'}) diff --git a/tests/test_apiclient_registration.py b/tests/test_apiclient_registration.py index 7f43d581..beb943af 100644 --- a/tests/test_apiclient_registration.py +++ b/tests/test_apiclient_registration.py @@ -87,6 +87,13 @@ class TestRegistrationManager(unittest.TestCase): retval = self.r.get('test_client_id') self.assertIsNone(retval) + @patch('freezer.apiclient.registration.requests') + def test_get_raises_ApiClientException_on_error_not_404(self, mock_requests): + mock_response = Mock() + mock_response.status_code = 500 + mock_requests.get.return_value = mock_response + self.assertRaises(exceptions.ApiClientException, self.r.get, 'test_client_id') + @patch('freezer.apiclient.registration.requests') def test_list_ok(self, mock_requests): mock_response = Mock() diff --git a/tests/test_apiclient_sessions.py b/tests/test_apiclient_sessions.py new file mode 100644 index 00000000..a4446e2f --- /dev/null +++ b/tests/test_apiclient_sessions.py @@ -0,0 +1,227 @@ +""" +Copyright 2015 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import unittest +from mock import Mock, patch + +from freezer.apiclient import exceptions +from freezer.apiclient import sessions + + +class TestSessionManager(unittest.TestCase): + + def setUp(self): + self.mock_client = Mock() + self.mock_response = Mock() + self.mock_client.endpoint = 'http://testendpoint:9999' + self.mock_client.auth_token = 'testtoken' + self.mock_client.client_id = 'test_client_id_78900987' + self.session_manager = sessions.SessionManager(self.mock_client) + self.endpoint = 'http://testendpoint:9999/v1/sessions/' + self.headers = {'X-Auth-Token': 'testtoken'} + + @patch('freezer.apiclient.sessions.requests') + def test_create(self, mock_requests): + self.assertEqual(self.session_manager.endpoint, self.endpoint) + self.assertEqual(self.session_manager.headers, self.headers) + + @patch('freezer.apiclient.sessions.requests') + def test_create_ok(self, mock_requests): + self.mock_response.status_code = 201 + self.mock_response.json.return_value = {'session_id': 'qwerqwer'} + mock_requests.post.return_value = self.mock_response + retval = self.session_manager.create({'session': 'metadata'}) + self.assertEqual(retval, 'qwerqwer') + + @patch('freezer.apiclient.sessions.requests') + def test_create_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.post.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.create, {'session': 'metadata'}) + + @patch('freezer.apiclient.sessions.requests') + def test_delete_ok(self, mock_requests): + self.mock_response.status_code = 204 + mock_requests.delete.return_value = self.mock_response + retval = self.session_manager.delete('test_session_id') + self.assertIsNone(retval) + + @patch('freezer.apiclient.sessions.requests') + def test_delete_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.delete.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.delete, 'test_session_id') + + @patch('freezer.apiclient.sessions.requests') + def test_get_ok(self, mock_requests): + self.mock_response.status_code = 200 + self.mock_response.json.return_value = {'session_id': 'qwerqwer'} + mock_requests.get.return_value = self.mock_response + retval = self.session_manager.get('test_session_id') + self.assertEqual(retval, {'session_id': 'qwerqwer'}) + + @patch('freezer.apiclient.sessions.requests') + def test_get_raise_ApiClientException_when_api_return_error_different_from_404(self, mock_requests): + self.mock_response.status_code = 500 + mock_requests.get.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.get, 'test_session_id') + + @patch('freezer.apiclient.sessions.requests') + def test_get_none(self, mock_requests): + self.mock_response.status_code = 404 + mock_requests.get.return_value = self.mock_response + retval = self.session_manager.get('test_session_id') + self.assertIsNone(retval) + + @patch('freezer.apiclient.sessions.requests') + def test_list_ok(self, mock_requests): + self.mock_response.status_code = 200 + session_list = [{'session_id_0': 'bomboloid'}, {'session_id_1': 'asdfasdf'}] + self.mock_response.json.return_value = {'sessions': session_list} + mock_requests.get.return_value = self.mock_response + retval = self.session_manager.list() + self.assertEqual(retval, session_list) + + @patch('freezer.apiclient.sessions.requests') + def test_list_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + self.mock_response.status_code = 404 + session_list = [{'session_id_0': 'bomboloid'}, {'session_id_1': 'asdfasdf'}] + self.mock_response.json.return_value = {'clients': session_list} + mock_requests.get.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.list) + + @patch('freezer.apiclient.sessions.requests') + def test_update_ok(self, mock_requests): + self.mock_response.status_code = 200 + self.mock_response.json.return_value = { + "patch": {"status": "bamboozled"}, + "version": 12, + "session_id": "d454beec-1f3c-4d11-aa1a-404116a40502" + } + mock_requests.patch.return_value = self.mock_response + retval = self.session_manager.update('d454beec-1f3c-4d11-aa1a-404116a40502', {'status': 'bamboozled'}) + self.assertEqual(retval, 12) + + @patch('freezer.apiclient.sessions.requests') + def test_update_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + self.mock_response.json.return_value = { + "patch": {"status": "bamboozled"}, + "version": 12, + "session_id": "d454beec-1f3c-4d11-aa1a-404116a40502" + } + self.mock_response.status_code = 404 + self.mock_response.text = '{"title": "Not Found","description":"No document found with ID d454beec-1f3c-4d11-aa1a-404116a40502x"}' + mock_requests.patch.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.update, + 'd454beec-1f3c-4d11-aa1a-404116a40502', {'status': 'bamboozled'}) + + @patch('freezer.apiclient.sessions.requests') + def test_add_job_uses_proper_endpoint(self, mock_requests): + session_id, job_id = 'sessionqwerty1234', 'jobqwerty1234' + self.mock_response.status_code = 204 + mock_requests.put.return_value = self.mock_response + endpoint = '{0}{1}/jobs/{2}'.format(self.endpoint, session_id, job_id) + + retval = self.session_manager.add_job(session_id, job_id) + + self.assertEqual(retval, None) + mock_requests.put.assert_called_with(endpoint, headers=self.headers) + + @patch('freezer.apiclient.sessions.requests') + def test_add_job_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + session_id, job_id = 'sessionqwerty1234', 'jobqwerty1234' + self.mock_response.status_code = 500 + mock_requests.put.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.add_job, session_id, job_id) + + @patch('freezer.apiclient.sessions.requests') + def test_remove_job_uses_proper_endpoint(self, mock_requests): + session_id, job_id = 'sessionqwerty1234', 'jobqwerty1234' + self.mock_response.status_code = 204 + mock_requests.delete.return_value = self.mock_response + endpoint = '{0}{1}/jobs/{2}'.format(self.endpoint, session_id, job_id) + + retval = self.session_manager.remove_job(session_id, job_id) + + self.assertEqual(retval, None) + mock_requests.delete.assert_called_with(endpoint, headers=self.headers) + + @patch('freezer.apiclient.sessions.requests') + def test_remove_job_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + session_id, job_id = 'sessionqwerty1234', 'jobqwerty1234' + self.mock_response.status_code = 500 + mock_requests.delete.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.remove_job, session_id, job_id) + + @patch('freezer.apiclient.sessions.requests') + def test_start_session_posts_proper_data(self, mock_requests): + session_id, job_id, tag = 'sessionqwerty1234', 'jobqwerty1234', 23 + self.mock_response.status_code = 202 + self.mock_response.json.return_value = {'result': 'success', 'session_tag': 24} + mock_requests.post.return_value = self.mock_response + # /v1/sessions/{sessions_id}/action + endpoint = '{0}{1}/action'.format(self.endpoint, session_id) + data = {"start": {"current_tag": 23, "job_id": "jobqwerty1234"}} + retval = self.session_manager.start_session(session_id, job_id, tag) + self.assertEqual(retval, {'result': 'success', 'session_tag': 24}) + + args = mock_requests.post.call_args[0] + kwargs = mock_requests.post.call_args[1] + self.assertEquals(endpoint, args[0]) + self.assertEquals(data, json.loads(kwargs['data'])) + self.assertEquals(self.headers, kwargs['headers']) + + @patch('freezer.apiclient.sessions.requests') + def test_start_session_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + session_id, job_id, tag = 'sessionqwerty1234', 'jobqwerty1234', 23 + self.mock_response.status_code = 500 + self.mock_response.json.return_value = {'result': 'success', 'session_tag': 24} + mock_requests.post.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.start_session, + session_id, job_id, tag) + + @patch('freezer.apiclient.sessions.requests') + def test_end_session_posts_proper_data(self, mock_requests): + session_id, job_id, tag = 'sessionqwerty1234', 'jobqwerty1234', 23 + self.mock_response.status_code = 202 + self.mock_response.json.return_value = {'result': 'success', 'session_tag': 24} + mock_requests.post.return_value = self.mock_response + # /v1/sessions/{sessions_id}/action + endpoint = '{0}{1}/action'.format(self.endpoint, session_id) + data = {"end": {"current_tag": 23, "job_id": "jobqwerty1234", "result": "fail"}} + retval = self.session_manager.end_session(session_id, job_id, tag, 'fail') + self.assertEqual(retval, {'result': 'success', 'session_tag': 24}) + + args = mock_requests.post.call_args[0] + kwargs = mock_requests.post.call_args[1] + self.assertEquals(endpoint, args[0]) + self.assertEquals(data, json.loads(kwargs['data'])) + self.assertEquals(self.headers, kwargs['headers']) + + @patch('freezer.apiclient.sessions.requests') + def test_end_session_raise_ApiClientException_when_api_return_error_code(self, mock_requests): + session_id, job_id, tag = 'sessionqwerty1234', 'jobqwerty1234', 23 + self.mock_response.status_code = 500 + self.mock_response.json.return_value = {'result': 'success', 'session_tag': 24} + mock_requests.post.return_value = self.mock_response + self.assertRaises(exceptions.ApiClientException, self.session_manager.end_session, + session_id, job_id, tag, 'fail') diff --git a/tests/test_scheduler_arguments.py b/tests/test_scheduler_arguments.py new file mode 100644 index 00000000..ad9b5bec --- /dev/null +++ b/tests/test_scheduler_arguments.py @@ -0,0 +1,84 @@ +# Copyright 2015 Hewlett-Packard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +import unittest +from mock import Mock, patch + +from freezer.scheduler import arguments + + +class TestOpenstackOptions(unittest.TestCase): + + def setUp(self): + self.args = Mock() + self.args.os_username = 'janedoe' + self.args.os_tenant_name = 'hertenant' + self.args.os_auth_url = 'herauthurl' + self.args.os_password = 'herpassword' + self.args.os_tenant_id = 'hertenantid' + self.args.os_region_name = 'herregion' + self.args.os_endpoint = 'herpublicurl' + self.empty_args = Mock() + self.empty_args.os_username = '' + self.empty_args.os_tenant_name = '' + self.empty_args.os_auth_url = '' + self.empty_args.os_password = '' + self.empty_args.os_tenant_id = '' + self.empty_args.os_region_name = '' + self.empty_args.os_endpoint = '' + + self.env_dict = { + 'OS_USERNAME': 'johndoe', + 'OS_TENANT_NAME': 'histenant', + 'OS_AUTH_URL': 'hisauthurl', + 'OS_PASSWORD': 'hispassword', + 'OS_TENANT_ID': 'histenantid', + 'OS_REGION_NAME': 'hisregion', + 'OS_SERVICE_ENDPOINT': 'hispublicurl' + } + + def test_create_with_args_and_env(self): + os = arguments.OpenstackOptions(self.args, self.env_dict) + self.assertIsInstance(os, arguments.OpenstackOptions) + + def test_create_with_empty_args_and_empty_env(self): + os = arguments.OpenstackOptions(self.empty_args, self.env_dict) + self.assertIsInstance(os, arguments.OpenstackOptions) + + def test_create_with_args_and_empty_env(self): + os = arguments.OpenstackOptions(self.args, {}) + self.assertIsInstance(os, arguments.OpenstackOptions) + + def test_create_raises_Exception_when_missing_username(self): + self.args.os_username = '' + self.assertRaises(Exception, arguments.OpenstackOptions, self.args, {}) + + def test_create_raises_Exception_when_missing_p(self): + self.args.os_password = '' + self.assertRaises(Exception, arguments.OpenstackOptions, self.args, {}) + + def test_create_raises_Exception_when_missing_parameter(self): + self.args.os_username = '' + self.assertRaises(Exception, arguments.OpenstackOptions, self.args, {}) + + def test_str(self): + os = arguments.OpenstackOptions(self.args, self.env_dict) + s = str(os) + self.assertIsInstance(s, str) + + +class TestGetArgs(unittest.TestCase): + + @patch('freezer.scheduler.arguments.argparse.ArgumentParser') + def test_get_args_calls_add_argument(self, mock_ArgumentParser): + mock_arg_parser = Mock() + mock_ArgumentParser.return_value = mock_arg_parser + + retval = arguments.get_args(['alpha', 'bravo']) + call_count = mock_arg_parser.add_argument.call_count + self.assertGreater(call_count, 15) diff --git a/tests/test_scheduler_daemon.py b/tests/test_scheduler_daemon.py new file mode 100644 index 00000000..740c0426 --- /dev/null +++ b/tests/test_scheduler_daemon.py @@ -0,0 +1,56 @@ +# Copyright 2015 Hewlett-Packard +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +import unittest +from mock import Mock, patch + +import signal + +from freezer.scheduler import daemon + + +class TestOpenstackOptions(unittest.TestCase): + + def setUp(self): + self.daemonizable = Mock() + self.daemon = daemon.Daemon(daemonizable=self.daemonizable) + + def test_create(self): + self.assertIsInstance(self.daemon, daemon.Daemon) + + @patch('freezer.scheduler.daemon.logging') + def test_setup_logging_default(self, mock_logging): + res = self.daemon.setup_logging(None) + self.assertEqual(res, '/var/log/freezer-scheduler.log') + + @patch('freezer.scheduler.daemon.create_dir') + @patch('freezer.scheduler.daemon.logging') + def test_setup_logging_userdefined(self, mock_logging, mock_createdir): + res = self.daemon.setup_logging('mylogfile') + self.assertEqual(res, 'mylogfile') + + def test_handle_program_exit_calls_scheduler_stop(self): + self.daemon.handle_program_exit(Mock(), Mock()) + self.daemonizable.stop.assert_called_with() + + def test_handle_program_reload_calls_scheduler_reload(self): + self.daemon.handle_reload(Mock(), Mock()) + self.daemonizable.reload.assert_called_with() + + def test_signal_map_handlers(self): + signal_map = self.daemon.signal_map + self.assertEqual(signal_map[signal.SIGTERM], self.daemon.handle_program_exit) + self.assertEqual(signal_map[signal.SIGHUP], self.daemon.handle_reload) + + @patch('freezer.scheduler.daemon.gettempdir') + @patch('freezer.scheduler.daemon.os.path.expanduser') + def test_pid_fname_in_tempdir(self, mock_expanduser, mock_gettempdir): + mock_expanduser.return_value = '/home/chet' + mock_gettempdir.return_value = '/tempus_fugit' + retval = self.daemon.pid_fname + self.assertEqual(retval, '/tempus_fugit/freezer_sched_chet.pid') diff --git a/tox.ini b/tox.ini index 489ce1a9..76c232a4 100644 --- a/tox.ini +++ b/tox.ini @@ -13,6 +13,8 @@ deps = pymysql python-openstackclient mock + pep3143daemon + apscheduler pylint>=1.3.1 install_command = pip install -U {opts} {packages}