diff --git a/freezer/apiclient/__init__.py b/freezer/apiclient/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer/apiclient/backups.py b/freezer/apiclient/backups.py new file mode 100644 index 00000000..1bfb0919 --- /dev/null +++ b/freezer/apiclient/backups.py @@ -0,0 +1,68 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import requests + +from freezer.freezerclient import exceptions + + +class BackupsManager(object): + + def __init__(self, client): + self.client = client + self.endpoint = self.client.endpoint + 'backups/' + self.headers = {'X-Auth-Token': client.token} + + def create(self, backup_metadata, username=None, tenant_name=None): + r = requests.post(self.endpoint, + data=json.dumps(backup_metadata), + headers=self.headers) + if r.status_code != 201: + raise exceptions.MetadataCreationFailure( + "[*] Error {0}".format(r.status_code)) + backup_id = r.json()['backup_id'] + return backup_id + + def delete(self, backup_id, username=None, tenant_name=None): + endpoint = self.endpoint + backup_id + r = requests.delete(endpoint, headers=self.headers) + if r.status_code != 204: + raise exceptions.MetadataDeleteFailure( + "[*] Error {0}".format(r.status_code)) + + def list(self, username=None, tenant_name=None): + r = requests.get(self.endpoint, headers=self.headers) + if r.status_code != 200: + raise exceptions.MetadataGetFailure( + "[*] Error {0}".format(r.status_code)) + + return r.json()['backups'] + + def get(self, backup_id, username=None, tenant_name=None): + endpoint = self.endpoint + backup_id + r = requests.get(endpoint, headers=self.headers) + if r.status_code == 200: + return r.json() + if r.status_code == 404: + return None + raise exceptions.MetadataGetFailure( + "[*] Error {0}".format(r.status_code)) diff --git a/freezer/apiclient/client.py b/freezer/apiclient/client.py new file mode 100644 index 00000000..5441df6c --- /dev/null +++ b/freezer/apiclient/client.py @@ -0,0 +1,65 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import os +import sys + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'freezer', '__init__.py')): + sys.path.insert(0, possible_topdir) + +import keystoneclient + +from freezer.freezerclient.backups import BackupsManager + + +class Client(object): + + def __init__(self, version='1', + token=None, + username=None, + password=None, + tenant_name=None, + auth_url=None, + endpoint=None, + session=None): + if endpoint is None: + raise Exception('Missing endpoint information') + self.endpoint = endpoint + + if token is not None: + # validate the token ? + self.token = token + elif session is not None: + pass + # TODO: handle session auth + # assert isinstance(session, keystoneclient.session.Session) + else: + self.username = username + self.tenant_name = tenant_name + kc = keystoneclient.v2_0.client.Client( + username=username, + password=password, + tenant_name=tenant_name, + auth_url=auth_url) + self.token = kc.auth_token + self.backups = BackupsManager(self) diff --git a/freezer/apiclient/exceptions.py b/freezer/apiclient/exceptions.py new file mode 100644 index 00000000..4b7b4fe9 --- /dev/null +++ b/freezer/apiclient/exceptions.py @@ -0,0 +1,46 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +class FreezerClientException(Exception): + """ + Base Freezer API Exception + """ + message = ("Unknown exception occurred") + + def __init__(self, message=None, *args, **kwargs): + if not message: + message = self.message + message = message % kwargs + + Exception.__init__(self, message) + + +class MetadataCreationFailure(FreezerClientException): + message = "Metadata creation failed: %reason" + + +class MetadataGetFailure(FreezerClientException): + message = "Metadata read failed: %reason" + + +class MetadataDeleteFailure(FreezerClientException): + message = "Metadata deletion failed: %reason" diff --git a/freezer_api/.coveragerc b/freezer_api/.coveragerc new file mode 100644 index 00000000..6a58b9b9 --- /dev/null +++ b/freezer_api/.coveragerc @@ -0,0 +1,3 @@ +[run] +omit=*__init__.py + diff --git a/freezer_api/README.rst b/freezer_api/README.rst new file mode 100644 index 00000000..f712a178 --- /dev/null +++ b/freezer_api/README.rst @@ -0,0 +1,132 @@ +=========== +Freezer API +=========== + + +Installation +============ + +Install required packages +------------------------- + # pip install keystonemiddleware falcon + +Elasticsearch support:: + # pip install elasticsearch + + +Install freezer_api +------------------- + # git clone https://github.com/stackforge/freezer.git + # cd freezer/freezer_api && sudo python setup.py install + +this will install into /usr/local + + +edit config file +---------------- + # sudo vi /etc/freezer-api.conf + + +run simple instance +------------------- + # freezer-api + + +examples running using uwsgi +---------------------------- + # uwsgi --http :9090 --need-app --master --module freezer_api.cmd.api:application + + # uwsgi --https :9090,foobar.crt,foobar.key --need-app --master --module freezer_api.cmd.api:application + + +Concepts and definitions +======================== + +*hostname* is _probably_ going to be the host fqdn. + +*backup_id* +defined as "container_hostname_backupname_timestamp_level" uniquely +identifies a backup + +*backup_set* +defined as "container_hostname_backupname" identifies a group of related +backups which share the same container,hostname and backupname + +*backup_session* +is a group of backups which share container,hostname and backupname, but +are also related by dependency. + +*backup_session_id* +utilizes the timestamp of the first (level 0) backup in the session +It is identified by (container, hostname, backupname, timestamp-of-level-0) + + +API routes +========== + +General +------- +GET / List API version +GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03 + +Backup metadata +--------------- +GET /v1/backups(?limit,marker) Lists backups +POST /v1/backups Creates backup entry + +GET /v1/backups/{backup_id} Get backup details +UPDATE /v1/backups/{backup_id} Updates the specified backup +DELETE /v1/backups/{backup_id} Deletes the specified backup + + +Data Structures +=============== + +Backup metadata structure +------------------------- +NOTE: sizes are in MB + +backup_metadata:= +{ + "container": string, + "host_name": string, # fqdn, client has to provide consistent information here ! + "backup_name": string, + "timestamp": int, + "level": int, + "backup_session": int, + "max_level": int, + "mode" : string, (fs mongo mysql) + "fs_real_path": string, + "vol_snap_path": string, + "total_broken_links" : int, + "total_fs_files" : int, + "total_directories" : int, + "backup_size_uncompressed" : int, + "backup_size_compressed" : int, + "total_backup_session_size" : int, + "compression_alg": string, (gzip bzip xz) + "encrypted": bool, + "client_os": string + "broken_links" : [string, string, string], + "excluded_files" : [string, string, string] + "cli": string, equivalent cli used when executing the backup ? + "version": string +} + + +The api wraps backup_metadata dictionary with some additional information. +It stores and returns the information provided in this form: + +{ + "backup_id": string # container_hostname_backupname_timestamp_level + "user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided) + "user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided) + + "backup_metadata": { #--- actual backup_metadata provided + "container": string, + "host_name": string, + "backup_name": string, + "timestamp": int, + ... + } +} diff --git a/freezer_api/etc/freezer-api.conf b/freezer_api/etc/freezer-api.conf new file mode 100644 index 00000000..6e479b0a --- /dev/null +++ b/freezer_api/etc/freezer-api.conf @@ -0,0 +1,19 @@ +[DEFAULT] +verbose = false +use_syslogd = false +logging_file = freezer-api.log + + +[keystone_authtoken] +identity_uri = http://keystone:35357/ +auth_uri = http://keystone:5000/ +admin_user = admin +admin_password = secrete +admin_tenant_name = admin +include_service_catalog = False +delay_auth_decision = False + + +[storage] +db=elasticsearch +endpoint=http://localhost:9200 diff --git a/freezer_api/freezer_api/__init__.py b/freezer_api/freezer_api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/api/__init__.py b/freezer_api/freezer_api/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/api/common/__init__.py b/freezer_api/freezer_api/api/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/api/common/middleware.py b/freezer_api/freezer_api/api/common/middleware.py new file mode 100644 index 00000000..f265c3a4 --- /dev/null +++ b/freezer_api/freezer_api/api/common/middleware.py @@ -0,0 +1,48 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import falcon + + +class JSONTranslator(object): + + def process_request(self, req, resp): + if req.content_length in (None, 0): + # Nothing to do + return + + body = req.stream.read() + if not body: + raise falcon.HTTPBadRequest('Empty request body', + 'A valid JSON document is required.') + try: + req.context['doc'] = json.loads(body.decode('utf-8')) + + except (ValueError, UnicodeDecodeError): + raise falcon.HTTPError(falcon.HTTP_753, + 'Malformed JSON') + + def process_response(self, req, resp, resource): + if 'result' not in req.context: + return + + resp.body = json.dumps(req.context['result']) diff --git a/freezer_api/freezer_api/api/v1/__init__.py b/freezer_api/freezer_api/api/v1/__init__.py new file mode 100644 index 00000000..50914925 --- /dev/null +++ b/freezer_api/freezer_api/api/v1/__init__.py @@ -0,0 +1,48 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +from freezer_api.api.v1 import backups +from freezer_api.api.v1 import homedoc + +VERSION = { + 'id': '1', + 'status': 'CURRENT', + 'updated': '2015-03-23T13:45:00', + 'links': [ + { + 'href': '/v1/', + 'rel': 'self' + } + ] +} + + +def public_endpoints(storage_driver): + return [ + ('/', + homedoc.Resource()), + + ('/backups', + backups.BackupsCollectionResource(storage_driver)), + + ('/backups/{backup_id}', + backups.BackupsResource(storage_driver)) + ] diff --git a/freezer_api/freezer_api/api/v1/backups.py b/freezer_api/freezer_api/api/v1/backups.py new file mode 100644 index 00000000..a2c7cc83 --- /dev/null +++ b/freezer_api/freezer_api/api/v1/backups.py @@ -0,0 +1,74 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import falcon +from freezer_api.common import exceptions + + +class BackupsCollectionResource(object): + """ + Handler for endpoint: /v1/backups + """ + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp): + # GET /v1/backups(?limit,marker) Lists backups + user_id = req.get_header('X-User-ID') + obj_list = self.db.get_backup_list(user_id=user_id) + req.context['result'] = {'backups': obj_list} + + def on_post(self, req, resp): + # POST /v1/backups Creates backup entry + try: + doc = req.context['doc'] + except KeyError: + raise exceptions.BadDataFormat( + message='Missing request body', + resp_body={'error': 'missing request body'}) + user_name = req.get_header('X-User-Name') + user_id = req.get_header('X-User-ID') + backup_id = self.db.add_backup( + user_id=user_id, user_name=user_name, data=doc) + resp.status = falcon.HTTP_201 + req.context['result'] = {'backup_id': backup_id} + + +class BackupsResource(object): + """ + Handler for endpoint: /v1/backups/{backup_id} + """ + def __init__(self, storage_driver): + self.db = storage_driver + + def on_get(self, req, resp, backup_id): + # GET /v1/backups/{backup_id} Get backup details + user_id = req.get_header('X-User-ID') + obj = self.db.get_backup(user_id=user_id, backup_id=backup_id) + req.context['result'] = obj + + def on_delete(self, req, resp, backup_id): + # DELETE /v1/backups/{backup_id} Deletes the specified backup + user_id = req.get_header('X-User-ID') + self.db.delete_backup( + user_id=user_id, backup_id=backup_id) + req.context['result'] = {'backup_id': backup_id} + resp.status = falcon.HTTP_204 diff --git a/freezer_api/freezer_api/api/v1/homedoc.py b/freezer_api/freezer_api/api/v1/homedoc.py new file mode 100644 index 00000000..42fb493c --- /dev/null +++ b/freezer_api/freezer_api/api/v1/homedoc.py @@ -0,0 +1,52 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== + +http://tools.ietf.org/html/draft-nottingham-json-home-03 +""" + +import json + +HOME_DOC = { + 'resources': { + 'rel/backups': { + 'href-template': '/v1/backups/{backup_id}', + 'href-vars': { + 'backup_id': 'param/backup_id' + }, + 'hints': { + 'allow': ['GET'], + 'formats': { + 'application/json': {}, + }, + }, + }, + } +} + + +class Resource(object): + + def __init__(self): + document = json.dumps(HOME_DOC, ensure_ascii=False, indent=4) + self.document_utf8 = document.encode('utf-8') + + def on_get(self, req, resp): + resp.data = self.document_utf8 + resp.content_type = 'application/json-home' diff --git a/freezer_api/freezer_api/api/versions.py b/freezer_api/freezer_api/api/versions.py new file mode 100644 index 00000000..67978338 --- /dev/null +++ b/freezer_api/freezer_api/api/versions.py @@ -0,0 +1,43 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import json +import falcon + +from freezer_api.api import v1 + + +VERSIONS = { + 'versions': [ + v1.VERSION + ] +} + + +class Resource(object): + + def __init__(self): + self.versions = json.dumps(VERSIONS, ensure_ascii=False) + + def on_get(self, req, resp): + resp.data = self.versions + + resp.status = falcon.HTTP_300 diff --git a/freezer_api/freezer_api/cmd/__init__.py b/freezer_api/freezer_api/cmd/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/cmd/api.py b/freezer_api/freezer_api/cmd/api.py new file mode 100644 index 00000000..19d6ea1c --- /dev/null +++ b/freezer_api/freezer_api/cmd/api.py @@ -0,0 +1,93 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import logging +import os +import sys +from wsgiref import simple_server +import falcon +from keystonemiddleware import auth_token +from oslo.config import cfg + +possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]), + os.pardir, os.pardir, os.pardir)) +if os.path.exists(os.path.join(possible_topdir, 'freezer_api', '__init__.py')): + sys.path.insert(0, possible_topdir) + +from freezer_api.common import config, log, exceptions +from freezer_api.api import v1 +from freezer_api.api import versions +from freezer_api.api.common import middleware +from freezer_api.storage import driver + + +def get_application(db): + app = falcon.API(middleware=[middleware.JSONTranslator()]) + + for exception_class in exceptions.exception_handlers_catalog: + app.add_error_handler(exception_class, exception_class.handle) + + endpoint_catalog = [ + ('/v1', v1.public_endpoints(db)), + ('/', [('', versions.Resource())]) + ] + for version_path, endpoints in endpoint_catalog: + for route, resource in endpoints: + app.add_route(version_path + route, resource) + + if 'keystone_authtoken' in config.CONF: + app = auth_token.AuthProtocol(app, {}) + else: + logging.warning("keystone authentication disabled") + return app + +config_file = '/etc/freezer-api.conf' +config_files_list = [config_file] if os.path.isfile(config_file) else [] +config.parse_args(args=[], default_config_files=config_files_list) +log.setup() +logging.info("Freezer API starting") +logging.info("Freezer config file(s) used: {0}".format( + ', '.join(cfg.CONF.config_file))) +try: + db = driver.get_db() + application = get_application(db) +except Exception as err: + message = 'Unable to start server: {0}'.format(err) + print message + logging.fatal(message) + sys.exit(1) + + +def main(): + ip, port = '127.0.0.1', 9090 + httpd = simple_server.make_server(ip, port, application) + message = 'Server listening on {0}:{1}'.format(ip, port) + print message + logging.info(message) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print "\nThanks, Bye" + sys.exit(0) + + +if __name__ == '__main__': + main() diff --git a/freezer_api/freezer_api/common/__init__.py b/freezer_api/freezer_api/common/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/common/config.py b/freezer_api/freezer_api/common/config.py new file mode 100644 index 00000000..53595b6c --- /dev/null +++ b/freezer_api/freezer_api/common/config.py @@ -0,0 +1,43 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +from oslo.config import cfg + + +common_cli_opts = [ + cfg.BoolOpt('verbose', + short='v', + default=False, + help='Print more verbose output.'), + cfg.BoolOpt('debug', + short='d', + default=False, + help='Print debugging output.'), +] + +CONF = cfg.CONF +CONF.register_cli_opts(common_cli_opts) + + +def parse_args(args=[], usage=None, default_config_files=[]): + CONF(args=args, + project='freezer', + default_config_files=default_config_files) diff --git a/freezer_api/freezer_api/common/exceptions.py b/freezer_api/freezer_api/common/exceptions.py new file mode 100644 index 00000000..29b0697b --- /dev/null +++ b/freezer_api/freezer_api/common/exceptions.py @@ -0,0 +1,83 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== + +""" + +import falcon +import logging + + +class FreezerAPIException(Exception): + """ + Base Freezer API Exception + """ + json_message = ({'error': 'Unknown exception occurred'}) + + def __init__(self, message=None, resp_body={}): + if message: + self.message = message + self.resp_body = resp_body + logging.error(message) + Exception.__init__(self, message) + + @staticmethod + def handle(ex, req, resp, params): + resp.status = falcon.HTTP_500 + req.context['result'] = {'error': 'internal server error'} + + +class ObjectNotFound(FreezerAPIException): + @staticmethod + def handle(ex, req, resp, params): + resp.status = falcon.HTTP_404 + ex.resp_body.update({'found': False}) + req.context['result'] = ex.resp_body + + +class BadDataFormat(FreezerAPIException): + @staticmethod + def handle(ex, req, resp, params): + resp.status = falcon.HTTP_400 + ex.resp_body.update({'error': 'bad data format'}) + req.context['result'] = ex.resp_body + + +class DocumentExists(FreezerAPIException): + @staticmethod + def handle(ex, req, resp, params): + resp.status = falcon.HTTP_409 + ex.resp_body.update({'error': 'document already exists'}) + req.context['result'] = ex.resp_body + + +class StorageEngineError(FreezerAPIException): + @staticmethod + def handle(ex, req, resp, params): + resp.status = falcon.HTTP_500 + ex.resp_body.update({'error': 'storage engine'}) + req.context['result'] = ex.resp_body + + +exception_handlers_catalog = [ + ObjectNotFound, + BadDataFormat, + DocumentExists, + StorageEngineError +] diff --git a/freezer_api/freezer_api/common/log.py b/freezer_api/freezer_api/common/log.py new file mode 100644 index 00000000..70a70992 --- /dev/null +++ b/freezer_api/freezer_api/common/log.py @@ -0,0 +1,66 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +from oslo.config import cfg +import logging + + +logging_cli_opts = [ + cfg.StrOpt('log-file', + metavar='PATH', + help='(Optional) Name of log file to output to. ' + 'If no default is set, logging will go to stdout.'), + cfg.BoolOpt('use-syslog', + help='Use syslog for logging.'), + cfg.StrOpt('syslog-log-facility', + help='syslog facility to receive log lines') +] + +logging_opts = [ + cfg.StrOpt('logging_file', + metavar='PATH', + default='freezer-api.log', + help='(Optional) Name of log file to output to. ' + 'If no default is set, logging will go to stdout.'), + cfg.BoolOpt('logging_use_syslog', + default=False, + help='Use syslog for logging.'), + cfg.StrOpt('logging_syslog_log_facility', + default='LOG_USER', + help='syslog facility to receive log lines') +] + + +CONF = cfg.CONF +CONF.register_opts(logging_opts) +CONF.register_cli_opts(logging_cli_opts) + + +def setup(): + try: + log_file = CONF['log-file'] # cli provided + except: + log_file = CONF['logging_file'] # .conf file + logging.basicConfig( + filename=log_file, + level=logging.INFO, + format='%(asctime)s %(name)s %(levelname)s %(message)s') diff --git a/freezer_api/freezer_api/common/utils.py b/freezer_api/freezer_api/common/utils.py new file mode 100644 index 00000000..05db6e42 --- /dev/null +++ b/freezer_api/freezer_api/common/utils.py @@ -0,0 +1,68 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +class BackupMetadataDoc: + """ + Wraps a backup_metadata dict and adds some utility methods, + and fields + """ + def __init__(self, user_id='', user_name='', data={}): + self.user_id = user_id + self.user_name = user_name + self.data = data + + def is_valid(self): + try: + assert (self.backup_id is not '') + assert (self.user_id is not '') + except: + return False + return True + + def serialize(self): + return {'backup_id': self.backup_id, + 'user_id': self.user_id, + 'user_name': self.user_name, + 'backup_metadata': self.data} + + @staticmethod + def un_serialize(d): + return BackupMetadataDoc( + user_id=d['user_id'], + user_name=d['user_name'], + data=d['backup_metadata']) + + @property + def backup_set_id(self): + return '{0}_{1}_{2}'.format( + self.data['container'], + self.data['host_name'], + self.data['backup_name'] + ) + + @property + def backup_id(self): + return '{0}_{1}_{2}'.format( + self.backup_set_id, + self.data['timestamp'], + self.data['level'] + ) diff --git a/freezer_api/freezer_api/storage/__init__.py b/freezer_api/freezer_api/storage/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/freezer_api/storage/driver.py b/freezer_api/freezer_api/storage/driver.py new file mode 100644 index 00000000..fc0f21cb --- /dev/null +++ b/freezer_api/freezer_api/storage/driver.py @@ -0,0 +1,57 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +from oslo.config import cfg +import logging + +from freezer_api.storage import simpledict, elastic + + +opt_group = cfg.OptGroup(name='storage', + title='Freezer Storage Engine') + +storage_opts = [ + cfg.StrOpt('db', + default='simpledict', + help='specify the storage db to use: simpledoct (default),' + ' elasticsearch'), + cfg.StrOpt('endpoint', + default='http://localhost:9200', + help='specify the storage endpoint') +] + +CONF = cfg.CONF +CONF.register_group(opt_group) +CONF.register_opts(storage_opts, opt_group) + + +def get_db(): + db_engine = CONF.storage.db + if db_engine == 'simpledict': + logging.info('Storage backend: simple dictionary') + db = simpledict.SimpleDictStorageEngine() + elif db_engine == 'elasticsearch': + endpoint = CONF.storage.endpoint + logging.info('Storage backend: Elasticsearch at {0}'.format(endpoint)) + db = elastic.ElasticSearchEngine(endpoint) + else: + raise Exception('Database Engine {0} not supported'.format(db_engine)) + return db diff --git a/freezer_api/freezer_api/storage/elastic.py b/freezer_api/freezer_api/storage/elastic.py new file mode 100644 index 00000000..d15cc5ba --- /dev/null +++ b/freezer_api/freezer_api/storage/elastic.py @@ -0,0 +1,110 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import elasticsearch +import logging +from freezer_api.common.utils import BackupMetadataDoc +from freezer_api.common import exceptions + + +class ElasticSearchEngine(object): + + def __init__(self, hosts): + # logging.getLogger('elasticsearch').addHandler(logging.NullHandler()) + self.es = elasticsearch.Elasticsearch(hosts) + logging.info('Using Elasticsearch host {0}'.format(hosts)) + self.index = "freezer" + + def _get_backup(self, user_id, backup_id=None): + # raises only on severe engine errors + if backup_id: + query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id) + else: + query = '+user_id:{0}'.format(user_id) + try: + res = self.es.search(index=self.index, doc_type='backups', + q=query) + except Exception as e: + raise exceptions.StorageEngineError( + message='search operation failed', + resp_body={'engine exception': '{0}'.format(e)}) + hit_list = res['hits']['hits'] + return [x['_source'] for x in hit_list] + + def _index(self, doc): + # raises only on severe engine errors + try: + res = self.es.index(index=self.index, doc_type='backups', + body=doc) + except Exception as e: + raise exceptions.StorageEngineError( + message='index operation failed', + resp_body={'engine exception': '{0}'.format(e)}) + return res['created'] + + def _delete_backup(self, user_id, backup_id): + query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id) + try: + self.es.delete_by_query(index=self.index, + doc_type='backups', + q=query) + except Exception as e: + raise exceptions.StorageEngineError( + message='search operation failed', + resp_body={'engine exception': '{0}'.format(e)}) + + def get_backup(self, user_id, backup_id): + # raises is data not found, so reply will be HTTP_404 + backup_metadata = self._get_backup(user_id, backup_id) + if not backup_metadata: + raise exceptions.ObjectNotFound( + message='Requested backup data not found: {0}'. + format(backup_id), + resp_body={'backup_id': backup_id}) + return backup_metadata + + def get_backup_list(self, user_id): + # TODO: elasticsearch reindex for paging + return self._get_backup(user_id) + + def add_backup(self, user_id, user_name, data): + # raises if data is malformed (HTTP_400) or already present (HTTP_409) + backup_metadata_doc = BackupMetadataDoc(user_id, user_name, data) + if not backup_metadata_doc.is_valid(): + raise exceptions.BadDataFormat(message='Bad Data Format') + backup_id = backup_metadata_doc.backup_id + existing_data = self._get_backup(user_id, backup_id) + if existing_data: + raise exceptions.DocumentExists( + message='Backup data already existing ({0})'.format(backup_id), + resp_body={'backup_id': backup_id}) + if not self._index(backup_metadata_doc.serialize()): + # should never happen + raise exceptions.StorageEngineError( + message='index operation failed', + resp_body={'backup_id': backup_id}) + logging.info('Backup metadata indexed, backup_id: {0}'. + format(backup_id)) + return backup_id + + def delete_backup(self, user_id, backup_id): + self._delete_backup(user_id, backup_id) + return backup_id diff --git a/freezer_api/freezer_api/storage/simpledict.py b/freezer_api/freezer_api/storage/simpledict.py new file mode 100644 index 00000000..0fbd769f --- /dev/null +++ b/freezer_api/freezer_api/storage/simpledict.py @@ -0,0 +1,69 @@ +""" +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import logging +from freezer_api.common.utils import BackupMetadataDoc +from freezer_api.common import exceptions + + +class SimpleDictStorageEngine(object): + + def __init__(self): + self._map = {} + + def get_backup(self, user_id, backup_id): + try: + backup_data = self._map[(user_id, backup_id)] + except: + raise exceptions.ObjectNotFound( + message='Requested backup data not found: {0}'. + format(backup_id), + resp_body={'backup_id': backup_id}) + return backup_data + + def get_backup_list(self, user_id): + backup_list = [] + for (key, backup_data) in self._map.iteritems(): + if key[0] == user_id: + backup_list.append(backup_data) + return backup_list + + def add_backup(self, user_id, user_name, data): + backup_metadata_doc = BackupMetadataDoc(user_id, user_name, data) + if not backup_metadata_doc.is_valid(): + raise exceptions.BadDataFormat(message='Bad Data Format') + backup_id = backup_metadata_doc.backup_id + if (user_id, backup_id) in self._map: + raise exceptions.DocumentExists( + message='Backup data already existing ({0})'.format(backup_id), + resp_body={'backup_id': backup_id}) + self._map[(user_id, backup_id)] = backup_metadata_doc.serialize() + logging.info('Adding backup data with backup_id {0}'.format(backup_id)) + return backup_id + + def delete_backup(self, user_id, backup_id): + try: + self._map.pop((user_id, backup_id)) + except KeyError: + raise exceptions.ObjectNotFound( + message='Object to remove not found: {0}'.format(backup_id), + resp_body={'backup_id': backup_id}) + return backup_id diff --git a/freezer_api/setup.cfg b/freezer_api/setup.cfg new file mode 100644 index 00000000..d8fef976 --- /dev/null +++ b/freezer_api/setup.cfg @@ -0,0 +1,50 @@ +[metadata] +name = freezer_api + +version = 2015.1 + +summary = OpenStack Backup and Restore Service +description-file = + README.rst + +author = Fausto Marzi, Fabrizio Fresco, Fabrizio Vanni', +author_email = fausto.marzi@hp.com, fabrizio.vanni@hp.com, fabrizio.fresco@hp.com + +home-page = https://github.com/stackforge/freezer +classifier = + Environment :: OpenStack + Programming Language :: Python + Development Status :: 5 - Production/Stable + Natural Language :: English + Intended Audience :: Developers + Intended Audience :: Financial and Insurance Industry + Intended Audience :: Information Technology + Intended Audience :: System Administrators + Intended Audience :: Telecommunications Industry + License :: OSI Approved :: Apache Software License + Operating System :: MacOS + Operating System :: POSIX :: BSD :: FreeBSD + Operating System :: POSIX :: BSD :: NetBSD + Operating System :: POSIX :: BSD :: OpenBSD + Operating System :: POSIX :: Linux + Operating System :: Unix + Topic :: System :: Archiving :: Backup + Topic :: System :: Archiving :: Compression + Topic :: System :: Archiving + +[files] +packages = + freezer_api +data_files = + /etc = etc/* + +[entry_points] +console_scripts = + freezer-api = freezer_api.cmd.api:main + +[pytests] +where=tests +verbosity=2 + +[pbr] +warnerrors = True diff --git a/freezer_api/setup.py b/freezer_api/setup.py new file mode 100644 index 00000000..aa2d8a01 --- /dev/null +++ b/freezer_api/setup.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python + +from setuptools import setup + +setup( + setup_requires=['pbr'], + pbr=True, +) diff --git a/freezer_api/specs/Freezer API spec.odt b/freezer_api/specs/Freezer API spec.odt new file mode 100644 index 00000000..a249cec9 Binary files /dev/null and b/freezer_api/specs/Freezer API spec.odt differ diff --git a/freezer_api/specs/Freezer API spec.pdf b/freezer_api/specs/Freezer API spec.pdf new file mode 100644 index 00000000..470dc2d4 Binary files /dev/null and b/freezer_api/specs/Freezer API spec.pdf differ diff --git a/freezer_api/tests/__init__.py b/freezer_api/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/tests/common.py b/freezer_api/tests/common.py new file mode 100644 index 00000000..4537c382 --- /dev/null +++ b/freezer_api/tests/common.py @@ -0,0 +1,301 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import io + + +fake_data_0_backup_id = 'freezer_container_alpha_important_data_backup_8475903425_0' +fake_data_0_user_id = 'qwerty1234' +fake_data_0_user_name = 'asdffdsa' + +fake_data_0_wrapped_backup_metadata = { + 'backup_id': 'freezer_container_alpha_important_data_backup_8475903425_0', + 'user_id': 'qwerty1234', + 'user_name': 'asdffdsa', + 'backup_metadata': { + "container": "freezer_container", + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 8475903425, + "level": 0, + "backup_session": 8475903425, + "max_level": 5, + "mode" : "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files" : 11, + "total_directories" : 2, + "backup_size_uncompressed" : 4567, + "backup_size_compressed" : 1212, + "total_backup_session_size" : 6789, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + "cli": "" + } +} + +fake_data_0_backup_metadata = { + "container": "freezer_container", + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 8475903425, + "level": 0, + "backup_session": 8475903425, + "max_level": 5, + "mode": "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files" : 11, + "total_directories" : 2, + "backup_size_uncompressed" : 4567, + "backup_size_compressed" : 1212, + "total_backup_session_size" : 6789, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + "cli": "" +} + +fake_malformed_data_0_backup_metadata = { + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 8475903425, + "level": 0, + "backup_session": 8475903425, + "max_level": 5, + "mode": "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files" : 11, + "total_directories" : 2, + "backup_size_uncompressed" : 4567, + "backup_size_compressed" : 1212, + "total_backup_session_size" : 6789, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + "cli": "" +} + + +fake_data_0_elasticsearch_hit = { + "_shards": { + "failed": 0, + "successful": 5, + "total": 5 + }, + "hits": { + "hits": [ + { + "_id": "AUx_iu-ewlhuOVELWtH0", + "_index": "freezer", + "_score": 1.0, + "_type": "backups", + "_source": { + "container": "freezer_container", + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 8475903425, + "level": 0, + "backup_session": 8475903425, + "max_level": 5, + "mode" : "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files" : 11, + "total_directories" : 2, + "backup_size_uncompressed" : 4567, + "backup_size_compressed" : 1212, + "total_backup_session_size" : 6789, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + "cli": "" + } + } + ], + "max_score": 1.0, + "total": 1 + }, + "timed_out": False, + "took": 3 +} + + +fake_data_0_elasticsearch_miss = { + "_shards": { + "failed": 0, + "successful": 5, + "total": 5 + }, + "hits": { + "hits": [], + "max_score": None, + "total": 0 + }, + "timed_out": False, + "took": 1 +} + + +class FakeReqResp: + + def __init__(self, method='GET', body=''): + self.method = method + self.body = body + self.stream = io.BytesIO(body) + self.content_length = len(body) + self.context = {} + self.header = {} + + def get_header(self, key): + return self.header.get(key, None) + + +class FakeElasticsearch_hit: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + return fake_data_0_elasticsearch_hit + + def index(self, index, doc_type, body): + return {'created': True} + + def delete_by_query(self, index, doc_type, q): + pass + + +class FakeElasticsearch_insert_ok: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + return fake_data_0_elasticsearch_miss + + def index(self, index, doc_type, body): + return {'created': True} + + def delete_by_query(self, index, doc_type, q): + pass + + +class FakeElasticsearch_miss: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + return fake_data_0_elasticsearch_miss + + def index(self, index, doc_type, body): + return {'created': False} + + def delete_by_query(self, index, doc_type, q): + pass + + +class FakeElasticsearch_index_raise: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + return fake_data_0_elasticsearch_miss + + def index(self, index, doc_type, body): + raise Exception + + def delete_by_query(self, index, doc_type, q): + pass + + +class FakeElasticsearch_search_raise: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + raise Exception + + def index(self, index, doc_type, body): + return {'created': True} + + def delete_by_query(self, index, doc_type, q): + pass + +class FakeElasticsearch_delete_raise: + def __init__(self, host=None): + pass + + def search(self, index, doc_type, q): + return fake_data_0_elasticsearch_miss + + def index(self, index, doc_type, body): + return {'created': True} + + def delete_by_query(self, index, doc_type, q): + raise Exception + + +class FakeLogging: + + def __init__(self): + return None + + def __call__(self, *args, **kwargs): + return True + + @classmethod + def logging(cls, opt1=True): + return True + + @classmethod + def info(cls, opt1=True): + return True + + @classmethod + def warning(cls, opt1=True): + return True + + @classmethod + def critical(cls, opt1=True): + return True + + @classmethod + def exception(cls, opt1=True): + return True + + @classmethod + def error(cls, opt1=True): + return True diff --git a/freezer_api/tests/test_api.py b/freezer_api/tests/test_api.py new file mode 100644 index 00000000..98552b87 --- /dev/null +++ b/freezer_api/tests/test_api.py @@ -0,0 +1,46 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import pytest + +from common import * +from freezer_api.common.exceptions import * +from keystonemiddleware import auth_token + + +from freezer_api.cmd import api + + +class TestAPI: + + def patch_logging(self, monkeypatch): + fakelogging = FakeLogging() + monkeypatch.setattr(logging, 'critical', fakelogging.critical) + monkeypatch.setattr(logging, 'warning', fakelogging.warning) + monkeypatch.setattr(logging, 'exception', fakelogging.exception) + monkeypatch.setattr(logging, 'error', fakelogging.error) + + def test_auth_install(self, monkeypatch): + self.patch_logging(monkeypatch) + app = api.get_application(None) + assert isinstance(app, auth_token.AuthProtocol) diff --git a/freezer_api/tests/test_backups.py b/freezer_api/tests/test_backups.py new file mode 100644 index 00000000..36d99e23 --- /dev/null +++ b/freezer_api/tests/test_backups.py @@ -0,0 +1,118 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import unittest + +import falcon +from freezer_api.api.v1 import backups +from freezer_api.storage import simpledict + +from common import * +from freezer_api.common.exceptions import * + + +class TestBackupsCollectionResource(unittest.TestCase): + + def setUp(self): + self.db = simpledict.SimpleDictStorageEngine() + self.resource = backups.BackupsCollectionResource(self.db) + self.req = FakeReqResp() + self.req.header['X-User-ID'] = fake_data_0_user_id + + def test_on_get_return_empty_list(self): + expected_result = {'backups': []} + self.resource.on_get(self.req, self.req) + result = self.req.context['result'] + self.assertEqual(result, expected_result) + + def test_on_get_return_correct_list(self): + self.db.add_backup(user_id=fake_data_0_user_id, + user_name=fake_data_0_user_name, + data=fake_data_0_backup_metadata) + self.resource.on_get(self.req, self.req) + result = self.req.context['result'] + expected_result = {'backups': [fake_data_0_wrapped_backup_metadata]} + self.assertEqual(result, expected_result) + + def test_on_get_return_empty_list_without_user_id(self): + self.req.header.pop('X-User-ID') + self.db.add_backup(user_id=fake_data_0_user_id, + user_name=fake_data_0_user_name, + data=fake_data_0_backup_metadata) + self.resource.on_get(self.req, self.req) + result = self.req.context['result'] + expected_result = {'backups': []} + self.assertEqual(result, expected_result) + + def test_on_get_return_empty_list_with_different_user_id(self): + self.req.header['X-User-ID'] = 'LupinIII' + self.db.add_backup(user_id=fake_data_0_user_id, + user_name=fake_data_0_user_name, + data=fake_data_0_backup_metadata) + self.resource.on_get(self.req, self.req) + result = self.req.context['result'] + expected_result = {'backups': []} + self.assertEqual(result, expected_result) + + def test_on_post_raises_when_missing_body(self): + self.assertRaises(BadDataFormat, self.resource.on_post, self.req, self.req) + + def test_on_post_inserts_correct_data(self): + self.req.context['doc'] = fake_data_0_backup_metadata + self.resource.on_post(self.req, self.req) + self.assertEquals(self.req.status, falcon.HTTP_201) + expected_result = {'backup_id': fake_data_0_backup_id} + self.assertEquals(self.req.context['result'], expected_result) + + +class TestBackupsResource(unittest.TestCase): + + def setUp(self): + self.db = simpledict.SimpleDictStorageEngine() + self.resource = backups.BackupsResource(self.db) + self.req = FakeReqResp() + self.req.header['X-User-ID'] = fake_data_0_user_id + + def test_on_get_raises_when_not_found(self): + self.assertRaises(ObjectNotFound, self.resource.on_get, self.req, self.req, fake_data_0_backup_id) + + def test_on_get_return_correct_data(self): + self.db.add_backup(user_id=fake_data_0_user_id, + user_name=fake_data_0_user_name, + data=fake_data_0_backup_metadata) + self.resource.on_get(self.req, self.req, fake_data_0_backup_id) + result = self.req.context['result'] + self.assertEqual(result, fake_data_0_wrapped_backup_metadata) + + def test_on_delete_raises_when_not_found(self): + self.assertRaises(ObjectNotFound, self.resource.on_delete, self.req, self.req, fake_data_0_backup_id) + + def test_on_delete_removes_proper_data(self): + self.db.add_backup(user_id=fake_data_0_user_id, + user_name=fake_data_0_user_name, + data=fake_data_0_backup_metadata) + self.resource.on_delete(self.req, self.req, fake_data_0_backup_id) + result = self.req.context['result'] + expected_result = {'backup_id': fake_data_0_backup_id} + self.assertEquals(self.req.status, falcon.HTTP_204) + self.assertEqual(result, expected_result) diff --git a/freezer_api/tests/test_config.py b/freezer_api/tests/test_config.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/tests/test_driver.py b/freezer_api/tests/test_driver.py new file mode 100644 index 00000000..d8634d1f --- /dev/null +++ b/freezer_api/tests/test_driver.py @@ -0,0 +1,62 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import unittest + +import pytest + +import falcon + +from common import * +from freezer_api.common.exceptions import * +from oslo.config import cfg + + +from freezer_api.storage import driver, elastic, simpledict + + +class TestStorageDriver: + + def patch_logging(self, monkeypatch): + fakelogging = FakeLogging() + monkeypatch.setattr(logging, 'critical', fakelogging.critical) + monkeypatch.setattr(logging, 'warning', fakelogging.warning) + monkeypatch.setattr(logging, 'exception', fakelogging.exception) + monkeypatch.setattr(logging, 'error', fakelogging.error) + + def test_get_db_raises_when_db_not_supported(self, monkeypatch): + self.patch_logging(monkeypatch) + cfg.CONF.storage.db = 'nodb' + pytest.raises(Exception, driver.get_db) + + def test_get_db_simpledict(self, monkeypatch): + self.patch_logging(monkeypatch) + cfg.CONF.storage.db = 'simpledict' + db = driver.get_db() + assert isinstance(db, simpledict.SimpleDictStorageEngine) + + def test_get_db_elastic(self, monkeypatch): + self.patch_logging(monkeypatch) + cfg.CONF.storage.db = 'elasticsearch' + db = driver.get_db() + assert isinstance(db, elastic.ElasticSearchEngine) diff --git a/freezer_api/tests/test_elastic.py b/freezer_api/tests/test_elastic.py new file mode 100644 index 00000000..5a67fe4c --- /dev/null +++ b/freezer_api/tests/test_elastic.py @@ -0,0 +1,124 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import unittest + +import pytest + +from freezer_api.storage import elastic + +from common import * +from freezer_api.common.exceptions import * + +import elasticsearch + + +class TestElasticSearchEngine: + + def patch_logging(self, monkeypatch): + fakelogging = FakeLogging() + monkeypatch.setattr(logging, 'critical', fakelogging.critical) + monkeypatch.setattr(logging, 'warning', fakelogging.warning) + monkeypatch.setattr(logging, 'exception', fakelogging.exception) + monkeypatch.setattr(logging, 'error', fakelogging.error) + + +class TestElasticSearchEngine_get_backup(TestElasticSearchEngine): + + def test_get_backup_userid_and_backup_id_return_ok(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit) + engine = elastic.ElasticSearchEngine('host') + res = engine.get_backup(fake_data_0_user_id, fake_data_0_backup_id) + assert (res == [fake_data_0_backup_metadata, ]) + + def test_get_backup_raises_when_query_has_no_hits(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_miss) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(ObjectNotFound, engine.get_backup, fake_data_0_user_id, fake_data_0_backup_id) + + +class TestElasticSearchEngine_get_backup_list(TestElasticSearchEngine): + + def test_get_backup_list_return_ok(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit) + engine = elastic.ElasticSearchEngine('host') + res = engine.get_backup_list(fake_data_0_user_id) + assert (res == [fake_data_0_backup_metadata, ]) + + +class TestElasticSearchEngine_add_backup(TestElasticSearchEngine): + + def test_index_backup_success(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_insert_ok) + engine = elastic.ElasticSearchEngine('host') + res = engine.add_backup(fake_data_0_user_id, fake_data_0_user_name, fake_data_0_backup_metadata) + assert (res == fake_data_0_backup_id) + + def test_index_backup_raise_when_data_exists(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(DocumentExists, engine.add_backup, fake_data_0_user_id, + fake_data_0_user_name, fake_data_0_backup_metadata) + + def test_index_backup_raise_when_es_index_raises(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_index_raise) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(StorageEngineError, engine.add_backup, fake_data_0_user_id, + fake_data_0_user_name, fake_data_0_backup_metadata) + + def test_index_backup_raise_when_es_search_raises(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_search_raise) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(StorageEngineError, engine.add_backup, fake_data_0_user_id, + fake_data_0_user_name, fake_data_0_backup_metadata) + + def test_index_backup_raise_when_data_is_malformed(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_insert_ok) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(BadDataFormat, engine.add_backup, fake_data_0_user_id, + fake_data_0_user_name, fake_malformed_data_0_backup_metadata) + + +class TestElasticSearchEngine_delete_backup(TestElasticSearchEngine): + + def test_delete_backup_raise_when_es_delete_raises(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_delete_raise) + engine = elastic.ElasticSearchEngine('host') + pytest.raises(StorageEngineError, engine.delete_backup, fake_data_0_user_id, fake_data_0_backup_id) + + def test_delete_backup_ok(self, monkeypatch): + self.patch_logging(monkeypatch) + monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit) + engine = elastic.ElasticSearchEngine('host') + res = engine.delete_backup(fake_data_0_user_id, fake_data_0_backup_id) + assert (res == fake_data_0_backup_id) + diff --git a/freezer_api/tests/test_homedoc.py b/freezer_api/tests/test_homedoc.py new file mode 100644 index 00000000..135b97e2 --- /dev/null +++ b/freezer_api/tests/test_homedoc.py @@ -0,0 +1,39 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import unittest +from common import FakeReqResp +from freezer_api.api import v1 +import json + + +class TestHomedocResource(unittest.TestCase): + + def setUp(self): + self.resource = v1.homedoc.Resource() + self.req = FakeReqResp() + + def test_on_get_return_resources_information(self): + self.resource.on_get(self.req, self.req) + result = json.loads(self.req.data) + expected_result = v1.homedoc.HOME_DOC + self.assertEquals(result, expected_result) diff --git a/freezer_api/tests/test_middleware.py b/freezer_api/tests/test_middleware.py new file mode 100644 index 00000000..8650e51f --- /dev/null +++ b/freezer_api/tests/test_middleware.py @@ -0,0 +1,62 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import json +import unittest + +import falcon + +from freezer_api.api.common import middleware + +from common import FakeReqResp + + +class TestBackupMetadataDoc(unittest.TestCase): + + def setUp(self): + self.json_translator = middleware.JSONTranslator() + + def test_process_request_with_no_body_returns_none(self): + req = FakeReqResp() + self.assertIsNone(self.json_translator.process_request(req, req)) + + def test_process_request_with_positive_length_and_no_body_raises(self): + req = FakeReqResp() + req.content_length = 1 + self.assertRaises(falcon.HTTPBadRequest, self.json_translator.process_request, req, req) + + def test_process_response_with_no_result_returns_none(self): + req = FakeReqResp() + self.assertIsNone(self.json_translator.process_response(req, req, None)) + + def test_process_response_create_correct_json_body(self): + req = FakeReqResp() + d = {'key1': 'value1', 'key2': 'value2'} + req.context['result'] = d + correct_json_body = json.dumps(d) + self.json_translator.process_response(req, req, None) + self.assertEqual(correct_json_body, req.body) + + def test_process_request_with_malformed_body_raises(self): + req = FakeReqResp(body='{"key2": "value2",{ "key1": "value1"}') + self.assertRaises(falcon.HTTPError, self.json_translator.process_request, req, req) diff --git a/freezer_api/tests/test_simpledict.py b/freezer_api/tests/test_simpledict.py new file mode 100644 index 00000000..e69de29b diff --git a/freezer_api/tests/test_utils.py b/freezer_api/tests/test_utils.py new file mode 100644 index 00000000..bbd4dd5a --- /dev/null +++ b/freezer_api/tests/test_utils.py @@ -0,0 +1,111 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + + +import unittest +from freezer_api.common import utils + + +DATA_backup_metadata = { + "container": "freezer_container", + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 12341234, + "level": 0, + "backup_session": 12341234, + "max_level": 5, + "mode" : "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files": 11, + "total_directories": 2, + "backup_size_uncompressed": 12112342, + "backup_size_compressed": 1214322, + "total_backup_session_size": 1212, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + 'cli': 'whatever' +} + +DATA_user_id = 'AUx6F07NwlhuOVELWtHx' + +DATA_user_name = 'gegrex55NPlwlhuOVELWtHv' + +DATA_backup_id = 'freezer_container_alpha_important_data_backup_12341234_0' + +DATA_wrapped_backup_metadata = { + 'user_id': DATA_user_id, + 'user_name': DATA_user_name, + 'backup_id': DATA_backup_id, + 'backup_medatada': { + "container": "freezer_container", + "host_name": "alpha", + "backup_name": "important_data_backup", + "timestamp": 12341234, + "level": 0, + "backup_session": 12341234, + "max_level": 5, + "mode": "fs", + "fs_real_path": "/blabla", + "vol_snap_path": "/blablasnap", + "total_broken_links" : 0, + "total_fs_files": 11, + "total_directories": 2, + "backup_size_uncompressed": 12112342, + "backup_size_compressed": 1214322, + "total_backup_session_size": 1212, + "compression_alg": "None", + "encrypted": "false", + "client_os": "linux", + "broken_links": ["link_01", "link_02"], + "excluded_files": ["excluded_file_01", "excluded_file_02"], + 'cli': 'whatever' + } +} + + +class TestBackupMetadataDoc(unittest.TestCase): + + def setUp(self): + self.backup_metadata = utils.BackupMetadataDoc( + user_id=DATA_user_id, + user_name=DATA_user_name, + data=DATA_backup_metadata) + + def test_backup_id(self): + assert (self.backup_metadata.backup_id == DATA_backup_id) + + def test_is_valid_return_True_when_valid(self): + self.assertTrue(self.backup_metadata.is_valid()) + + def test_is_valid_returns_False_when_user_id_empty(self): + self.backup_metadata.user_id = '' + self.assertFalse(self.backup_metadata.is_valid()) + + def test_backup_id_correct(self): + self.assertEqual(self.backup_metadata.backup_id, DATA_backup_id) + self.backup_metadata.data['container'] = 'different' + self.assertNotEqual(self.backup_metadata.backup_id, DATA_backup_id) diff --git a/freezer_api/tests/test_versions.py b/freezer_api/tests/test_versions.py new file mode 100644 index 00000000..d55b0281 --- /dev/null +++ b/freezer_api/tests/test_versions.py @@ -0,0 +1,41 @@ +"""Freezer swift.py related tests + +Copyright 2014 Hewlett-Packard + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +This product includes cryptographic software written by Eric Young +(eay@cryptsoft.com). This product includes software written by Tim +Hudson (tjh@cryptsoft.com). +======================================================================== +""" + +import unittest +import falcon +from common import FakeReqResp +from freezer_api.api import versions +from freezer_api.api import v1 +import json + + +class TestVersionResource(unittest.TestCase): + + def setUp(self): + self.resource = versions.Resource() + self.req = FakeReqResp() + + def test_on_get_return_versions(self): + self.resource.on_get(self.req, self.req) + self.assertEquals(self.req.status, falcon.HTTP_300) + expected_result = json.dumps({'versions': [v1.VERSION]}) + self.assertEquals(self.req.data, expected_result) diff --git a/freezer_api/tox.ini b/freezer_api/tox.ini new file mode 100644 index 00000000..8c57e03c --- /dev/null +++ b/freezer_api/tox.ini @@ -0,0 +1,33 @@ +[tox] +envlist = py27,pep8 +skipsdist = True + +[testenv] +usedevelop = True +deps = + pytest + coverage + flake8 + pytest-cov + pytest-xdist + pymysql + falcon + keystonemiddleware + elasticsearch + +install_command = pip install -U {opts} {packages} +setenv = VIRTUAL_ENV={envdir} + +commands = + py.test -v --cov-report term-missing --cov freezer_api + +[pytest] +python_files = test_*.py +norecursedirs = .tox .venv specs + +[testenv:pep8] +commands = flake8 freezer_api + +[flake8] +show-source = True +exclude = .venv,.tox,dist,doc,test,*egg,tests,specs,build diff --git a/setup.py b/setup.py index 89b253fc..2394c544 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ from setuptools import setup, find_packages from setuptools.command.test import test as TestCommand import os +import io here = os.path.abspath(os.path.dirname(__file__)) diff --git a/tox.ini b/tox.ini index 2f47d1e8..10cf193d 100644 --- a/tox.ini +++ b/tox.ini @@ -18,7 +18,7 @@ commands = python runtests.py -v -n 2 --cov-report term-missing --cov freezer [pytest] python_files = test_*.py -norecursedirs = .tox .venv +norecursedirs = .tox .venv freezer_api [testenv:pep8] commands = flake8 freezer