Freezer API

First implementation of the freezer API.
Slightly more than a skeleton with basic functionality

Change-Id: Iae04affea3aa0f4a943599b528df49d9d4a5b845
Implements: blueprint freezer-api-first-rel
This commit is contained in:
Fabrizio Vanni 2015-03-24 17:15:20 +00:00
parent ef58b8b158
commit e4238272c5
45 changed files with 2184 additions and 1 deletions

View File

View File

@ -0,0 +1,68 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import json
import requests
from freezer.freezerclient import exceptions
class BackupsManager(object):
def __init__(self, client):
self.client = client
self.endpoint = self.client.endpoint + 'backups/'
self.headers = {'X-Auth-Token': client.token}
def create(self, backup_metadata, username=None, tenant_name=None):
r = requests.post(self.endpoint,
data=json.dumps(backup_metadata),
headers=self.headers)
if r.status_code != 201:
raise exceptions.MetadataCreationFailure(
"[*] Error {0}".format(r.status_code))
backup_id = r.json()['backup_id']
return backup_id
def delete(self, backup_id, username=None, tenant_name=None):
endpoint = self.endpoint + backup_id
r = requests.delete(endpoint, headers=self.headers)
if r.status_code != 204:
raise exceptions.MetadataDeleteFailure(
"[*] Error {0}".format(r.status_code))
def list(self, username=None, tenant_name=None):
r = requests.get(self.endpoint, headers=self.headers)
if r.status_code != 200:
raise exceptions.MetadataGetFailure(
"[*] Error {0}".format(r.status_code))
return r.json()['backups']
def get(self, backup_id, username=None, tenant_name=None):
endpoint = self.endpoint + backup_id
r = requests.get(endpoint, headers=self.headers)
if r.status_code == 200:
return r.json()
if r.status_code == 404:
return None
raise exceptions.MetadataGetFailure(
"[*] Error {0}".format(r.status_code))

View File

@ -0,0 +1,65 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import os
import sys
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir, os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'freezer', '__init__.py')):
sys.path.insert(0, possible_topdir)
import keystoneclient
from freezer.freezerclient.backups import BackupsManager
class Client(object):
def __init__(self, version='1',
token=None,
username=None,
password=None,
tenant_name=None,
auth_url=None,
endpoint=None,
session=None):
if endpoint is None:
raise Exception('Missing endpoint information')
self.endpoint = endpoint
if token is not None:
# validate the token ?
self.token = token
elif session is not None:
pass
# TODO: handle session auth
# assert isinstance(session, keystoneclient.session.Session)
else:
self.username = username
self.tenant_name = tenant_name
kc = keystoneclient.v2_0.client.Client(
username=username,
password=password,
tenant_name=tenant_name,
auth_url=auth_url)
self.token = kc.auth_token
self.backups = BackupsManager(self)

View File

@ -0,0 +1,46 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
class FreezerClientException(Exception):
"""
Base Freezer API Exception
"""
message = ("Unknown exception occurred")
def __init__(self, message=None, *args, **kwargs):
if not message:
message = self.message
message = message % kwargs
Exception.__init__(self, message)
class MetadataCreationFailure(FreezerClientException):
message = "Metadata creation failed: %reason"
class MetadataGetFailure(FreezerClientException):
message = "Metadata read failed: %reason"
class MetadataDeleteFailure(FreezerClientException):
message = "Metadata deletion failed: %reason"

3
freezer_api/.coveragerc Normal file
View File

@ -0,0 +1,3 @@
[run]
omit=*__init__.py

132
freezer_api/README.rst Normal file
View File

@ -0,0 +1,132 @@
===========
Freezer API
===========
Installation
============
Install required packages
-------------------------
# pip install keystonemiddleware falcon
Elasticsearch support::
# pip install elasticsearch
Install freezer_api
-------------------
# git clone https://github.com/stackforge/freezer.git
# cd freezer/freezer_api && sudo python setup.py install
this will install into /usr/local
edit config file
----------------
# sudo vi /etc/freezer-api.conf
run simple instance
-------------------
# freezer-api
examples running using uwsgi
----------------------------
# uwsgi --http :9090 --need-app --master --module freezer_api.cmd.api:application
# uwsgi --https :9090,foobar.crt,foobar.key --need-app --master --module freezer_api.cmd.api:application
Concepts and definitions
========================
*hostname* is _probably_ going to be the host fqdn.
*backup_id*
defined as "container_hostname_backupname_timestamp_level" uniquely
identifies a backup
*backup_set*
defined as "container_hostname_backupname" identifies a group of related
backups which share the same container,hostname and backupname
*backup_session*
is a group of backups which share container,hostname and backupname, but
are also related by dependency.
*backup_session_id*
utilizes the timestamp of the first (level 0) backup in the session
It is identified by (container, hostname, backupname, timestamp-of-level-0)
API routes
==========
General
-------
GET / List API version
GET /v1 JSON Home document, see http://tools.ietf.org/html/draft-nottingham-json-home-03
Backup metadata
---------------
GET /v1/backups(?limit,marker) Lists backups
POST /v1/backups Creates backup entry
GET /v1/backups/{backup_id} Get backup details
UPDATE /v1/backups/{backup_id} Updates the specified backup
DELETE /v1/backups/{backup_id} Deletes the specified backup
Data Structures
===============
Backup metadata structure
-------------------------
NOTE: sizes are in MB
backup_metadata:=
{
"container": string,
"host_name": string, # fqdn, client has to provide consistent information here !
"backup_name": string,
"timestamp": int,
"level": int,
"backup_session": int,
"max_level": int,
"mode" : string, (fs mongo mysql)
"fs_real_path": string,
"vol_snap_path": string,
"total_broken_links" : int,
"total_fs_files" : int,
"total_directories" : int,
"backup_size_uncompressed" : int,
"backup_size_compressed" : int,
"total_backup_session_size" : int,
"compression_alg": string, (gzip bzip xz)
"encrypted": bool,
"client_os": string
"broken_links" : [string, string, string],
"excluded_files" : [string, string, string]
"cli": string, equivalent cli used when executing the backup ?
"version": string
}
The api wraps backup_metadata dictionary with some additional information.
It stores and returns the information provided in this form:
{
"backup_id": string # container_hostname_backupname_timestamp_level
"user_id": string, # owner of the backup metadata (OS X-User-Id, keystone provided)
"user_name": string # owner of the backup metadata (OS X-User-Name, keystone provided)
"backup_metadata": { #--- actual backup_metadata provided
"container": string,
"host_name": string,
"backup_name": string,
"timestamp": int,
...
}
}

View File

@ -0,0 +1,19 @@
[DEFAULT]
verbose = false
use_syslogd = false
logging_file = freezer-api.log
[keystone_authtoken]
identity_uri = http://keystone:35357/
auth_uri = http://keystone:5000/
admin_user = admin
admin_password = secrete
admin_tenant_name = admin
include_service_catalog = False
delay_auth_decision = False
[storage]
db=elasticsearch
endpoint=http://localhost:9200

View File

View File

View File

@ -0,0 +1,48 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import json
import falcon
class JSONTranslator(object):
def process_request(self, req, resp):
if req.content_length in (None, 0):
# Nothing to do
return
body = req.stream.read()
if not body:
raise falcon.HTTPBadRequest('Empty request body',
'A valid JSON document is required.')
try:
req.context['doc'] = json.loads(body.decode('utf-8'))
except (ValueError, UnicodeDecodeError):
raise falcon.HTTPError(falcon.HTTP_753,
'Malformed JSON')
def process_response(self, req, resp, resource):
if 'result' not in req.context:
return
resp.body = json.dumps(req.context['result'])

View File

@ -0,0 +1,48 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
from freezer_api.api.v1 import backups
from freezer_api.api.v1 import homedoc
VERSION = {
'id': '1',
'status': 'CURRENT',
'updated': '2015-03-23T13:45:00',
'links': [
{
'href': '/v1/',
'rel': 'self'
}
]
}
def public_endpoints(storage_driver):
return [
('/',
homedoc.Resource()),
('/backups',
backups.BackupsCollectionResource(storage_driver)),
('/backups/{backup_id}',
backups.BackupsResource(storage_driver))
]

View File

@ -0,0 +1,74 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import falcon
from freezer_api.common import exceptions
class BackupsCollectionResource(object):
"""
Handler for endpoint: /v1/backups
"""
def __init__(self, storage_driver):
self.db = storage_driver
def on_get(self, req, resp):
# GET /v1/backups(?limit,marker) Lists backups
user_id = req.get_header('X-User-ID')
obj_list = self.db.get_backup_list(user_id=user_id)
req.context['result'] = {'backups': obj_list}
def on_post(self, req, resp):
# POST /v1/backups Creates backup entry
try:
doc = req.context['doc']
except KeyError:
raise exceptions.BadDataFormat(
message='Missing request body',
resp_body={'error': 'missing request body'})
user_name = req.get_header('X-User-Name')
user_id = req.get_header('X-User-ID')
backup_id = self.db.add_backup(
user_id=user_id, user_name=user_name, data=doc)
resp.status = falcon.HTTP_201
req.context['result'] = {'backup_id': backup_id}
class BackupsResource(object):
"""
Handler for endpoint: /v1/backups/{backup_id}
"""
def __init__(self, storage_driver):
self.db = storage_driver
def on_get(self, req, resp, backup_id):
# GET /v1/backups/{backup_id} Get backup details
user_id = req.get_header('X-User-ID')
obj = self.db.get_backup(user_id=user_id, backup_id=backup_id)
req.context['result'] = obj
def on_delete(self, req, resp, backup_id):
# DELETE /v1/backups/{backup_id} Deletes the specified backup
user_id = req.get_header('X-User-ID')
self.db.delete_backup(
user_id=user_id, backup_id=backup_id)
req.context['result'] = {'backup_id': backup_id}
resp.status = falcon.HTTP_204

View File

@ -0,0 +1,52 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
http://tools.ietf.org/html/draft-nottingham-json-home-03
"""
import json
HOME_DOC = {
'resources': {
'rel/backups': {
'href-template': '/v1/backups/{backup_id}',
'href-vars': {
'backup_id': 'param/backup_id'
},
'hints': {
'allow': ['GET'],
'formats': {
'application/json': {},
},
},
},
}
}
class Resource(object):
def __init__(self):
document = json.dumps(HOME_DOC, ensure_ascii=False, indent=4)
self.document_utf8 = document.encode('utf-8')
def on_get(self, req, resp):
resp.data = self.document_utf8
resp.content_type = 'application/json-home'

View File

@ -0,0 +1,43 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import json
import falcon
from freezer_api.api import v1
VERSIONS = {
'versions': [
v1.VERSION
]
}
class Resource(object):
def __init__(self):
self.versions = json.dumps(VERSIONS, ensure_ascii=False)
def on_get(self, req, resp):
resp.data = self.versions
resp.status = falcon.HTTP_300

View File

View File

@ -0,0 +1,93 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import logging
import os
import sys
from wsgiref import simple_server
import falcon
from keystonemiddleware import auth_token
from oslo.config import cfg
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir, os.pardir, os.pardir))
if os.path.exists(os.path.join(possible_topdir, 'freezer_api', '__init__.py')):
sys.path.insert(0, possible_topdir)
from freezer_api.common import config, log, exceptions
from freezer_api.api import v1
from freezer_api.api import versions
from freezer_api.api.common import middleware
from freezer_api.storage import driver
def get_application(db):
app = falcon.API(middleware=[middleware.JSONTranslator()])
for exception_class in exceptions.exception_handlers_catalog:
app.add_error_handler(exception_class, exception_class.handle)
endpoint_catalog = [
('/v1', v1.public_endpoints(db)),
('/', [('', versions.Resource())])
]
for version_path, endpoints in endpoint_catalog:
for route, resource in endpoints:
app.add_route(version_path + route, resource)
if 'keystone_authtoken' in config.CONF:
app = auth_token.AuthProtocol(app, {})
else:
logging.warning("keystone authentication disabled")
return app
config_file = '/etc/freezer-api.conf'
config_files_list = [config_file] if os.path.isfile(config_file) else []
config.parse_args(args=[], default_config_files=config_files_list)
log.setup()
logging.info("Freezer API starting")
logging.info("Freezer config file(s) used: {0}".format(
', '.join(cfg.CONF.config_file)))
try:
db = driver.get_db()
application = get_application(db)
except Exception as err:
message = 'Unable to start server: {0}'.format(err)
print message
logging.fatal(message)
sys.exit(1)
def main():
ip, port = '127.0.0.1', 9090
httpd = simple_server.make_server(ip, port, application)
message = 'Server listening on {0}:{1}'.format(ip, port)
print message
logging.info(message)
try:
httpd.serve_forever()
except KeyboardInterrupt:
print "\nThanks, Bye"
sys.exit(0)
if __name__ == '__main__':
main()

View File

@ -0,0 +1,43 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
from oslo.config import cfg
common_cli_opts = [
cfg.BoolOpt('verbose',
short='v',
default=False,
help='Print more verbose output.'),
cfg.BoolOpt('debug',
short='d',
default=False,
help='Print debugging output.'),
]
CONF = cfg.CONF
CONF.register_cli_opts(common_cli_opts)
def parse_args(args=[], usage=None, default_config_files=[]):
CONF(args=args,
project='freezer',
default_config_files=default_config_files)

View File

@ -0,0 +1,83 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import falcon
import logging
class FreezerAPIException(Exception):
"""
Base Freezer API Exception
"""
json_message = ({'error': 'Unknown exception occurred'})
def __init__(self, message=None, resp_body={}):
if message:
self.message = message
self.resp_body = resp_body
logging.error(message)
Exception.__init__(self, message)
@staticmethod
def handle(ex, req, resp, params):
resp.status = falcon.HTTP_500
req.context['result'] = {'error': 'internal server error'}
class ObjectNotFound(FreezerAPIException):
@staticmethod
def handle(ex, req, resp, params):
resp.status = falcon.HTTP_404
ex.resp_body.update({'found': False})
req.context['result'] = ex.resp_body
class BadDataFormat(FreezerAPIException):
@staticmethod
def handle(ex, req, resp, params):
resp.status = falcon.HTTP_400
ex.resp_body.update({'error': 'bad data format'})
req.context['result'] = ex.resp_body
class DocumentExists(FreezerAPIException):
@staticmethod
def handle(ex, req, resp, params):
resp.status = falcon.HTTP_409
ex.resp_body.update({'error': 'document already exists'})
req.context['result'] = ex.resp_body
class StorageEngineError(FreezerAPIException):
@staticmethod
def handle(ex, req, resp, params):
resp.status = falcon.HTTP_500
ex.resp_body.update({'error': 'storage engine'})
req.context['result'] = ex.resp_body
exception_handlers_catalog = [
ObjectNotFound,
BadDataFormat,
DocumentExists,
StorageEngineError
]

View File

@ -0,0 +1,66 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
from oslo.config import cfg
import logging
logging_cli_opts = [
cfg.StrOpt('log-file',
metavar='PATH',
help='(Optional) Name of log file to output to. '
'If no default is set, logging will go to stdout.'),
cfg.BoolOpt('use-syslog',
help='Use syslog for logging.'),
cfg.StrOpt('syslog-log-facility',
help='syslog facility to receive log lines')
]
logging_opts = [
cfg.StrOpt('logging_file',
metavar='PATH',
default='freezer-api.log',
help='(Optional) Name of log file to output to. '
'If no default is set, logging will go to stdout.'),
cfg.BoolOpt('logging_use_syslog',
default=False,
help='Use syslog for logging.'),
cfg.StrOpt('logging_syslog_log_facility',
default='LOG_USER',
help='syslog facility to receive log lines')
]
CONF = cfg.CONF
CONF.register_opts(logging_opts)
CONF.register_cli_opts(logging_cli_opts)
def setup():
try:
log_file = CONF['log-file'] # cli provided
except:
log_file = CONF['logging_file'] # .conf file
logging.basicConfig(
filename=log_file,
level=logging.INFO,
format='%(asctime)s %(name)s %(levelname)s %(message)s')

View File

@ -0,0 +1,68 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
class BackupMetadataDoc:
"""
Wraps a backup_metadata dict and adds some utility methods,
and fields
"""
def __init__(self, user_id='', user_name='', data={}):
self.user_id = user_id
self.user_name = user_name
self.data = data
def is_valid(self):
try:
assert (self.backup_id is not '')
assert (self.user_id is not '')
except:
return False
return True
def serialize(self):
return {'backup_id': self.backup_id,
'user_id': self.user_id,
'user_name': self.user_name,
'backup_metadata': self.data}
@staticmethod
def un_serialize(d):
return BackupMetadataDoc(
user_id=d['user_id'],
user_name=d['user_name'],
data=d['backup_metadata'])
@property
def backup_set_id(self):
return '{0}_{1}_{2}'.format(
self.data['container'],
self.data['host_name'],
self.data['backup_name']
)
@property
def backup_id(self):
return '{0}_{1}_{2}'.format(
self.backup_set_id,
self.data['timestamp'],
self.data['level']
)

View File

@ -0,0 +1,57 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
from oslo.config import cfg
import logging
from freezer_api.storage import simpledict, elastic
opt_group = cfg.OptGroup(name='storage',
title='Freezer Storage Engine')
storage_opts = [
cfg.StrOpt('db',
default='simpledict',
help='specify the storage db to use: simpledoct (default),'
' elasticsearch'),
cfg.StrOpt('endpoint',
default='http://localhost:9200',
help='specify the storage endpoint')
]
CONF = cfg.CONF
CONF.register_group(opt_group)
CONF.register_opts(storage_opts, opt_group)
def get_db():
db_engine = CONF.storage.db
if db_engine == 'simpledict':
logging.info('Storage backend: simple dictionary')
db = simpledict.SimpleDictStorageEngine()
elif db_engine == 'elasticsearch':
endpoint = CONF.storage.endpoint
logging.info('Storage backend: Elasticsearch at {0}'.format(endpoint))
db = elastic.ElasticSearchEngine(endpoint)
else:
raise Exception('Database Engine {0} not supported'.format(db_engine))
return db

View File

@ -0,0 +1,110 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import elasticsearch
import logging
from freezer_api.common.utils import BackupMetadataDoc
from freezer_api.common import exceptions
class ElasticSearchEngine(object):
def __init__(self, hosts):
# logging.getLogger('elasticsearch').addHandler(logging.NullHandler())
self.es = elasticsearch.Elasticsearch(hosts)
logging.info('Using Elasticsearch host {0}'.format(hosts))
self.index = "freezer"
def _get_backup(self, user_id, backup_id=None):
# raises only on severe engine errors
if backup_id:
query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id)
else:
query = '+user_id:{0}'.format(user_id)
try:
res = self.es.search(index=self.index, doc_type='backups',
q=query)
except Exception as e:
raise exceptions.StorageEngineError(
message='search operation failed',
resp_body={'engine exception': '{0}'.format(e)})
hit_list = res['hits']['hits']
return [x['_source'] for x in hit_list]
def _index(self, doc):
# raises only on severe engine errors
try:
res = self.es.index(index=self.index, doc_type='backups',
body=doc)
except Exception as e:
raise exceptions.StorageEngineError(
message='index operation failed',
resp_body={'engine exception': '{0}'.format(e)})
return res['created']
def _delete_backup(self, user_id, backup_id):
query = '+user_id:{0} +backup_id:{1}'.format(user_id, backup_id)
try:
self.es.delete_by_query(index=self.index,
doc_type='backups',
q=query)
except Exception as e:
raise exceptions.StorageEngineError(
message='search operation failed',
resp_body={'engine exception': '{0}'.format(e)})
def get_backup(self, user_id, backup_id):
# raises is data not found, so reply will be HTTP_404
backup_metadata = self._get_backup(user_id, backup_id)
if not backup_metadata:
raise exceptions.ObjectNotFound(
message='Requested backup data not found: {0}'.
format(backup_id),
resp_body={'backup_id': backup_id})
return backup_metadata
def get_backup_list(self, user_id):
# TODO: elasticsearch reindex for paging
return self._get_backup(user_id)
def add_backup(self, user_id, user_name, data):
# raises if data is malformed (HTTP_400) or already present (HTTP_409)
backup_metadata_doc = BackupMetadataDoc(user_id, user_name, data)
if not backup_metadata_doc.is_valid():
raise exceptions.BadDataFormat(message='Bad Data Format')
backup_id = backup_metadata_doc.backup_id
existing_data = self._get_backup(user_id, backup_id)
if existing_data:
raise exceptions.DocumentExists(
message='Backup data already existing ({0})'.format(backup_id),
resp_body={'backup_id': backup_id})
if not self._index(backup_metadata_doc.serialize()):
# should never happen
raise exceptions.StorageEngineError(
message='index operation failed',
resp_body={'backup_id': backup_id})
logging.info('Backup metadata indexed, backup_id: {0}'.
format(backup_id))
return backup_id
def delete_backup(self, user_id, backup_id):
self._delete_backup(user_id, backup_id)
return backup_id

View File

@ -0,0 +1,69 @@
"""
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import logging
from freezer_api.common.utils import BackupMetadataDoc
from freezer_api.common import exceptions
class SimpleDictStorageEngine(object):
def __init__(self):
self._map = {}
def get_backup(self, user_id, backup_id):
try:
backup_data = self._map[(user_id, backup_id)]
except:
raise exceptions.ObjectNotFound(
message='Requested backup data not found: {0}'.
format(backup_id),
resp_body={'backup_id': backup_id})
return backup_data
def get_backup_list(self, user_id):
backup_list = []
for (key, backup_data) in self._map.iteritems():
if key[0] == user_id:
backup_list.append(backup_data)
return backup_list
def add_backup(self, user_id, user_name, data):
backup_metadata_doc = BackupMetadataDoc(user_id, user_name, data)
if not backup_metadata_doc.is_valid():
raise exceptions.BadDataFormat(message='Bad Data Format')
backup_id = backup_metadata_doc.backup_id
if (user_id, backup_id) in self._map:
raise exceptions.DocumentExists(
message='Backup data already existing ({0})'.format(backup_id),
resp_body={'backup_id': backup_id})
self._map[(user_id, backup_id)] = backup_metadata_doc.serialize()
logging.info('Adding backup data with backup_id {0}'.format(backup_id))
return backup_id
def delete_backup(self, user_id, backup_id):
try:
self._map.pop((user_id, backup_id))
except KeyError:
raise exceptions.ObjectNotFound(
message='Object to remove not found: {0}'.format(backup_id),
resp_body={'backup_id': backup_id})
return backup_id

50
freezer_api/setup.cfg Normal file
View File

@ -0,0 +1,50 @@
[metadata]
name = freezer_api
version = 2015.1
summary = OpenStack Backup and Restore Service
description-file =
README.rst
author = Fausto Marzi, Fabrizio Fresco, Fabrizio Vanni',
author_email = fausto.marzi@hp.com, fabrizio.vanni@hp.com, fabrizio.fresco@hp.com
home-page = https://github.com/stackforge/freezer
classifier =
Environment :: OpenStack
Programming Language :: Python
Development Status :: 5 - Production/Stable
Natural Language :: English
Intended Audience :: Developers
Intended Audience :: Financial and Insurance Industry
Intended Audience :: Information Technology
Intended Audience :: System Administrators
Intended Audience :: Telecommunications Industry
License :: OSI Approved :: Apache Software License
Operating System :: MacOS
Operating System :: POSIX :: BSD :: FreeBSD
Operating System :: POSIX :: BSD :: NetBSD
Operating System :: POSIX :: BSD :: OpenBSD
Operating System :: POSIX :: Linux
Operating System :: Unix
Topic :: System :: Archiving :: Backup
Topic :: System :: Archiving :: Compression
Topic :: System :: Archiving
[files]
packages =
freezer_api
data_files =
/etc = etc/*
[entry_points]
console_scripts =
freezer-api = freezer_api.cmd.api:main
[pytests]
where=tests
verbosity=2
[pbr]
warnerrors = True

8
freezer_api/setup.py Normal file
View File

@ -0,0 +1,8 @@
#!/usr/bin/env python
from setuptools import setup
setup(
setup_requires=['pbr'],
pbr=True,
)

Binary file not shown.

Binary file not shown.

View File

301
freezer_api/tests/common.py Normal file
View File

@ -0,0 +1,301 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import io
fake_data_0_backup_id = 'freezer_container_alpha_important_data_backup_8475903425_0'
fake_data_0_user_id = 'qwerty1234'
fake_data_0_user_name = 'asdffdsa'
fake_data_0_wrapped_backup_metadata = {
'backup_id': 'freezer_container_alpha_important_data_backup_8475903425_0',
'user_id': 'qwerty1234',
'user_name': 'asdffdsa',
'backup_metadata': {
"container": "freezer_container",
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 8475903425,
"level": 0,
"backup_session": 8475903425,
"max_level": 5,
"mode" : "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files" : 11,
"total_directories" : 2,
"backup_size_uncompressed" : 4567,
"backup_size_compressed" : 1212,
"total_backup_session_size" : 6789,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
"cli": ""
}
}
fake_data_0_backup_metadata = {
"container": "freezer_container",
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 8475903425,
"level": 0,
"backup_session": 8475903425,
"max_level": 5,
"mode": "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files" : 11,
"total_directories" : 2,
"backup_size_uncompressed" : 4567,
"backup_size_compressed" : 1212,
"total_backup_session_size" : 6789,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
"cli": ""
}
fake_malformed_data_0_backup_metadata = {
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 8475903425,
"level": 0,
"backup_session": 8475903425,
"max_level": 5,
"mode": "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files" : 11,
"total_directories" : 2,
"backup_size_uncompressed" : 4567,
"backup_size_compressed" : 1212,
"total_backup_session_size" : 6789,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
"cli": ""
}
fake_data_0_elasticsearch_hit = {
"_shards": {
"failed": 0,
"successful": 5,
"total": 5
},
"hits": {
"hits": [
{
"_id": "AUx_iu-ewlhuOVELWtH0",
"_index": "freezer",
"_score": 1.0,
"_type": "backups",
"_source": {
"container": "freezer_container",
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 8475903425,
"level": 0,
"backup_session": 8475903425,
"max_level": 5,
"mode" : "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files" : 11,
"total_directories" : 2,
"backup_size_uncompressed" : 4567,
"backup_size_compressed" : 1212,
"total_backup_session_size" : 6789,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
"cli": ""
}
}
],
"max_score": 1.0,
"total": 1
},
"timed_out": False,
"took": 3
}
fake_data_0_elasticsearch_miss = {
"_shards": {
"failed": 0,
"successful": 5,
"total": 5
},
"hits": {
"hits": [],
"max_score": None,
"total": 0
},
"timed_out": False,
"took": 1
}
class FakeReqResp:
def __init__(self, method='GET', body=''):
self.method = method
self.body = body
self.stream = io.BytesIO(body)
self.content_length = len(body)
self.context = {}
self.header = {}
def get_header(self, key):
return self.header.get(key, None)
class FakeElasticsearch_hit:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
return fake_data_0_elasticsearch_hit
def index(self, index, doc_type, body):
return {'created': True}
def delete_by_query(self, index, doc_type, q):
pass
class FakeElasticsearch_insert_ok:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
return fake_data_0_elasticsearch_miss
def index(self, index, doc_type, body):
return {'created': True}
def delete_by_query(self, index, doc_type, q):
pass
class FakeElasticsearch_miss:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
return fake_data_0_elasticsearch_miss
def index(self, index, doc_type, body):
return {'created': False}
def delete_by_query(self, index, doc_type, q):
pass
class FakeElasticsearch_index_raise:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
return fake_data_0_elasticsearch_miss
def index(self, index, doc_type, body):
raise Exception
def delete_by_query(self, index, doc_type, q):
pass
class FakeElasticsearch_search_raise:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
raise Exception
def index(self, index, doc_type, body):
return {'created': True}
def delete_by_query(self, index, doc_type, q):
pass
class FakeElasticsearch_delete_raise:
def __init__(self, host=None):
pass
def search(self, index, doc_type, q):
return fake_data_0_elasticsearch_miss
def index(self, index, doc_type, body):
return {'created': True}
def delete_by_query(self, index, doc_type, q):
raise Exception
class FakeLogging:
def __init__(self):
return None
def __call__(self, *args, **kwargs):
return True
@classmethod
def logging(cls, opt1=True):
return True
@classmethod
def info(cls, opt1=True):
return True
@classmethod
def warning(cls, opt1=True):
return True
@classmethod
def critical(cls, opt1=True):
return True
@classmethod
def exception(cls, opt1=True):
return True
@classmethod
def error(cls, opt1=True):
return True

View File

@ -0,0 +1,46 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import pytest
from common import *
from freezer_api.common.exceptions import *
from keystonemiddleware import auth_token
from freezer_api.cmd import api
class TestAPI:
def patch_logging(self, monkeypatch):
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
def test_auth_install(self, monkeypatch):
self.patch_logging(monkeypatch)
app = api.get_application(None)
assert isinstance(app, auth_token.AuthProtocol)

View File

@ -0,0 +1,118 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
import falcon
from freezer_api.api.v1 import backups
from freezer_api.storage import simpledict
from common import *
from freezer_api.common.exceptions import *
class TestBackupsCollectionResource(unittest.TestCase):
def setUp(self):
self.db = simpledict.SimpleDictStorageEngine()
self.resource = backups.BackupsCollectionResource(self.db)
self.req = FakeReqResp()
self.req.header['X-User-ID'] = fake_data_0_user_id
def test_on_get_return_empty_list(self):
expected_result = {'backups': []}
self.resource.on_get(self.req, self.req)
result = self.req.context['result']
self.assertEqual(result, expected_result)
def test_on_get_return_correct_list(self):
self.db.add_backup(user_id=fake_data_0_user_id,
user_name=fake_data_0_user_name,
data=fake_data_0_backup_metadata)
self.resource.on_get(self.req, self.req)
result = self.req.context['result']
expected_result = {'backups': [fake_data_0_wrapped_backup_metadata]}
self.assertEqual(result, expected_result)
def test_on_get_return_empty_list_without_user_id(self):
self.req.header.pop('X-User-ID')
self.db.add_backup(user_id=fake_data_0_user_id,
user_name=fake_data_0_user_name,
data=fake_data_0_backup_metadata)
self.resource.on_get(self.req, self.req)
result = self.req.context['result']
expected_result = {'backups': []}
self.assertEqual(result, expected_result)
def test_on_get_return_empty_list_with_different_user_id(self):
self.req.header['X-User-ID'] = 'LupinIII'
self.db.add_backup(user_id=fake_data_0_user_id,
user_name=fake_data_0_user_name,
data=fake_data_0_backup_metadata)
self.resource.on_get(self.req, self.req)
result = self.req.context['result']
expected_result = {'backups': []}
self.assertEqual(result, expected_result)
def test_on_post_raises_when_missing_body(self):
self.assertRaises(BadDataFormat, self.resource.on_post, self.req, self.req)
def test_on_post_inserts_correct_data(self):
self.req.context['doc'] = fake_data_0_backup_metadata
self.resource.on_post(self.req, self.req)
self.assertEquals(self.req.status, falcon.HTTP_201)
expected_result = {'backup_id': fake_data_0_backup_id}
self.assertEquals(self.req.context['result'], expected_result)
class TestBackupsResource(unittest.TestCase):
def setUp(self):
self.db = simpledict.SimpleDictStorageEngine()
self.resource = backups.BackupsResource(self.db)
self.req = FakeReqResp()
self.req.header['X-User-ID'] = fake_data_0_user_id
def test_on_get_raises_when_not_found(self):
self.assertRaises(ObjectNotFound, self.resource.on_get, self.req, self.req, fake_data_0_backup_id)
def test_on_get_return_correct_data(self):
self.db.add_backup(user_id=fake_data_0_user_id,
user_name=fake_data_0_user_name,
data=fake_data_0_backup_metadata)
self.resource.on_get(self.req, self.req, fake_data_0_backup_id)
result = self.req.context['result']
self.assertEqual(result, fake_data_0_wrapped_backup_metadata)
def test_on_delete_raises_when_not_found(self):
self.assertRaises(ObjectNotFound, self.resource.on_delete, self.req, self.req, fake_data_0_backup_id)
def test_on_delete_removes_proper_data(self):
self.db.add_backup(user_id=fake_data_0_user_id,
user_name=fake_data_0_user_name,
data=fake_data_0_backup_metadata)
self.resource.on_delete(self.req, self.req, fake_data_0_backup_id)
result = self.req.context['result']
expected_result = {'backup_id': fake_data_0_backup_id}
self.assertEquals(self.req.status, falcon.HTTP_204)
self.assertEqual(result, expected_result)

View File

View File

@ -0,0 +1,62 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
import pytest
import falcon
from common import *
from freezer_api.common.exceptions import *
from oslo.config import cfg
from freezer_api.storage import driver, elastic, simpledict
class TestStorageDriver:
def patch_logging(self, monkeypatch):
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
def test_get_db_raises_when_db_not_supported(self, monkeypatch):
self.patch_logging(monkeypatch)
cfg.CONF.storage.db = 'nodb'
pytest.raises(Exception, driver.get_db)
def test_get_db_simpledict(self, monkeypatch):
self.patch_logging(monkeypatch)
cfg.CONF.storage.db = 'simpledict'
db = driver.get_db()
assert isinstance(db, simpledict.SimpleDictStorageEngine)
def test_get_db_elastic(self, monkeypatch):
self.patch_logging(monkeypatch)
cfg.CONF.storage.db = 'elasticsearch'
db = driver.get_db()
assert isinstance(db, elastic.ElasticSearchEngine)

View File

@ -0,0 +1,124 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
import pytest
from freezer_api.storage import elastic
from common import *
from freezer_api.common.exceptions import *
import elasticsearch
class TestElasticSearchEngine:
def patch_logging(self, monkeypatch):
fakelogging = FakeLogging()
monkeypatch.setattr(logging, 'critical', fakelogging.critical)
monkeypatch.setattr(logging, 'warning', fakelogging.warning)
monkeypatch.setattr(logging, 'exception', fakelogging.exception)
monkeypatch.setattr(logging, 'error', fakelogging.error)
class TestElasticSearchEngine_get_backup(TestElasticSearchEngine):
def test_get_backup_userid_and_backup_id_return_ok(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit)
engine = elastic.ElasticSearchEngine('host')
res = engine.get_backup(fake_data_0_user_id, fake_data_0_backup_id)
assert (res == [fake_data_0_backup_metadata, ])
def test_get_backup_raises_when_query_has_no_hits(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_miss)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(ObjectNotFound, engine.get_backup, fake_data_0_user_id, fake_data_0_backup_id)
class TestElasticSearchEngine_get_backup_list(TestElasticSearchEngine):
def test_get_backup_list_return_ok(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit)
engine = elastic.ElasticSearchEngine('host')
res = engine.get_backup_list(fake_data_0_user_id)
assert (res == [fake_data_0_backup_metadata, ])
class TestElasticSearchEngine_add_backup(TestElasticSearchEngine):
def test_index_backup_success(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_insert_ok)
engine = elastic.ElasticSearchEngine('host')
res = engine.add_backup(fake_data_0_user_id, fake_data_0_user_name, fake_data_0_backup_metadata)
assert (res == fake_data_0_backup_id)
def test_index_backup_raise_when_data_exists(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(DocumentExists, engine.add_backup, fake_data_0_user_id,
fake_data_0_user_name, fake_data_0_backup_metadata)
def test_index_backup_raise_when_es_index_raises(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_index_raise)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(StorageEngineError, engine.add_backup, fake_data_0_user_id,
fake_data_0_user_name, fake_data_0_backup_metadata)
def test_index_backup_raise_when_es_search_raises(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_search_raise)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(StorageEngineError, engine.add_backup, fake_data_0_user_id,
fake_data_0_user_name, fake_data_0_backup_metadata)
def test_index_backup_raise_when_data_is_malformed(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_insert_ok)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(BadDataFormat, engine.add_backup, fake_data_0_user_id,
fake_data_0_user_name, fake_malformed_data_0_backup_metadata)
class TestElasticSearchEngine_delete_backup(TestElasticSearchEngine):
def test_delete_backup_raise_when_es_delete_raises(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_delete_raise)
engine = elastic.ElasticSearchEngine('host')
pytest.raises(StorageEngineError, engine.delete_backup, fake_data_0_user_id, fake_data_0_backup_id)
def test_delete_backup_ok(self, monkeypatch):
self.patch_logging(monkeypatch)
monkeypatch.setattr(elasticsearch, 'Elasticsearch', FakeElasticsearch_hit)
engine = elastic.ElasticSearchEngine('host')
res = engine.delete_backup(fake_data_0_user_id, fake_data_0_backup_id)
assert (res == fake_data_0_backup_id)

View File

@ -0,0 +1,39 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
from common import FakeReqResp
from freezer_api.api import v1
import json
class TestHomedocResource(unittest.TestCase):
def setUp(self):
self.resource = v1.homedoc.Resource()
self.req = FakeReqResp()
def test_on_get_return_resources_information(self):
self.resource.on_get(self.req, self.req)
result = json.loads(self.req.data)
expected_result = v1.homedoc.HOME_DOC
self.assertEquals(result, expected_result)

View File

@ -0,0 +1,62 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import json
import unittest
import falcon
from freezer_api.api.common import middleware
from common import FakeReqResp
class TestBackupMetadataDoc(unittest.TestCase):
def setUp(self):
self.json_translator = middleware.JSONTranslator()
def test_process_request_with_no_body_returns_none(self):
req = FakeReqResp()
self.assertIsNone(self.json_translator.process_request(req, req))
def test_process_request_with_positive_length_and_no_body_raises(self):
req = FakeReqResp()
req.content_length = 1
self.assertRaises(falcon.HTTPBadRequest, self.json_translator.process_request, req, req)
def test_process_response_with_no_result_returns_none(self):
req = FakeReqResp()
self.assertIsNone(self.json_translator.process_response(req, req, None))
def test_process_response_create_correct_json_body(self):
req = FakeReqResp()
d = {'key1': 'value1', 'key2': 'value2'}
req.context['result'] = d
correct_json_body = json.dumps(d)
self.json_translator.process_response(req, req, None)
self.assertEqual(correct_json_body, req.body)
def test_process_request_with_malformed_body_raises(self):
req = FakeReqResp(body='{"key2": "value2",{ "key1": "value1"}')
self.assertRaises(falcon.HTTPError, self.json_translator.process_request, req, req)

View File

View File

@ -0,0 +1,111 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
from freezer_api.common import utils
DATA_backup_metadata = {
"container": "freezer_container",
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 12341234,
"level": 0,
"backup_session": 12341234,
"max_level": 5,
"mode" : "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files": 11,
"total_directories": 2,
"backup_size_uncompressed": 12112342,
"backup_size_compressed": 1214322,
"total_backup_session_size": 1212,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
'cli': 'whatever'
}
DATA_user_id = 'AUx6F07NwlhuOVELWtHx'
DATA_user_name = 'gegrex55NPlwlhuOVELWtHv'
DATA_backup_id = 'freezer_container_alpha_important_data_backup_12341234_0'
DATA_wrapped_backup_metadata = {
'user_id': DATA_user_id,
'user_name': DATA_user_name,
'backup_id': DATA_backup_id,
'backup_medatada': {
"container": "freezer_container",
"host_name": "alpha",
"backup_name": "important_data_backup",
"timestamp": 12341234,
"level": 0,
"backup_session": 12341234,
"max_level": 5,
"mode": "fs",
"fs_real_path": "/blabla",
"vol_snap_path": "/blablasnap",
"total_broken_links" : 0,
"total_fs_files": 11,
"total_directories": 2,
"backup_size_uncompressed": 12112342,
"backup_size_compressed": 1214322,
"total_backup_session_size": 1212,
"compression_alg": "None",
"encrypted": "false",
"client_os": "linux",
"broken_links": ["link_01", "link_02"],
"excluded_files": ["excluded_file_01", "excluded_file_02"],
'cli': 'whatever'
}
}
class TestBackupMetadataDoc(unittest.TestCase):
def setUp(self):
self.backup_metadata = utils.BackupMetadataDoc(
user_id=DATA_user_id,
user_name=DATA_user_name,
data=DATA_backup_metadata)
def test_backup_id(self):
assert (self.backup_metadata.backup_id == DATA_backup_id)
def test_is_valid_return_True_when_valid(self):
self.assertTrue(self.backup_metadata.is_valid())
def test_is_valid_returns_False_when_user_id_empty(self):
self.backup_metadata.user_id = ''
self.assertFalse(self.backup_metadata.is_valid())
def test_backup_id_correct(self):
self.assertEqual(self.backup_metadata.backup_id, DATA_backup_id)
self.backup_metadata.data['container'] = 'different'
self.assertNotEqual(self.backup_metadata.backup_id, DATA_backup_id)

View File

@ -0,0 +1,41 @@
"""Freezer swift.py related tests
Copyright 2014 Hewlett-Packard
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
This product includes cryptographic software written by Eric Young
(eay@cryptsoft.com). This product includes software written by Tim
Hudson (tjh@cryptsoft.com).
========================================================================
"""
import unittest
import falcon
from common import FakeReqResp
from freezer_api.api import versions
from freezer_api.api import v1
import json
class TestVersionResource(unittest.TestCase):
def setUp(self):
self.resource = versions.Resource()
self.req = FakeReqResp()
def test_on_get_return_versions(self):
self.resource.on_get(self.req, self.req)
self.assertEquals(self.req.status, falcon.HTTP_300)
expected_result = json.dumps({'versions': [v1.VERSION]})
self.assertEquals(self.req.data, expected_result)

33
freezer_api/tox.ini Normal file
View File

@ -0,0 +1,33 @@
[tox]
envlist = py27,pep8
skipsdist = True
[testenv]
usedevelop = True
deps =
pytest
coverage
flake8
pytest-cov
pytest-xdist
pymysql
falcon
keystonemiddleware
elasticsearch
install_command = pip install -U {opts} {packages}
setenv = VIRTUAL_ENV={envdir}
commands =
py.test -v --cov-report term-missing --cov freezer_api
[pytest]
python_files = test_*.py
norecursedirs = .tox .venv specs
[testenv:pep8]
commands = flake8 freezer_api
[flake8]
show-source = True
exclude = .venv,.tox,dist,doc,test,*egg,tests,specs,build

View File

@ -3,6 +3,7 @@
from setuptools import setup, find_packages
from setuptools.command.test import test as TestCommand
import os
import io
here = os.path.abspath(os.path.dirname(__file__))

View File

@ -18,7 +18,7 @@ commands = python runtests.py -v -n 2 --cov-report term-missing --cov freezer
[pytest]
python_files = test_*.py
norecursedirs = .tox .venv
norecursedirs = .tox .venv freezer_api
[testenv:pep8]
commands = flake8 freezer