openstack/freezer
Code Issues Proposed changes

list backups stored in API

Browse Source 
The freezer-scheduler can now be used to query the list
of backups stored in the API.
Two new fields are added to the backup-metadata doc to
better identify the resources:

  client-id
  job-id

Adds the following action to the freezer-scheduler:

  backup-list

Adds the following options to the freezer-scheduler parameters:

  --all   Used when querying the API, to retrieve any document
          regardless of the client-id

  --long  List additional fields in output

Implements blueprint: list-stored-backups

Change-Id: I7c6e666d94c64c25d340fc9826628fb001b4b7f6
This commit is contained in:
Fabrizio Vanni 2016-02-08 11:53:05 +00:00
parent 1bf965888f
commit 87b4ab66b8
5 changed files with 111 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
freezer/apiclient/backups.py
View File

@ -32,6 +32,8 @@ class BackupsManager(object):
return {'X-Auth-Token': self.client.auth_token}
def create(self, backup_metadata):
if not backup_metadata.get('client_id', ''):
backup_metadata['client_id'] = self.client.client_id
r = requests.post(self.endpoint,
data=json.dumps(backup_metadata),
headers=self.headers,
@ -48,7 +50,7 @@ class BackupsManager(object):
if r.status_code != 204:
raise exceptions.ApiClientException(r)
def list(self, limit=10, offset=0, search=None):
def list_all(self, limit=10, offset=0, search=None):
"""
Retrieves a list of backup infos
@ -70,6 +72,13 @@ class BackupsManager(object):
return r.json()['backups']
def list(self, limit=10, offset=0, search={}, client_id=None):
client_id = client_id or self.client.client_id
new_search = search.copy()
new_search['match'] = search.get('match', [])
new_search['match'].append({'client_id': client_id})
return self.list_all(limit, offset, new_search)
def get(self, backup_id):
endpoint = self.endpoint + backup_id
r = requests.get(endpoint, headers=self.headers, verify=self.verify)

11
freezer/scheduler/arguments.py
View File

@ -105,7 +105,16 @@ def get_common_opts():
default=False,
dest='disable_exec',
help='Allow Freezer Scheduler to deny jobs that execute '
'commands for security reasons')
'commands for security reasons'),
cfg.BoolOpt('all',
default=False,
dest='all',
help='Used when querying the API, to retrieve any '
'document regardless of the client-id'),
cfg.BoolOpt('long',
default=False,
dest='long',
help='List additional fields in output')
]
return _COMMON

1
freezer/scheduler/scheduler_job.py
View File

@ -275,6 +275,7 @@ class Job(object):
try:
metadata = json.loads(metadata_string)
if metadata:
metadata['job_id'] = self.id
self.scheduler.upload_metadata(metadata)
logging.info("[*] Job {0}, freezer action metadata uploaded"
.format(self.id))

48
freezer/scheduler/shell.py
View File

@ -21,6 +21,8 @@ import os
import six
import utils
from freezer.utils.utils import DateTime
from prettytable import PrettyTable
try:
@ -191,21 +193,22 @@ def do_job_upload(client, args):
def _job_list(client, args):
list_func = client.jobs.list_all if args.all else client.jobs.list
search = {}
if args.active_only:
search = {"match_not": [{"status": "completed"}]}
client_docs = client.jobs.list(search=search)
client_docs = list_func(search=search)
offset = 0
while client_docs:
offset += len(client_docs)
for doc in client_docs:
yield doc
client_docs = client.jobs.list(offset=offset, search=search)
client_docs = list_func(offset=offset, search=search)
raise StopIteration
def do_job_list(client, args):
table = PrettyTable(["job_id", "description", "# actions",
table = PrettyTable(["job_id", "client-id", "description", "# actions",
"status", "event", "result", "session_id"])
for doc in _job_list(client, args):
job_scheduling = doc.get('job_schedule', {})
@ -213,6 +216,7 @@ def do_job_list(client, args):
job_event = job_scheduling.get('event', '')
job_result = job_scheduling.get('result', '')
table.add_row([doc['job_id'],
doc.get('client_id'),
doc.get('description', ''),
len(doc.get('job_actions', [])),
job_status,
@ -235,3 +239,41 @@ def do_client_list(client, args):
client_doc.get('description', '')])
l = client.registration.list(offset=offset)
print(table)
def do_backup_list(client, args):
list_func = client.backups.list_all if args.all else client.backups.list
if args.long:
fields = ["backup uuid", "job-id", "client-id", "container",
"hostname", "backup name", "timestamp", "level", "path"]
else:
fields = ["backup uuid", "container", "backup name", "timestamp",
"level", "path"]
table = PrettyTable(fields)
l = list_func()
offset = 0
while l:
offset += len(l)
for doc in l:
metadata_doc = doc['backup_metadata']
timestamp = int(metadata_doc.get('time_stamp', 0))
if args.long:
row = [doc['backup_uuid'],
metadata_doc.get('job_id', ''),
metadata_doc.get('client_id', ''),
metadata_doc.get('container', ''),
metadata_doc.get('hostname', ''),
metadata_doc.get('backup_name', ''),
str(DateTime(timestamp)),
metadata_doc.get('curr_backup_level', ''),
metadata_doc.get('fs_real_path', '')]
else:
row = [doc['backup_uuid'],
metadata_doc.get('container', ''),
metadata_doc.get('backup_name', ''),
str(DateTime(timestamp)),
metadata_doc.get('curr_backup_level', ''),
metadata_doc.get('fs_real_path', '')]
table.add_row(row)
l = list_func(offset=offset)
print(table)

51
tests/unit/apiclient/test_apiclient_backup.py
View File

@ -15,6 +15,7 @@ limitations under the License.
"""
import json
import unittest
from mock import Mock, patch
@ -22,10 +23,30 @@ from freezer.apiclient import exceptions
from freezer.apiclient import backups
class CallArgs(object):
def __init__(self, mock_obj=Mock()):
self.args = mock_obj.call_args[0]
self.kwargs = mock_obj.call_args[1]
def _check_arg(self, arg, value, load_json):
if load_json:
arg = json.loads(arg)
if (arg != value):
raise Exception("Argument doesn't match: {0} != {1}".format(arg, value))
return
def check_arg(self, pos, value, load_json=False):
if isinstance(pos, int):
return self._check_arg(self.args[pos], value, load_json)
return self._check_arg(self.kwargs[pos], value, load_json)
class TestBackupManager(unittest.TestCase):
def setUp(self):
self.mock_client = Mock()
self.mock_client.client_id = "jumpingjack"
self.mock_client.endpoint = 'http://testendpoint:9999'
self.mock_client.auth_token = 'testtoken'
self.b = backups.BackupsManager(self.mock_client)
@ -42,6 +63,25 @@ class TestBackupManager(unittest.TestCase):
mock_response.json.return_value = {'backup_id': 'qwerqwer'}
mock_requests.post.return_value = mock_response
retval = self.b.create(backup_metadata={'backup': 'metadata'})
ca = CallArgs(mock_requests.post)
ca.check_arg(0, 'http://testendpoint:9999/v1/backups/')
ca.check_arg('data', {"backup": "metadata", "client_id": "jumpingjack"}, load_json=True)
ca.check_arg('headers', {'X-Auth-Token': 'testtoken'})
ca.check_arg('verify', True)
self.assertEqual('qwerqwer', retval)
@patch('freezer.apiclient.backups.requests')
def test_create_ok_with_client_id(self, mock_requests):
mock_response = Mock()
mock_response.status_code = 201
mock_response.json.return_value = {'backup_id': 'qwerqwer'}
mock_requests.post.return_value = mock_response
retval = self.b.create(backup_metadata={'backup': 'metadata', 'client_id': 'wahwah'})
ca = CallArgs(mock_requests.post)
ca.check_arg(0, 'http://testendpoint:9999/v1/backups/')
ca.check_arg('data', {"backup": "metadata", "client_id": "wahwah"}, load_json=True)
ca.check_arg('headers', {'X-Auth-Token': 'testtoken'})
ca.check_arg('verify', True)
self.assertEqual('qwerqwer', retval)
@patch('freezer.apiclient.backups.requests')
@ -112,12 +152,11 @@ class TestBackupManager(unittest.TestCase):
retval = self.b.list(limit=5,
offset=5,
search={"time_before": 1428529956})
mock_requests.get.assert_called_with(
'http://testendpoint:9999/v1/backups/',
params={'limit': 5, 'offset': 5},
data='{"time_before": 1428529956}',
headers={'X-Auth-Token': 'testtoken'},
verify=True)
ca = CallArgs(mock_requests.get)
ca.check_arg(0, 'http://testendpoint:9999/v1/backups/')
ca.check_arg('data', {"time_before": 1428529956, "match": [{"client_id": "jumpingjack"}]}, load_json=True)
ca.check_arg('headers', {'X-Auth-Token': 'testtoken'})
ca.check_arg('verify', True)
self.assertEqual(backup_list, retval)
@patch('freezer.apiclient.backups.requests')