Command for changing logging level.

And pep8 fixing
(suddenly this pull request was rejected by jenkins and pep8.
so this commit also contains some small style changes for pep8.)

Currently the default value for logging is INFO and it is hardcoded.
Additional command will be helpful for debug/development process.

Implements blueprint: logging-level-cli

Change-Id: I1105e3da277e2a948f3ff50b04751169eddf26ed
This commit is contained in:
eldar nugaev 2016-01-12 16:41:24 +00:00
parent 276dc51ed6
commit 8f4b53da68
24 changed files with 160 additions and 159 deletions

View File

@ -17,7 +17,7 @@ limitations under the License.
import json
import requests
import exceptions
from freezer.apiclient import exceptions
class BackupsManager(object):

View File

@ -16,20 +16,19 @@ limitations under the License.
client interface to the Freezer API
"""
import argparse
import os
import socket
from keystoneclient.auth.identity import v2
from keystoneclient.auth.identity import v3
from keystoneclient import session as ksc_session
from backups import BackupsManager
from registration import RegistrationManager
from jobs import JobManager
from actions import ActionManager
from sessions import SessionManager
from oslo_config import cfg
from freezer.apiclient import actions
from freezer.apiclient import backups
from freezer.apiclient import jobs
from freezer.apiclient import registration
from freezer.apiclient import sessions
from freezer.utils import Namespace
CONF = cfg.CONF
@ -225,11 +224,11 @@ class Client(object):
self._session = session
self.version = version
self.backups = BackupsManager(self, verify=verify)
self.registration = RegistrationManager(self, verify=verify)
self.jobs = JobManager(self, verify=verify)
self.actions = ActionManager(self, verify=verify)
self.sessions = SessionManager(self, verify=verify)
self.backups = backups.BackupsManager(self, verify=verify)
self.registration = registration.RegistrationManager(self, verify=verify)
self.jobs = jobs.JobManager(self, verify=verify)
self.actions = actions.ActionManager(self, verify=verify)
self.sessions = sessions.SessionManager(self, verify=verify)
@cached_property

View File

@ -38,7 +38,7 @@ class ApiClientException(Exception):
message = "[*] Error {0}: {1}".format(
r.status_code,
body['description'])
except:
except Exception:
message = None
return message
@ -55,7 +55,7 @@ class ApiClientException(Exception):
message = "[*] Error {0}: {1}".format(
r.status_code,
r.text)
except:
except Exception:
message = None
return message
@ -65,6 +65,6 @@ class ApiClientException(Exception):
str(r)
try:
self.status_code = r.status_code
except:
except Exception:
self.status_code = None
super(ApiClientException, self).__init__(message)

View File

@ -17,7 +17,7 @@ limitations under the License.
import json
import requests
import exceptions
from freezer.apiclient import exceptions
class SessionManager(object):

View File

@ -16,28 +16,26 @@
from __future__ import print_function
from freezer import winutils
from freezer import config
import argparse
try:
import configparser
except ImportError:
import ConfigParser as configparser
import logging
import os
from os.path import expanduser
import socket
import sys
from freezer import utils
from distutils import spawn as distspawn
from oslo_utils import encodeutils
from tempfile import NamedTemporaryFile
home = expanduser("~")
from freezer import config
from freezer import utils
from freezer import winutils
home = os.path.expanduser("~")
DEFAULT_LVM_SNAPNAME = 'freezer_backup_snap'
DEFAULT_LVM_SNAPSIZE = '1G'
@ -63,7 +61,7 @@ DEFAULT_PARAMS = {
'remove_older_than': None, 'restore_from_date': False,
'upload_limit': None, 'always_level': False, 'version': False,
'dry_run': False, 'lvm_snapsize': DEFAULT_LVM_SNAPSIZE,
'restore_abs_path': False, 'log_file': None,
'restore_abs_path': False, 'log_file': None, 'log_level': "info",
'mode': 'fs', 'action': 'backup',
'vssadmin': False, 'shadow': '', 'shadow_path': '',
'windows_volume': '', 'command': None, 'metadata_out': False,
@ -273,6 +271,12 @@ def backup_arguments():
'If that file is not writable, freezer tries to log'
'to ~/.freezer/freezer.log',
dest='log_file', default=None)
arg_parser.add_argument(
'--log-level', action='store', dest="log_level",
default=DEFAULT_PARAMS['log_level'],
choices=['all', 'debug', 'info', 'warn', 'error', 'critical'],
help='Set logging level. Can be all, debug, info, warn,'
'error, critical. Default value - info')
arg_parser.add_argument(
'--exclude', action='store', help="Exclude files,\
given as a PATTERN.Ex: --exclude '*.log' will exclude any file \

View File

@ -17,20 +17,15 @@ Freezer Backup modes related functions
"""
import logging
import os
from os.path import expanduser
import time
from freezer import utils
from freezer import lvm
from freezer.vss import vss_create_shadow_copy
from freezer.vss import vss_delete_shadow_copy
from freezer.winutils import start_sql_server
from freezer.winutils import stop_sql_server
from freezer.winutils import use_shadow
from freezer.winutils import is_windows
from freezer import config
from freezer import lvm
from freezer import utils
from freezer import vss
from freezer import winutils
home = expanduser("~")
home = os.path.expanduser("~")
def backup_mode_sql_server(backup_opt_dict, storage):
@ -47,13 +42,13 @@ def backup_mode_sql_server(backup_opt_dict, storage):
# Dirty hack - please remove any modification of backup_opt_dict
backup_opt_dict.sql_server_instance = sql_server_instance
try:
stop_sql_server(sql_server_instance)
winutils.stop_sql_server(sql_server_instance)
backup(backup_opt_dict, storage, backup_opt_dict.engine)
finally:
if not backup_opt_dict.vssadmin:
# if vssadmin is false, wait until the backup is complete
# to start sql server again
start_sql_server(sql_server_instance)
winutils.start_sql_server(sql_server_instance)
def backup_mode_mysql(backup_opt_dict, storage):
@ -217,22 +212,22 @@ def snapshot_create(backup_opt_dict):
:return: boolean value, True if snapshot has been taken, false otherwise
"""
if is_windows():
if winutils.is_windows():
# vssadmin is to be deprecated in favor of the --snapshot flag
if backup_opt_dict.snapshot:
backup_opt_dict.vssadmin = True
if backup_opt_dict.vssadmin:
# Create a shadow copy.
backup_opt_dict.shadow_path, backup_opt_dict.shadow = \
vss_create_shadow_copy(backup_opt_dict.windows_volume)
vss.vss_create_shadow_copy(backup_opt_dict.windows_volume)
backup_opt_dict.path_to_backup = use_shadow(
backup_opt_dict.path_to_backup = winutils.use_shadow(
backup_opt_dict.path_to_backup,
backup_opt_dict.windows_volume)
# execute this after the snapshot creation
if backup_opt_dict.mode == 'sqlserver':
start_sql_server(backup_opt_dict.sql_server_instance)
winutils.start_sql_server(backup_opt_dict.sql_server_instance)
return True
return False
@ -243,9 +238,9 @@ def snapshot_create(backup_opt_dict):
def snapshot_remove(backup_opt_dict, shadow, windows_volume):
if is_windows():
if winutils.is_windows():
# Delete the shadow copy after the backup
vss_delete_shadow_copy(shadow, windows_volume)
vss.vss_delete_shadow_copy(shadow, windows_volume)
else:
# Unmount and remove lvm snapshot volume
lvm.lvm_snap_remove(backup_opt_dict)

View File

@ -18,7 +18,6 @@ Freezer general utils functions
import logging
import multiprocessing
import time
from freezer.streaming import RichQueue, QueuedThread
from freezer import streaming
from freezer import utils
@ -72,14 +71,13 @@ class BackupEngine(object):
:return: stream
"""
manifest = backup.storage.download_meta_file(backup)
input_queue = RichQueue(queue_size)
read_stream = QueuedThread(self.backup_stream,
input_queue,
kwargs={"backup_path": backup_path,
"manifest_path": manifest})
write_stream = QueuedThread(backup.storage.write_backup,
input_queue,
kwargs={"backup": backup})
input_queue = streaming.RichQueue(queue_size)
read_stream = streaming.QueuedThread(
self.backup_stream, input_queue,
kwargs={"backup_path": backup_path, "manifest_path": manifest})
write_stream = streaming.QueuedThread(
backup.storage.write_backup, input_queue,
kwargs={"backup": backup})
read_stream.daemon = True
write_stream.daemon = True
read_stream.start()

View File

@ -16,12 +16,12 @@ limitations under the License.
Freezer LVM related functions
"""
from freezer import utils
import re
import os
import subprocess
import logging
import os
import re
import subprocess
from freezer import utils
def lvm_snap_remove(backup_opt_dict):

View File

@ -15,27 +15,25 @@ limitations under the License.
Freezer main execution function
"""
import json
import logging
import os
import subprocess
import logging
import sys
import json
from freezer.arguments import backup_arguments
from freezer.bandwidth import monkeypatch_socket_bandwidth
from freezer import arguments
from freezer import bandwidth
from freezer import config
from freezer import job
from freezer.osclients import ClientManager
from freezer.storage import swift
from freezer.storage import local
from freezer.storage import ssh
from freezer.storage import multiple
from freezer import utils
from freezer.engine.tar import tar_engine
from freezer import winutils
# Initialize backup options
from freezer import job
from freezer import osclients
from freezer.storage import local
from freezer.storage import multiple
from freezer.storage import ssh
from freezer.storage import swift
from freezer import utils
from freezer import validator
from freezer import winutils
def freezer_main(backup_args):
@ -49,23 +47,33 @@ def freezer_main(backup_args):
if backup_args.dry_run:
dry_run_message = '[DRY_RUN] '
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
def configure_logging(log_file, str_level):
levels = {
'all': logging.NOTSET,
'debug': logging.DEBUG,
'warn': logging.WARN,
'info': logging.INFO,
'error': logging.ERROR,
'critical': logging.CRITICAL
}
expanded_file_name = os.path.expanduser(log_file)
expanded_dir_name = os.path.dirname(expanded_file_name)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
level=levels[str_level],
format=('%(asctime)s %(name)s %(levelname)s {0}%(message)s'.
format(dry_run_message)))
return expanded_file_name
if backup_args.log_file:
return configure_logging(backup_args.log_file)
return configure_logging(backup_args.log_file,
backup_args.log_level)
for file_name in ['/var/log/freezer.log', '~/.freezer/freezer.log']:
try:
return configure_logging(file_name)
return configure_logging(file_name, backup_args.log_level)
except IOError:
pass
@ -100,7 +108,7 @@ def freezer_main(backup_args):
if backup_args.max_priority:
set_max_process_priority()
monkeypatch_socket_bandwidth(backup_args)
bandwidth.monkeypatch_socket_bandwidth(backup_args)
backup_args.__dict__['hostname_backup_name'] = "{0}_{1}".format(
backup_args.hostname, backup_args.backup_name)
@ -142,7 +150,7 @@ def freezer_main(backup_args):
stderr=subprocess.PIPE,
env=os.environ.copy())
while process.poll() is None:
print process.stdout.readline().rstrip()
print(process.stdout.readline().rstrip())
output, error = process.communicate()
if process.returncode:
@ -191,7 +199,7 @@ def storage_from_dict(backup_args, work_dir, max_segment_size,
options = utils.OpenstackOptions.create_from_env()
identity_api_version = (os_identity_api_version or
options.identity_api_version)
client_manager = ClientManager(
client_manager = osclients.ClientManager(
options=options,
insecure=backup_args.get('insecure') or False,
swift_auth_version=identity_api_version,
@ -215,10 +223,10 @@ def storage_from_dict(backup_args, work_dir, max_segment_size,
def main():
"""Freezerc binary main execution"""
(backup_args, opt_args) = backup_arguments()
(backup_args, opt_args) = arguments.backup_arguments()
try:
if backup_args.version:
print "freezer version {0}".format(backup_args.__version__)
print("freezer version {0}".format(backup_args.__version__))
sys.exit(1)
if len(sys.argv) < 2:

View File

@ -17,6 +17,7 @@ Freezer restore modes related functions
"""
import logging
from freezer import utils

View File

@ -15,12 +15,13 @@ limitations under the License.
"""
import argparse
import os
from oslo_config import cfg
from oslo_log import log
from freezer import __version__ as FREEZER_VERSION
import sys
from freezer import __version__ as FREEZER_VERSION
CONF = cfg.CONF
_LOG = log.getLogger(__name__)

View File

@ -24,7 +24,8 @@ from tempfile import gettempdir
from time import sleep
from freezer.lib.pep3143daemon import DaemonContext, PidFile
from freezer.lib.pep3143daemon import DaemonContext
from freezer.lib.pep3143daemon import PidFile
from freezer.utils import create_dir

View File

@ -15,28 +15,28 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import os
import sys
import threading
from distutils import spawn
from apscheduler.schedulers.blocking import BlockingScheduler
from freezer.apiclient import client
import arguments
import shell
import utils
from freezer import winutils
if winutils.is_windows():
from win_daemon import Daemon, NoDaemon
else:
from daemon import Daemon, NoDaemon
from scheduler_job import Job
from distutils import spawn
from oslo_config import cfg
from oslo_log import log
from freezer.apiclient import client
from freezer.scheduler import arguments
from freezer.scheduler import scheduler_job
from freezer.scheduler import shell
from freezer.scheduler import utils
from freezer import winutils
if winutils.is_windows():
from win_daemon import Daemon
from win_daemon import NoDaemon
else:
from daemon import Daemon
from daemon import NoDaemon
CONF = cfg.CONF
LOG = log.getLogger(__name__)
@ -122,7 +122,7 @@ class FreezerScheduler(object):
return self.scheduler.get_job(job_id) is not None
def create_job(self, job_doc):
job = Job.create(self, self.freezerc_executable, job_doc)
job = scheduler_job.Job.create(self, self.freezerc_executable, job_doc)
if job:
self.jobs[job.id] = job
LOG.info("Created job {0}".format(job.id))
@ -159,7 +159,7 @@ class FreezerScheduler(object):
def stop(self):
try:
self.scheduler.shutdown(wait=False)
except:
except Exception:
pass
def reload(self):
@ -200,7 +200,7 @@ def main():
apiclient.client_id = CONF.client_id
except Exception as e:
LOG.error(e)
print e
print(e)
sys.exit(1)
else:
if winutils.is_windows():

View File

@ -434,7 +434,7 @@ class Job(object):
try:
# already executing job are not present in the apscheduler list
self.scheduler.remove_job(job_id=self.id)
except:
except Exception:
pass
self.event = Job.NO_EVENT
self.job_doc_status = Job.STOP_STATUS

View File

@ -19,16 +19,17 @@ limitations under the License.
import json
import os
import utils
from freezer.utils import create_dir
from prettytable import PrettyTable
try:
from betterprint import pprint
except:
except Exception:
def pprint(doc):
print(json.dumps(doc, indent=4))
from freezer.utils import create_dir
def do_session_remove_job(client, args):
"""
@ -92,7 +93,7 @@ def do_session_create(client, args):
raise Exception("Parameter --file required")
session_doc = utils.load_doc_from_json_file(args.fname)
session_id = client.sessions.create(session_doc)
print "Created session {0}".format(session_id)
print ("Created session {0}".format(session_id))
def do_session_get(client, args):
@ -178,7 +179,7 @@ def do_job_download(client, args):
format(args.jobs_dir, doc['job_id']))
try:
utils.save_doc_to_json_file(doc, fname)
except:
except Exception:
print("Unable to write to file {0}".format(fname))

View File

@ -18,9 +18,10 @@ limitations under the License.
import json
import os
import socket
import freezer.apiclient.exceptions
import uuid
import freezer.apiclient.exceptions
CONFIG_FILE_EXT = '.conf'
@ -34,7 +35,7 @@ def do_register(client, args=None):
client.registration.create(client_info)
except freezer.apiclient.exceptions.ApiClientException as e:
if e.status_code == 409:
print "Client already registered"
print("Client already registered")
return 73 # os.EX_CANTCREAT
return 0 # os.EX_OK
@ -60,7 +61,7 @@ def load_doc_from_json_file(fname, debug=False):
except Exception as e:
raise Exception("Unable to load conf file. {0}".format(e))
if debug:
print "File {0} loaded: ".format(fname)
print("File {0} loaded: ".format(fname))
return doc
@ -68,7 +69,7 @@ def save_doc_to_json_file(doc, fname, debug=False):
with open(fname, 'w') as fd:
json.dump(doc, fd, indent=4)
if debug:
print 'Saved doc to file: {0}'.format(fname)
print('Saved doc to file: {0}'.format(fname))
def get_jobs_from_disk(path):

View File

@ -14,15 +14,11 @@
import logging
import os
import signal
import subprocess
import win32serviceutil
from freezer.utils import shield
from freezer.utils import create_subprocess
from freezer.utils import create_dir
from freezer.winutils import save_environment
from freezer import utils
from freezer import winutils
def setup_logging(log_file):
@ -34,7 +30,7 @@ def setup_logging(log_file):
def configure_logging(file_name):
expanded_file_name = os.path.expanduser(file_name)
expanded_dir_name = os.path.dirname(expanded_file_name)
create_dir(expanded_dir_name, do_log=False)
utils.create_dir(expanded_dir_name, do_log=False)
logging.basicConfig(
filename=expanded_file_name,
level=logging.INFO,
@ -75,13 +71,13 @@ class Daemon(object):
self.job_path = job_path or r'C:\.freezer\scheduler\conf.d'
self.insecure = insecure
@shield
@utils.shield
def start(self, log_file=None):
"""Initialize freezer-scheduler instance inside a windows service
"""
setup_logging(log_file)
create_dir(self.home)
utils.create_dir(self.home)
if self.insecure:
os.environ['SERVICE_INSECURE'] = 'True'
@ -90,18 +86,18 @@ class Daemon(object):
os.environ['SERVICE_JOB_PATH'] = self.job_path
os.environ['SERVICE_INTERVAL'] = str(self.interval)
save_environment(self.home)
winutils.save_environment(self.home)
print('Freezer Service is starting')
win32serviceutil.StartService(self.service_name)
@shield
@utils.shield
def reload(self):
"""Reload the windows service
"""
win32serviceutil.RestartService(self.service_name)
@shield
@utils.shield
def stop(self):
"""Stop the windows service by using sc queryex command, if we use
win32serviceutil.StoptService(self.service_name) it never gets stopped
@ -109,17 +105,17 @@ class Daemon(object):
prevents any new signal to reach the service.
"""
query = 'sc queryex {0}'.format(self.service_name)
out = create_subprocess(query)[0]
out = utils.create_subprocess(query)[0]
pid = None
for line in out.split('\n'):
if 'PID' in line:
pid = line.split(':')[1].strip()
command = 'taskkill /f /pid {0}'.format(pid)
create_subprocess(command)
utils.create_subprocess(command)
print('Freezer Service has stopped')
@shield
@utils.shield
def status(self):
"""Return running status of Freezer Service
by querying win32serviceutil.QueryServiceStatus()
@ -142,7 +138,7 @@ class NoDaemon(object):
# this is only need it in order to have the same interface as in linux
self.daemonizable = daemonizable
@shield
@utils.shield
def stop(self):
self.daemonizable.stop()
@ -156,6 +152,6 @@ class NoDaemon(object):
"""
pass
@shield
@utils.shield
def start(self, log_file=None):
self.daemonizable.start()

View File

@ -13,11 +13,11 @@
# limitations under the License.
import os
import sys
import servicemanager
import sys
import win32event
import win32service
import win32serviceutil
import win32event
from freezer.winutils import set_environment

View File

@ -15,9 +15,9 @@ limitations under the License.
"""
import io
import os
import shutil
import io
from freezer.storage import fslike
from freezer import utils

View File

@ -16,8 +16,8 @@ limitations under the License.
"""
import json
import time
import logging
import time
from freezer.storage import base
@ -134,16 +134,16 @@ class SwiftStorage(base.Storage):
ordered_container = {}
containers = self.swift().get_account()[1]
for container in containers:
print container
print(container)
ordered_container['container_name'] = container['name']
size = '{0}'.format((int(container['bytes']) / 1024) / 1024)
if size == '0':
size = '1'
ordered_container['size'] = '{0}MB'.format(size)
ordered_container['objects_count'] = container['count']
print json.dumps(
print(json.dumps(
ordered_container, indent=4,
separators=(',', ': '), sort_keys=True)
separators=(',', ': '), sort_keys=True))
def meta_file_abs_path(self, backup):
return backup.tar()

View File

@ -15,8 +15,8 @@ limitations under the License.
Freezer general utils functions
"""
import threading
import Queue
import threading
class Wait(Exception):

View File

@ -15,21 +15,19 @@ limitations under the License.
Freezer general utils functions
"""
import datetime
import errno
import logging
import os
import time
import datetime
import re
import subprocess
import errno
import sys
import time
from ConfigParser import ConfigParser
from distutils import spawn as distspawn
import sys
from functools import wraps
from collections import Mapping, Sequence
class OpenstackOptions:
"""
@ -142,7 +140,7 @@ class DateTime(object):
fmt = '%Y-%m-%dT%H:%M:%S'
try:
self.date_time = datetime.datetime.strptime(value, fmt)
except:
except Exception:
raise Exception('bad datetime format: "{0}'.format(value))
@property

View File

@ -12,12 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from freezer.winutils import DisableFileSystemRedirection
from freezer.utils import create_subprocess
import logging
import os
from freezer import utils
from freezer import winutils
def vss_create_shadow_copy(windows_volume):
"""
@ -61,13 +61,12 @@ def vss_create_shadow_copy(windows_volume):
vss_delete_symlink(windows_volume)
with DisableFileSystemRedirection():
with winutils.DisableFileSystemRedirection():
path = os.path.dirname(os.path.abspath(__file__))
script = '{0}\\scripts\\vss.ps1'.format(path)
(out, err) = create_subprocess(['powershell.exe',
'-executionpolicy', 'unrestricted',
'-command', script,
'-volume', windows_volume])
(out, err) = utils.create_subprocess(
['powershell.exe', '-executionpolicy', 'unrestricted',
'-command', script, '-volume', windows_volume])
if err != '':
raise Exception('[*] Error creating a new shadow copy on {0}'
', error {1}' .format(windows_volume, err))
@ -92,10 +91,10 @@ def vss_delete_shadow_copy(shadow_id, windows_volume):
:return: bool
"""
with DisableFileSystemRedirection():
with winutils.DisableFileSystemRedirection():
cmd = ['vssadmin', 'delete', 'shadows',
'/shadow={0}'.format(shadow_id), '/quiet']
(out, err) = create_subprocess(cmd)
(out, err) = utils.create_subprocess(cmd)
if err != '':
raise Exception('[*] Error deleting shadow copy with id {0}'
', error {1}' .format(shadow_id, err))

View File

@ -12,12 +12,11 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import ctypes
import logging
import sys
import os
import json
import logging
import os
import sys
from freezer.utils import create_subprocess