diff --git a/INSTALL.rst b/INSTALL.rst index 5cd4a4ae..00c8dbe4 100644 --- a/INSTALL.rst +++ b/INSTALL.rst @@ -1,6 +1,8 @@ Install instructions ==================== +Please check README.rst for further installation instructions. + Install from sources:: ---------------------- diff --git a/README.rst b/README.rst index 62e1e8ef..ed445065 100644 --- a/README.rst +++ b/README.rst @@ -76,11 +76,11 @@ Also the following ENV var are needed you can put them in ~/.bashrc:: $ source ~/.bashrc -Let's say you have a container called foobar-container, by executing +Let's say you have a container called freezer_foobar-container, by executing "swift list" you should see something like:: $ swift list - foobar-container-2 + freezer_foobar-container-2 $ These are just use case example using Swift in the HP Cloud. @@ -92,18 +92,22 @@ data inconsistencies and corruption.* Usage Example ============= +Freezer will automatically add the prefix "freezer_" to the container name, +where it is provided by the user and doesn't already start with this prefix. +If no container name is provided, the default is freezer_backups. + Backup ------ The most simple backup execution is a direct file system backup:: $ sudo freezerc --file-to-backup /data/dir/to/backup - --container new-data-backup --backup-name my-backup-name + --container freezer_new-data-backup --backup-name my-backup-name By default --mode fs is set. The command would generate a compressed tar gzip file of the directory /data/dir/to/backup. The generated file will be segmented in stream and uploaded in the swift container called -new-data-backup, with backup name my-backup-name +freezer_new-data-backup, with backup name my-backup-name Now check if your backup is executing correctly looking at /var/log/freezer.log @@ -129,19 +133,19 @@ is "mongo":: $ sudo freezerc --lvm-srcvol /dev/mongo/mongolv --lvm-dirmount /var/lib/snapshot-backup --lvm-volgroup mongo --file-to-backup /var/lib/snapshot-backup/mongod_ops2 - --container mongodb-backup-prod --exclude "*.lock" --mode mongo --backup-name mongod-ops2 + --container freezer_mongodb-backup-prod --exclude "*.lock" --mode mongo --backup-name mongod-ops2 Now freezerc create a lvm snapshot of the volume /dev/mongo/mongolv. If no options are provided, default snapshot name is freezer\_backup\_snap. The snap vol will be mounted automatically on /var/lib/snapshot-backup and the backup meta and segments will be upload in the container -mongodb-backup-prod with the namemongod-ops2. +mongodb-backup-prod with the name mongod-ops2. Execute a file system backup using lvm snapshot:: $ sudo freezerc --lvm-srcvol /dev/jenkins/jenkins-home --lvm-dirmount /var/snapshot-backup --lvm-volgroup jenkins - --file-to-backup /var/snapshot-backup --container jenkins-backup-prod + --file-to-backup /var/snapshot-backup --container freezer_jenkins-backup-prod --exclude "\*.lock" --mode fs --backup-name jenkins-ops2 MySQL backup require a basic configuration file. The following is an @@ -161,7 +165,7 @@ Execute a MySQL backup using lvm snapshot:: --lvm-dirmount /var/snapshot-backup --lvm-volgroup mysqlvg --file-to-backup /var/snapshot-backup --mysql-conf /root/.freezer/freezer-mysql.conf--container - mysql-backup-prod --mode mysql --backup-name mysql-ops002 + freezer_mysql-backup-prod --mode mysql --backup-name mysql-ops002 All the freezerc activities are logged into /var/log/freezer.log. @@ -178,7 +182,7 @@ File System Restore: Execute a file system restore of the backup name adminui.git:: - $ sudo freezerc --action restore --container foobar-container-2 + $ sudo freezerc --action restore --container freezer_foobar-container-2 --backup-name adminui.git --restore-from-host git-HP-DL380-host-001 --restore-abs-path /home/git/repositories/adminui.git/ @@ -193,7 +197,7 @@ Let's stop mysql service first:: Execute Restore:: - $ sudo freezerc --action restore --container foobar-container-2 + $ sudo freezerc --action restore --container freezer_foobar-container-2 --backup-name mysq-prod --restore-from-host db-HP-DL380-host-001 --restore-abs-path /var/lib/mysql --restore-from-date "2014-05-23T23:23:23" @@ -203,9 +207,9 @@ And finally restart mysql:: Execute a MongoDB restore of the backup name mongobigdata:: - $ sudo freezerc --action restore --container foobar-container-2 --backup-name mongobigdata - --restore-from-host db-HP-DL380-host-001 --restore-abs-path - /var/lib/mongo --restore-from-date "2014-05-23T23:23:23" + $ sudo freezerc --action restore --container freezer_foobar-container-2 + --backup-name mongobigdata --restore-from-host db-HP-DL380-host-001 + --restore-abs-path /var/lib/mongo --restore-from-date "2014-05-23T23:23:23" List remote containers:: @@ -214,12 +218,12 @@ List remote containers:: List remote objects in container:: - $ sudo freezerc --action info --container testcontainer -l + $ sudo freezerc --action info --container freezer_testcontainer -l Remove backups older then 1 day:: - $ freezerc --action admin --container freezer-dev-test --remove-older-then 1 --backup-name dev-test-01 + $ freezerc --action admin --container freezer_dev-test --remove-older-then 1 --backup-name dev-test-01 Architecture ============ @@ -241,7 +245,7 @@ Freezer is designed to reduce at the minimum I/O, CPU and Memory Usage. This is achieved by generating a data stream from tar (for archiving) and gzip (for compressing). Freezer segment the stream in a configurable chunk size (with the option --max-seg-size). The default segment size is -128MB, so it can be safely stored in memory, encrypted if the key is +64MB, so it can be safely stored in memory, encrypted if the key is provided, and uploaded to Swift as segment. Multiple segments are sequentially uploaded using the Swift Manifest. @@ -370,3 +374,153 @@ tar\_metadata\_backupname\_hostname\_timestamp\_backuplevel The hostname of the node where the Freezer perform the backup. This meta data is important to identify a backup with a specific node, thus avoid possible confusion and associate backup to the wrong node. + + +Miscellanea +----------- + +Available options:: + + $ freezerc + usage: freezerc [-h] [--action {backup,restore,info,admin}] [-F SRC_FILE] + [-N BACKUP_NAME] [-m MODE] [-C CONTAINER] [-L] [-l] + [-o OBJECT] [-d DST_FILE] [--lvm-auto-snap LVM_AUTO_SNAP] + [--lvm-srcvol LVM_SRCVOL] [--lvm-snapname LVM_SNAPNAME] + [--lvm-snapsize LVM_SNAPSIZE] [--lvm-dirmount LVM_DIRMOUNT] + [--lvm-volgroup LVM_VOLGROUP] [--max-level MAX_BACKUP_LEVEL] + [--always-level ALWAYS_BACKUP_LEVEL] + [--restart-always-level RESTART_ALWAYS_BACKUP] + [-R REMOVE_OLDER_THAN] [--no-incremental] + [--hostname HOSTNAME] [--mysql-conf MYSQL_CONF_FILE] + [--log-file LOG_FILE] [--exclude EXCLUDE] + [--dereference-symlink {none,soft,hard,all}] [-U] + [--encrypt-pass-file ENCRYPT_PASS_FILE] [-M MAX_SEG_SIZE] + [--restore-abs-path RESTORE_ABS_PATH] + [--restore-from-host RESTORE_FROM_HOST] + [--restore-from-date RESTORE_FROM_DATE] [--max-priority] [-V] + + optional arguments: + -h, --help show this help message and exit + --action {backup,restore,info,admin} + Set the action to be taken. backup and restore are + self explanatory, info is used to retrieve info from + the storage media, while maintenance is used to delete + old backups and other admin actions. Default backup. + -F SRC_FILE, --path-to-backup SRC_FILE, --file-to-backup SRC_FILE + The file or directory you want to back up to Swift + -N BACKUP_NAME, --backup-name BACKUP_NAME + The backup name you want to use to identify your + backup on Swift + -m MODE, --mode MODE Set the technology to back from. Options are, fs + (filesystem), mongo (MongoDB), mysql (MySQL). Default + set to fs + -C CONTAINER, --container CONTAINER + The Swift container used to upload files to + -L, --list-containers + List the Swift containers on remote Object Storage + Server + -l, --list-objects List the Swift objects stored in a container on remote + Object Storage Server. + -o OBJECT, --get-object OBJECT + The Object name you want to download on the local file + system. + -d DST_FILE, --dst-file DST_FILE + The file name used to save the object on your local + disk and upload file in swift + --lvm-auto-snap LVM_AUTO_SNAP + Automatically guess the volume group and volume name + for given PATH. + --lvm-srcvol LVM_SRCVOL + Set the lvm volume you want to take a snaphost from. + Default no volume + --lvm-snapname LVM_SNAPNAME + Set the lvm snapshot name to use. If the snapshot name + already exists, the old one will be used a no new one + will be created. Default freezer_backup_snap. + --lvm-snapsize LVM_SNAPSIZE + Set the lvm snapshot size when creating a new + snapshot. Please add G for Gigabytes or M for + Megabytes, i.e. 500M or 8G. Default 5G. + --lvm-dirmount LVM_DIRMOUNT + Set the directory you want to mount the lvm snapshot + to. Default not set + --lvm-volgroup LVM_VOLGROUP + Specify the volume group of your logical volume. This + is important to mount your snapshot volume. Default + not set + --max-level MAX_BACKUP_LEVEL + Set the backup level used with tar to implement + incremental backup. If a level 1 is specified but no + level 0 is already available, a level 0 will be done + and subesequently backs to level 1. Default 0 (No + Incremental) + --always-level ALWAYS_BACKUP_LEVEL + Set backup maximum level used with tar to implement + incremental backup. If a level 3 is specified, the + backup will be executed from level 0 to level 3 and to + that point always a backup level 3 will be executed. + It will not restart from level 0. This option has + precedence over --max-backup-level. Default False + (Disabled) + --restart-always-level RESTART_ALWAYS_BACKUP + Restart the backup from level 0 after n days. Valid + only if --always-level option if set. If --always- + level is used together with --remove-older-then, there + might be the chance where the initial level 0 will be + removed Default False (Disabled) + -R REMOVE_OLDER_THAN, --remove-older-then REMOVE_OLDER_THAN + Checks in the specified container for object older + then the specified days. If i.e. 30 is specified, it + will remove the remote object older than 30 days. + Default False (Disabled) + --no-incremental Disable incremantal feature. By default freezer build + the meta data even for level 0 backup. By setting this + option incremental meta data is not created at all. + Default disabled + --hostname HOSTNAME Set hostname to execute actions. If you are executing + freezer from one host but you want to delete objects + belonging to another host then you can set this option + that hostname and execute appropriate actions. Default + current node hostname. + --mysql-conf MYSQL_CONF_FILE + Set the MySQL configuration file where freezer + retrieve important information as db_name, user, + password, host. Following is an example of config + file: # cat ~/.freezer/backup_mysql_conf host = user = password = + --log-file LOG_FILE Set log file. By default logs to /var/log/freezer.log + --exclude EXCLUDE Exclude files, given as a PATTERN.Ex: --exclude + '*.log' will exclude any file with name ending with + .log. Default no exclude + --dereference-symlink {none,soft,hard,all} + Follow hard and soft links and archive and dump the + files they refer to. Default False. + -U, --upload Upload to Swift the destination file passed to the -d + option. Default upload the data + --encrypt-pass-file ENCRYPT_PASS_FILE + Passing a private key to this option, allow you to + encrypt the files before to be uploaded in Swift. + Default do not encrypt. + -M MAX_SEG_SIZE, --max-segment-size MAX_SEG_SIZE + Set the maximum file chunk size in bytes to upload to + swift Default 67108864 bytes (64MB) + --restore-abs-path RESTORE_ABS_PATH + Set the absolute path where you want your data + restored. Default False. + --restore-from-host RESTORE_FROM_HOST + Set the hostname used to identify the data you want to + restore from. If you want to restore data in the same + host where the backup was executed just type from your + shell: "$ hostname" and the output is the value that + needs to be passed to this option. Mandatory with + Restore Default False. + --restore-from-date RESTORE_FROM_DATE + Set the absolute path where you want your data + restored. Please provide datime in forma "YYYY-MM- + DDThh:mm:ss" i.e. "1979-10-03T23:23:23". Make sure the + "T" is between date and time Default False. + --max-priority Set the cpu process to the highest priority (i.e. -20 + on Linux) and real-time for I/O. The process priority + will be set only if nice and ionice are installed + Default disabled. Use with caution. + -V, --version Print the release version and exit diff --git a/freezer/arguments.py b/freezer/arguments.py index 3f9f07c8..ef794eb9 100644 --- a/freezer/arguments.py +++ b/freezer/arguments.py @@ -43,7 +43,7 @@ def backup_arguments(): " and other admin actions. Default backup."), dest='action', default='backup') arg_parser.add_argument( - '-F', '--file-to-backup', action='store', + '-F', '--path-to-backup', '--file-to-backup', action='store', help="The file or directory you want to back up to Swift", dest='src_file', default=False) arg_parser.add_argument( @@ -58,7 +58,7 @@ def backup_arguments(): arg_parser.add_argument( '-C', '--container', action='store', help="The Swift container used to upload files to", - dest='container', default=False) + dest='container', default='freezer_backups') arg_parser.add_argument( '-L', '--list-containers', action='store_true', help='''List the Swift containers on remote Object Storage Server''', @@ -224,6 +224,18 @@ def backup_arguments(): # Create a new namespace attribute for container_segments backup_args.__dict__['container_segments'] = u'{0}_segments'.format( backup_args.container) + + # The containers used by freezer to executed backups needs to have + # freezer_ prefix in the name. If the user provider container doesn't + # have the prefix, it is automatically added also to the container + # segments name. This is done to quickly identify the containers + # that contain freezer generated backups + if not backup_args.container.startswith('freezer_'): + backup_args.container = 'freezer_{0}'.format( + backup_args.container) + backup_args.container_segments = 'freezer_{0}'.format( + backup_args.container_segments) + # If hostname is not set, hostname of the current node will be used if not backup_args.hostname: backup_args.__dict__['hostname'] = os.uname()[1] @@ -258,6 +270,6 @@ def backup_arguments(): backup_args.__dict__['mysql_db_inst'] = '' # Freezer version - backup_args.__dict__['__version__'] = '1.0.9-2' + backup_args.__dict__['__version__'] = '1.1.0' return backup_args, arg_parser diff --git a/freezer/swift.py b/freezer/swift.py index 65bc02a0..b105020e 100644 --- a/freezer/swift.py +++ b/freezer/swift.py @@ -43,7 +43,7 @@ def create_containers(backup_opt): containers :param backup_opt: - :return: True if both containers are succesfully created + :return: True if both containers are successfully created """ # Create backup container diff --git a/setup.py b/setup.py index c4e10d4f..d40b85eb 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def read(*filenames, **kwargs): setup( name='freezer', - version='1.0.9-2', + version='1.1.0', url='https://github.com/stackforge/freezer', license='Apache Software License', author='Fausto Marzi, Ryszard Chojnacki, Emil Dimitrov',