Add storage code

This code will be the basis of writing out the data to the backing
storage with the initial driver being on disk storage.

Additionally move compression/encryption code to storage rather than
backup module

./backup.py --backup /dev/loop0 --manifest mani --location /path/to/store/backup/

Change-Id: I22d291ea3ac03b4d5ddb447915c972db4cdc195d

ekko/storage/_drivers/local.py

@@ -0,0 +1,43 @@
+# Copyright 2016 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from distutils.dir_util import mkpath
+import os
+import uuid
+
+from ekko.storage import drivers
+
+
+class LocalStorage(drivers.BaseStorage):
+
+    def put_data(self, data_segment):
+        data = data_segment[0]
+        segment = data_segment[1]
+        file_path = os.path.join(
+            self.storage_location,
+            str(uuid.UUID(bytes=segment.backupset_id)),
+            str(segment.incremental)
+        )
+
+        mkpath(file_path)
+
+        file_output = os.path.join(
+            file_path,
+            str(segment.segment)
+        )
+
+        with open(file_output, 'wb') as f:
+            f.write(data)
+
+        return segment

ekko/storage/drivers.py

@@ -0,0 +1,39 @@
+# Copyright 2016 Sam Yaple
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import abc
+
+import six
+
+
+@six.add_metaclass(abc.ABCMeta)
+class BaseStorage(object):
+    """Base class for storage drivers
+
+    :params storage_location: Location to store data
+    """
+
+    def __init__(self, storage_location):
+        self.storage_location = storage_location
+
+    @abc.abstractmethod
+    def put_data(self, data_segment):
+        """Write data to backing location
+
+        :params data_segment: A tuple with a raw chunk of data to write out and
+            An object of type manifest.structure.Segment with the metadata
+            to pair with the raw chunk of data
+        :returns: A generator with the segment objects
+        """
+        raise NotImplementedError()

setup.cfg

@@ -26,6 +26,8 @@ ekko.backup.backend =
 ekko.manifest.drivers =
     osdk = ekko.manifest._drivers.osdk:OSDKManifest
     sqlite = ekko.manifest._drivers.sqlite:SQLiteManifest
+ekko.storage.drivers =
+    local = ekko.storage._drivers.local:LocalStorage
 
 [files]
 packages =

tools/backup.py

@@ -38,6 +38,9 @@ def parse_args():
                         help='change block tracking info')
     parser.add_argument('--backend', required=False, default='raw',
                         choices=['raw'], help='backend driver')
+    parser.add_argument('--storage', required=False, default='local',
+                        choices=['local'], help='storage driver')
+    parser.add_argument('--location', required=True, help='storage path')
     parser.add_argument('--driver', required=False, default='sqlite',
                         choices=['osdk', 'sqlite'], help='manifest driver')
     return parser.parse_args()
@@ -65,7 +68,7 @@ def read_segments(segments, metadata, backend):
     for start, data in backend.get_data(reads):
         if data == zero_blob:
             continue
-        yield manifest_structure.Segment(
+        yield data, manifest_structure.Segment(
             metadata.backupset_id,
             metadata.incremental,
             start / size,
@@ -92,14 +95,21 @@ def main():
         invoke_args=[args.manifest]
     ).driver
 
-    backend = driver.DriverManager(
+    disk_backend = driver.DriverManager(
         namespace='ekko.backup.backend',
         name=args.backend,
         invoke_on_load=True,
         invoke_args=[args.backup]
     ).driver
 
-    size_of_disk = backend.get_size()
+    storage = driver.DriverManager(
+        namespace='ekko.storage.drivers',
+        name=args.storage,
+        invoke_on_load=True,
+        invoke_args=[args.location]
+    ).driver
+
+    size_of_disk = disk_backend.get_size()
     incremental = 0
     metadata = manifest_structure.Metadata(incremental, size_of_disk)
 
@@ -108,7 +118,11 @@ def main():
 
     segments_list = list(range(0, int(math.ceil(
         float(size_of_disk)/metadata.segment_size))))
-    segments = read_segments(segments_list, metadata, backend)
+    data_segments = read_segments(segments_list, metadata, disk_backend)
+
+    segments = [
+        storage.put_data(data_segment) for data_segment in data_segments
+    ]
 
     manifest.put_segments(segments, metadata)