From 983766902224ed9a1c7509b854b961e2ef9d7cde Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Fri, 15 Mar 2013 14:46:38 +0100 Subject: [PATCH] Add a basic syncer. - Just basic sync accounts, there is plenty of more works to do. --- bin/sync-accounts.py | 33 +++++++++++++++++++++ common/__init__.py | 0 common/utils.py | 53 ++++++++++++++++++++++++++++++++++ etc/config.ini | 5 ++++ sync/__init__.py | 1 + sync/containers.py | 58 +++++++++++++++++++++++++++++++++++++ sync/objects.py | 69 ++++++++++++++++++++++++++++++++++++++++++++ 7 files changed, 219 insertions(+) create mode 100755 bin/sync-accounts.py create mode 100644 common/__init__.py create mode 100644 common/utils.py create mode 100644 etc/config.ini create mode 100644 sync/__init__.py create mode 100644 sync/containers.py create mode 100644 sync/objects.py diff --git a/bin/sync-accounts.py b/bin/sync-accounts.py new file mode 100755 index 0000000..094a1c7 --- /dev/null +++ b/bin/sync-accounts.py @@ -0,0 +1,33 @@ +#!/usr/bin/python +# -*- encoding: utf-8 -*- +import swiftclient + +from sync.containers import sync_container +from utils import get_config, get_auth + +orig_auth_url = get_config('auth', 'keystone_origin') +orig_tenant, orig_user, orig_password = \ + get_config('auth', 'keystone_origin_credentials').split(':') + +dest_auth_url = get_config('auth', 'keystone_dest') +dest_tenant, dest_user, dest_password = \ + get_config('auth', 'keystone_dest_credentials').split(':') + +orig_storage_url, orig_token = \ + get_auth(orig_auth_url, orig_tenant, orig_user, orig_password) +orig_storage_cnx = swiftclient.http_connection(orig_storage_url) + +dest_storage_url, dest_token = \ + get_auth(dest_auth_url, dest_tenant, dest_user, dest_password) +dest_storage_cnx = swiftclient.http_connection(dest_storage_url) + + +orig_account_stats, orig_containers = swiftclient.get_account( + None, orig_token, http_conn=orig_storage_cnx, full_listing=True +) + +for container in orig_containers: + print "Synching %s.." % (container) + sync_container(orig_storage_cnx, orig_storage_url, orig_token, + dest_storage_cnx, dest_storage_url, dest_token, + container['name']) diff --git a/common/__init__.py b/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/common/utils.py b/common/utils.py new file mode 100644 index 0000000..230634a --- /dev/null +++ b/common/utils.py @@ -0,0 +1,53 @@ +# -*- encoding: utf-8 -*- +__author__ = "Chmouel Boudjnah " +import os +import ConfigParser +import swiftclient.client as swclient + + +CONFIG = {} +curdir = os.path.abspath(os.path.dirname(__file__)) +INIFILE = os.path.abspath(os.path.join(curdir, '..', 'etc', "config.ini")) + + +class ConfigurationError(Exception): + pass + + +def parse_ini(inifile=INIFILE): + if not os.path.exists(inifile): + raise ConfigurationError("Error while parsing inifile") + + config = ConfigParser.RawConfigParser() + config.read(inifile) + return config + + +def get_config(section, option, default=None): + """Get section/option from ConfigParser or print default if specified""" + global CONFIG + + if not CONFIG: + CONFIG = parse_ini() + + if not CONFIG.has_section(section): + raise ConfigurationError("Invalid configuration, missing section: %s" % + section) + if CONFIG.has_option(section, option): + return CONFIG.get(section, option) + elif not default is None: + return default + else: + raise ConfigurationError("Invalid configuration, missing " + "section/option: %s/%s" % (section, option)) + + +def get_auth(auth_url, tenant, user, password): + return swclient.Connection( + auth_url, + '%s:%s' % (tenant, user), + password, + auth_version=2).get_auth() + +if __name__ == '__main__': + get_config("foo", "bar") diff --git a/etc/config.ini b/etc/config.ini new file mode 100644 index 0000000..025f331 --- /dev/null +++ b/etc/config.ini @@ -0,0 +1,5 @@ +[auth] +keystone_origin = http://vm:5000/v2.0 +keystone_origin_credentials = demo:demo:ADMIN +keystone_dest = http://vm2:5000/v2.0 +keystone_dest_credentials = demo:demo:ADMIN diff --git a/sync/__init__.py b/sync/__init__.py new file mode 100644 index 0000000..dae354a --- /dev/null +++ b/sync/__init__.py @@ -0,0 +1 @@ +# -*- encoding: utf-8 -*- diff --git a/sync/containers.py b/sync/containers.py new file mode 100644 index 0000000..3d42444 --- /dev/null +++ b/sync/containers.py @@ -0,0 +1,58 @@ +import sys + +import swiftclient +import eventlet + +from sync.objects import sync_object + + +def sync_container(orig_storage_cnx, orig_storage_url, + orig_token, dest_storage_cnx, + dest_storage_url, dest_token, + container_name): + orig_container_stats, orig_objects = swiftclient.get_container( + None, orig_token, container_name, http_conn=orig_storage_cnx, + ) + + try: + swiftclient.head_container( + "", dest_token, container_name, http_conn=dest_storage_cnx + ) + except(swiftclient.client.ClientException): + container_headers = orig_container_stats.copy() + for h in ('x-container-object-count', 'x-trans-id', + 'x-container-bytes-used'): + del container_headers[h] + p = dest_storage_cnx[0] + url = "%s://%s%s" % (p.scheme, p.netloc, p.path) + swiftclient.put_container(url, + dest_token, container_name, + headers=container_headers) + + + dest_container_stats, dest_objects = swiftclient.get_container( + None, dest_token, container_name, http_conn=dest_storage_cnx, + ) + + set1 = set((x['hash'], x['name']) for x in orig_objects) + set2 = set((x['hash'], x['name']) for x in dest_objects) + diff = set1 - set2 + if not diff: + return + + pool = eventlet.GreenPool() + cnt = 0 + for obj in diff: + pool.spawn_n(sync_object, + orig_storage_url, + orig_token, + dest_storage_url, + dest_token, container_name, + obj) + if cnt == 20: + pool.waitall() + cnt = 0 + else: + cnt += 1 + + pool.waitall() diff --git a/sync/objects.py b/sync/objects.py new file mode 100644 index 0000000..953b989 --- /dev/null +++ b/sync/objects.py @@ -0,0 +1,69 @@ +# -*- encoding: utf-8 -*- +from swiftclient import client as swiftclient +from swift.common.bufferedhttp import http_connect_raw +from swift.common.http import is_success +from swift.container.sync import _Iter2FileLikeObject +from eventlet import Timeout +import urllib2 + + +def get_object(storage_url, token, + container_name, + object_name, + response_timeout=15, + conn_timeout=5, + resp_chunk_size=65536): + headers = {'x-auth-token': token} + x = urllib2.urlparse.urlparse(storage_url) + + path = x.path + '/' + container_name + '/' + object_name + with Timeout(conn_timeout): + conn = http_connect_raw( + x.hostname, + x.port, + 'GET', + path, + headers=headers, + ssl=False) + + with Timeout(response_timeout): + resp = conn.getresponse() + + if not is_success(resp.status): + resp.read() + raise swiftclient.ClientException( + 'status %s %s' % (resp.status, resp.reason)) + + if resp_chunk_size: + def _object_body(): + buf = resp.read(resp_chunk_size) + while buf: + yield buf + buf = resp.read(resp_chunk_size) + object_body = _object_body() + else: + object_body = resp.read() + + resp_headers = {} + for header, value in resp.getheaders(): + resp_headers[header.lower()] = value + + return (resp_headers, object_body) + + +def sync_object(orig_storage_url, orig_token, dest_storage_url, + dest_token, container_name, object_name_etag): + orig_headers, orig_body = get_object(orig_storage_url, + orig_token, + container_name, + object_name_etag[1], + ) + post_headers = orig_headers + post_headers['x-auth-token'] = dest_token + sync_to = dest_storage_url + "/" + container_name + swiftclient.put_object(sync_to, name=object_name_etag[1], + headers=post_headers, + contents=_Iter2FileLikeObject(orig_body)) + +if __name__ == '__main__': + pass