From 63de1098d173a16673156ff01988aca425b90c60 Mon Sep 17 00:00:00 2001 From: Chmouel Boudjnah Date: Wed, 20 Mar 2013 18:18:33 +0100 Subject: [PATCH] sync account. --- bin/filler.py | 1 - bin/sync-accounts.py | 33 -------------------- bin/syncit.py | 11 +++++++ etc/config-sample.ini | 1 + sync/accounts.py | 71 +++++++++++++++++++++++++++++++++++++++++++ sync/containers.py | 3 -- sync/objects.py | 16 +++++++++- 7 files changed, 98 insertions(+), 38 deletions(-) delete mode 100755 bin/sync-accounts.py create mode 100755 bin/syncit.py create mode 100644 sync/accounts.py diff --git a/bin/filler.py b/bin/filler.py index 34c5f56..80b394e 100755 --- a/bin/filler.py +++ b/bin/filler.py @@ -9,7 +9,6 @@ import sys import eventlet from keystoneclient.v2_0 import client as ksclient -sys.path.append("../") sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) from common.utils import get_config from sync.filler import (load_index, load_containers_index, diff --git a/bin/sync-accounts.py b/bin/sync-accounts.py deleted file mode 100755 index e55c8e5..0000000 --- a/bin/sync-accounts.py +++ /dev/null @@ -1,33 +0,0 @@ -#!/usr/bin/env python -# -*- encoding: utf-8 -*- -import swiftclient - -from sync.containers import sync_container -from utils import get_config, get_auth - -orig_auth_url = get_config('auth', 'keystone_origin') -orig_tenant, orig_user, orig_password = \ - get_config('auth', 'keystone_origin_credentials').split(':') - -dest_auth_url = get_config('auth', 'keystone_dest') -dest_tenant, dest_user, dest_password = \ - get_config('auth', 'keystone_dest_credentials').split(':') - -orig_storage_url, orig_token = \ - get_auth(orig_auth_url, orig_tenant, orig_user, orig_password) -orig_storage_cnx = swiftclient.http_connection(orig_storage_url) - -dest_storage_url, dest_token = \ - get_auth(dest_auth_url, dest_tenant, dest_user, dest_password) -dest_storage_cnx = swiftclient.http_connection(dest_storage_url) - - -orig_account_stats, orig_containers = swiftclient.get_account( - None, orig_token, http_conn=orig_storage_cnx, full_listing=True -) - -for container in orig_containers: - print "Synching %s.." % (container) - sync_container(orig_storage_cnx, orig_storage_url, orig_token, - dest_storage_cnx, dest_storage_url, dest_token, - container['name']) diff --git a/bin/syncit.py b/bin/syncit.py new file mode 100755 index 0000000..9b65ebe --- /dev/null +++ b/bin/syncit.py @@ -0,0 +1,11 @@ +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +import sys +import os +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from sync.accounts import sync_accounts + + +if __name__ == '__main__': + sync_accounts() diff --git a/etc/config-sample.ini b/etc/config-sample.ini index 0882422..f9479b7 100644 --- a/etc/config-sample.ini +++ b/etc/config-sample.ini @@ -3,6 +3,7 @@ keystone_origin = http://vm:5000/v2.0 keystone_origin_admin_credentials = admin:admin:ADMIN keystone_dest = http://vm2:5000/v2.0 + keystone_origin_demo_credentials = demo:demo:ADMIN keystone_dest_credentials = demo:demo:ADMIN diff --git a/sync/accounts.py b/sync/accounts.py new file mode 100644 index 0000000..2d69b81 --- /dev/null +++ b/sync/accounts.py @@ -0,0 +1,71 @@ +# -*- encoding: utf-8 -*- +import os +import sys + +import swiftclient +from keystoneclient.v2_0 import client as ksclient + +sys.path.append(os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) +from common.utils import get_config, get_auth +from sync.containers import sync_container + + +def get_ks_auth_orig(): + orig_auth_url = get_config('auth', 'keystone_origin') + tenant_name, username, password = \ + get_config('auth', 'keystone_origin_admin_credentials').split(':') + + return ksclient.Client(auth_url=orig_auth_url, + username=username, + password=password, + tenant_name=tenant_name) + + +def list_accounts(cnx): + for x in cnx.tenants.list(): + yield x + + +def sync_an_account(orig_storage_url, + orig_token, + dest_storage_url, + dest_token): + + orig_storage_cnx = swiftclient.http_connection(orig_storage_url) + dest_storage_cnx = swiftclient.http_connection(dest_storage_url) + + orig_account_stats, orig_containers = swiftclient.get_account( + None, orig_token, http_conn=orig_storage_cnx, full_listing=True) + + for container in orig_containers: + print "Synching %s.." % (container) + sync_container(orig_storage_cnx, orig_storage_url, orig_token, + dest_storage_cnx, dest_storage_url, dest_token, + container['name']) + + +def sync_accounts(): + orig_auth_url = get_config('auth', 'keystone_origin') + orig_admin_tenant, orig_admin_user, orig_admin_password = ( + get_config('auth', 'keystone_origin_admin_credentials').split(':')) + oa_st_url, orig_admin_token = (get_auth(orig_auth_url, orig_admin_tenant, + orig_admin_user, + orig_admin_password)) + + dest_auth_url = get_config('auth', 'keystone_dest') + # we assume orig and dest passwd are the same obv synchronized. + dst_st_url, dest_admin_token = (get_auth(dest_auth_url, orig_admin_tenant, + orig_admin_user, + orig_admin_password)) + + bare_oa_st_url = oa_st_url[:oa_st_url.find('AUTH_')] + "AUTH_" + bare_dst_st_url = dst_st_url[:dst_st_url.find('AUTH_')] + "AUTH_" + + for x in list_accounts(get_ks_auth_orig()): + user_orig_st_url = bare_oa_st_url + x.id + user_dst_st_url = bare_dst_st_url + x.id + + sync_an_account(user_orig_st_url, + orig_admin_token, + user_dst_st_url, + dest_admin_token) diff --git a/sync/containers.py b/sync/containers.py index 3d42444..969e486 100644 --- a/sync/containers.py +++ b/sync/containers.py @@ -1,5 +1,3 @@ -import sys - import swiftclient import eventlet @@ -29,7 +27,6 @@ def sync_container(orig_storage_cnx, orig_storage_url, dest_token, container_name, headers=container_headers) - dest_container_stats, dest_objects = swiftclient.get_container( None, dest_token, container_name, http_conn=dest_storage_cnx, ) diff --git a/sync/objects.py b/sync/objects.py index 953b989..58c4a1e 100644 --- a/sync/objects.py +++ b/sync/objects.py @@ -5,6 +5,16 @@ from swift.common.http import is_success from swift.container.sync import _Iter2FileLikeObject from eventlet import Timeout import urllib2 +from urllib import quote as urllib_quote + + +def quote(value, safe='/'): + """ + Patched version of urllib.quote that encodes utf-8 strings before quoting + """ + if isinstance(value, unicode): + value = value.encode('utf-8') + return urllib_quote(value, safe) def get_object(storage_url, token, @@ -17,6 +27,7 @@ def get_object(storage_url, token, x = urllib2.urlparse.urlparse(storage_url) path = x.path + '/' + container_name + '/' + object_name + path = quote(path) with Timeout(conn_timeout): conn = http_connect_raw( x.hostname, @@ -58,10 +69,13 @@ def sync_object(orig_storage_url, orig_token, dest_storage_url, container_name, object_name_etag[1], ) + container_name = quote(container_name) + object_name = quote(object_name_etag[1]) + post_headers = orig_headers post_headers['x-auth-token'] = dest_token sync_to = dest_storage_url + "/" + container_name - swiftclient.put_object(sync_to, name=object_name_etag[1], + swiftclient.put_object(sync_to, name=object_name, headers=post_headers, contents=_Iter2FileLikeObject(orig_body))