From 79fe8a9c7e5d8db964ead5f29d0eda6dd3976373 Mon Sep 17 00:00:00 2001 From: Yo Sub Kwon Date: Tue, 19 May 2015 16:09:55 -0700 Subject: [PATCH 01/12] Fix typo 'unknown' --- src/idp_test/package/authn_request.py | 6 +++--- src/saml2/entity.py | 2 +- src/saml2/pack.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/idp_test/package/authn_request.py b/src/idp_test/package/authn_request.py index e4b2691..ccebf59 100644 --- a/src/idp_test/package/authn_request.py +++ b/src/idp_test/package/authn_request.py @@ -30,7 +30,7 @@ class AuthnRequest_UnknownExtension(AuthnRequest): return message OPERATIONS = { - 'authn_unkown-issuer': { + 'authn_unknown-issuer': { "name": 'AuthnRequest with unknown issuer', "descr": 'AuthnRequest with unknown issuer', "sequence": [AuthnRequest_UnknownIssuer], @@ -38,7 +38,7 @@ OPERATIONS = { "tests": {"pre": [CheckSaml2IntMetaData], "post": [CheckSaml2IntAttributes]} }, - 'authn_unkown-extension': { + 'authn_unknown-extension': { "name": 'AuthnRequest with unknown extension', "descr": 'AuthnRequest with unknown extension', "sequence": [AuthnRequest_UnknownExtension], @@ -46,4 +46,4 @@ OPERATIONS = { "tests": {"pre": [CheckSaml2IntMetaData], "post": [CheckSaml2IntAttributes]} }, -} \ No newline at end of file +} diff --git a/src/saml2/entity.py b/src/saml2/entity.py index cfb04f0..6eed861 100644 --- a/src/saml2/entity.py +++ b/src/saml2/entity.py @@ -281,7 +281,7 @@ class Entity(HTTPBase): #logger.error("Bindings: %s" % bindings) #logger.error("Entities: %s" % self.metadata) - raise SAMLError("Unkown entity or unsupported bindings") + raise SAMLError("Unknown entity or unsupported bindings") def message_args(self, message_id=0): if not message_id: diff --git a/src/saml2/pack.py b/src/saml2/pack.py index 8057ad0..f045c7a 100644 --- a/src/saml2/pack.py +++ b/src/saml2/pack.py @@ -247,7 +247,7 @@ def packager(identifier): try: return PACKING[identifier] except KeyError: - raise Exception("Unkown binding type: %s" % identifier) + raise Exception("Unknown binding type: %s" % identifier) def factory(binding, message, location, relay_state="", typ="SAMLRequest"): From 317a975c561af7132176db05d3ba684e4ece58d6 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Sun, 17 May 2015 01:05:45 -0700 Subject: [PATCH 02/12] Update rndstr to use string.ascii_letters for py3k Python3 has eliminated string.letters, and also makes dealing with arrays of bytes somewhat tricky. Luckily string.ascii_letters returns str. --- src/saml2/s_utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/saml2/s_utils.py b/src/saml2/s_utils.py index 3aae0be..9d2391e 100644 --- a/src/saml2/s_utils.py +++ b/src/saml2/s_utils.py @@ -164,8 +164,8 @@ def rndstr(size=16, alphabet=""): """ rng = random.SystemRandom() if not alphabet: - alphabet = string.letters[0:52] + string.digits - return str().join(rng.choice(alphabet) for _ in range(size)) + alphabet = string.ascii_letters[0:52] + string.digits + return type(alphabet)().join(rng.choice(alphabet) for _ in range(size)) def sid(): From 3db1cb25d83e3665d030b1b81a5f1c0755cb1aa6 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 07:34:28 -0700 Subject: [PATCH 03/12] Add webob to test_requires for python3 Code paths seem different which ends up leading to webob needing to be present. --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 25006c8..7533e08 100755 --- a/setup.py +++ b/setup.py @@ -42,6 +42,7 @@ tests_require = [ 'python-memcached >= 1.51', 'pytest', 'mako', + 'webob', #'pytest-coverage', ] From 9c91638891c9b29d8b1b7032422af5f8aa596fd9 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 23:31:17 -0700 Subject: [PATCH 04/12] Fix more basestring py3k issues basestring has been removed from python 3. --- src/idp_test/__init__.py | 5 +++-- src/idp_test/interaction.py | 6 ++++-- src/saml2/entity.py | 3 ++- src/saml2/httpbase.py | 5 +++-- src/saml2/ident.py | 5 +++-- src/saml2/pack.py | 7 ++++--- src/saml2/s2repoze/plugins/sp.py | 3 ++- src/saml2/sigver.py | 5 +++-- src/saml2test/check.py | 3 ++- src/saml2test/interaction.py | 7 ++++--- src/saml2test/opfunc.py | 3 ++- src/saml2test/tool.py | 3 ++- src/sp_test/base.py | 3 ++- 13 files changed, 36 insertions(+), 22 deletions(-) diff --git a/src/idp_test/__init__.py b/src/idp_test/__init__.py index 61381c5..a061511 100644 --- a/src/idp_test/__init__.py +++ b/src/idp_test/__init__.py @@ -6,6 +6,7 @@ import pprint import types import argparse import sys +import six import logging import imp @@ -356,7 +357,7 @@ class SAML2client(object): item = {"id": key, "name": val["name"]} try: _desc = val["descr"] - if isinstance(_desc, basestring): + if isinstance(_desc, six.string_types): item["descr"] = _desc else: item["descr"] = "\n".join(_desc) @@ -377,7 +378,7 @@ class SAML2client(object): item = {"id": key, "name": val["name"]} try: _desc = val["descr"] - if isinstance(_desc, basestring): + if isinstance(_desc, six.string_types): item["descr"] = _desc else: item["descr"] = "\n".join(_desc) diff --git a/src/idp_test/interaction.py b/src/idp_test/interaction.py index 03da9c2..2fbc46b 100644 --- a/src/idp_test/interaction.py +++ b/src/idp_test/interaction.py @@ -2,6 +2,7 @@ __author__ = 'rohe0002' import json import logging +import six from urlparse import urlparse from bs4 import BeautifulSoup @@ -34,7 +35,8 @@ def pick_interaction(interactions, _base="", content="", req=None): _match += 1 else: _c = _bs.title.contents - if isinstance(_c, list) and not isinstance(_c, basestring): + if isinstance(_c, list) and not isinstance( + _c, six.string_types): for _line in _c: if val in _line: _match += 1 @@ -165,7 +167,7 @@ def pick_form(response, url=None, **kwargs): _default = _ava["value"] try: orig_val = form[prop] - if isinstance(orig_val, basestring): + if isinstance(orig_val, six.string_types): if orig_val == _default: _form = form elif _default in orig_val: diff --git a/src/saml2/entity.py b/src/saml2/entity.py index 6eed861..34b3e90 100644 --- a/src/saml2/entity.py +++ b/src/saml2/entity.py @@ -4,6 +4,7 @@ import logging from hashlib import sha1 from Crypto.PublicKey import RSA import requests +import six from saml2.metadata import ENDPOINTS from saml2.profile import paos, ecp from saml2.soap import parse_soap_enveloped_saml_artifact_resolve @@ -156,7 +157,7 @@ class Entity(HTTPBase): self.sec = security_context(self.config) if virtual_organization: - if isinstance(virtual_organization, basestring): + if isinstance(virtual_organization, six.string_types): self.vorg = self.config.vorg[virtual_organization] elif isinstance(virtual_organization, VirtualOrg): self.vorg = virtual_organization diff --git a/src/saml2/httpbase.py b/src/saml2/httpbase.py index 79d4ba0..2da045f 100644 --- a/src/saml2/httpbase.py +++ b/src/saml2/httpbase.py @@ -1,4 +1,5 @@ import calendar +import six from six.moves import http_cookiejar import copy import re @@ -260,7 +261,7 @@ class HTTPBase(object): :param typ: Whether a Request, Response or Artifact :return: dictionary """ - if not isinstance(message, basestring): + if not isinstance(message, six.string_types): message = "%s" % (message,) return http_form_post_message(message, destination, relay_state, typ) @@ -375,7 +376,7 @@ class HTTPBase(object): :param key: Key to use for signing :return: dictionary """ - if not isinstance(message, basestring): + if not isinstance(message, six.string_types): message = "%s" % (message,) return http_redirect_message(message, destination, relay_state, typ, diff --git a/src/saml2/ident.py b/src/saml2/ident.py index 0420747..c7090bc 100644 --- a/src/saml2/ident.py +++ b/src/saml2/ident.py @@ -1,6 +1,7 @@ import copy import shelve import logging +import six from hashlib import sha256 from urllib import quote @@ -66,7 +67,7 @@ class IdentDB(object): Keeps a list of all nameIDs returned per SP """ def __init__(self, db, domain="", name_qualifier=""): - if isinstance(db, basestring): + if isinstance(db, six.string_types): self.db = shelve.open(db) else: self.db = db @@ -94,7 +95,7 @@ class IdentDB(object): :param ident: user identifier :param name_id: NameID instance """ - if isinstance(ident, unicode): + if isinstance(ident, six.string_types): ident = ident.encode("utf-8") # One user may have more than one NameID defined diff --git a/src/saml2/pack.py b/src/saml2/pack.py index f045c7a..7c6b957 100644 --- a/src/saml2/pack.py +++ b/src/saml2/pack.py @@ -20,6 +20,7 @@ import logging from saml2.sigver import REQ_ORDER from saml2.sigver import RESP_ORDER from saml2.sigver import SIGNER_ALGS +import six logger = logging.getLogger(__name__) @@ -57,7 +58,7 @@ def http_form_post_message(message, location, relay_state="", """ response = ["", """SAML 2.0 POST""", ""] - if not isinstance(message, basestring): + if not isinstance(message, six.string_types): message = "%s" % (message,) if typ == "SAMLRequest" or typ == "SAMLResponse": @@ -94,7 +95,7 @@ def http_redirect_message(message, location, relay_state="", typ="SAMLRequest", :return: A tuple containing header information and a HTML message. """ - if not isinstance(message, basestring): + if not isinstance(message, six.string_types): message = "%s" % (message,) _order = None @@ -163,7 +164,7 @@ def make_soap_enveloped_saml_thingy(thingy, header_parts=None): body.tag = '{%s}Body' % NAMESPACE envelope.append(body) - if isinstance(thingy, basestring): + if isinstance(thingy, six.string_types): # remove the first XML version/encoding line logger.debug("thingy0: %s" % thingy) _part = thingy.split("\n") diff --git a/src/saml2/s2repoze/plugins/sp.py b/src/saml2/s2repoze/plugins/sp.py index bd05a06..2088a5f 100644 --- a/src/saml2/s2repoze/plugins/sp.py +++ b/src/saml2/s2repoze/plugins/sp.py @@ -12,6 +12,7 @@ import platform import shelve import traceback import saml2 +import six from urlparse import parse_qs, urlparse from saml2.md import Extensions from saml2 import xmldsig as ds @@ -555,7 +556,7 @@ class SAML2Plugin(object): def add_metadata(self, environ, identity): """ Add information to the knowledge I have about the user """ name_id = identity['repoze.who.userid'] - if isinstance(name_id, basestring): + if isinstance(name_id, six.string_types): try: # Make sure that userids authenticated by another plugin # don't cause problems here. diff --git a/src/saml2/sigver.py b/src/saml2/sigver.py index cbeec97..2ff13fc 100644 --- a/src/saml2/sigver.py +++ b/src/saml2/sigver.py @@ -15,6 +15,7 @@ import urllib from time import mktime from binascii import hexlify +import six from Crypto.PublicKey.RSA import importKey from Crypto.Signature import PKCS1_v1_5 @@ -728,7 +729,7 @@ class CryptoBackendXmlSec1(CryptoBackend): def __init__(self, xmlsec_binary, **kwargs): CryptoBackend.__init__(self, **kwargs) - assert (isinstance(xmlsec_binary, basestring)) + assert (isinstance(xmlsec_binary, six.string_types)) self.xmlsec = xmlsec_binary if os.environ.get('PYSAML2_KEEP_XMLSEC_TMP', None): self._xmlsec_delete_tmpfiles = False @@ -1353,7 +1354,7 @@ class SecurityContext(object): _certs = [] certs = [] for cert in _certs: - if isinstance(cert, basestring): + if isinstance(cert, six.string_types): certs.append(make_temp(pem_format(cert), suffix=".pem", decode=False, delete=self._xmlsec_delete_tmpfiles)) diff --git a/src/saml2test/check.py b/src/saml2test/check.py index 2433e31..7109371 100644 --- a/src/saml2test/check.py +++ b/src/saml2test/check.py @@ -1,5 +1,6 @@ import inspect import json +import six __author__ = 'rolandh' @@ -91,7 +92,7 @@ class ResponseInfo(Information): self._status = self.status _msg = conv.last_content - if isinstance(_msg, basestring): + if isinstance(_msg, six.string_types): self._message = _msg else: self._message = _msg.to_dict() diff --git a/src/saml2test/interaction.py b/src/saml2test/interaction.py index 0121fbc..ddfbeac 100644 --- a/src/saml2test/interaction.py +++ b/src/saml2test/interaction.py @@ -2,6 +2,7 @@ __author__ = 'rohe0002' import json import logging +import six from urlparse import urlparse from bs4 import BeautifulSoup @@ -125,8 +126,8 @@ class Interaction(object): _match += 1 else: _c = _bs.title.contents - if isinstance(_c, list) and not isinstance(_c, - basestring): + if isinstance(_c, list) and not isinstance( + _c, six.string_types): for _line in _c: if val in _line: _match += 1 @@ -186,7 +187,7 @@ class Interaction(object): _default = _ava["value"] try: orig_val = form[prop] - if isinstance(orig_val, basestring): + if isinstance(orig_val, six.string_types): if orig_val == _default: _form = form elif _default in orig_val: diff --git a/src/saml2test/opfunc.py b/src/saml2test/opfunc.py index 2f88c70..2d612ed 100644 --- a/src/saml2test/opfunc.py +++ b/src/saml2test/opfunc.py @@ -1,5 +1,6 @@ import logging import json +import six from urlparse import urlparse @@ -163,7 +164,7 @@ def pick_form(response, content, url=None, **kwargs): _default = _ava["value"] try: orig_val = form[prop] - if isinstance(orig_val, basestring): + if isinstance(orig_val, six.string_types): if orig_val == _default: _form = form elif _default in orig_val: diff --git a/src/saml2test/tool.py b/src/saml2test/tool.py index 6071408..d911359 100644 --- a/src/saml2test/tool.py +++ b/src/saml2test/tool.py @@ -3,6 +3,7 @@ import sys import traceback import logging from urlparse import parse_qs +import six from saml2test.opfunc import Operation from saml2test import CheckError, FatalError @@ -64,7 +65,7 @@ class Conversation(object): raise CheckError def do_check(self, test, **kwargs): - if isinstance(test, basestring): + if isinstance(test, six.string_types): chk = self.check_factory(test)(**kwargs) else: chk = test(**kwargs) diff --git a/src/sp_test/base.py b/src/sp_test/base.py index eaea070..1ed778c 100644 --- a/src/sp_test/base.py +++ b/src/sp_test/base.py @@ -5,6 +5,7 @@ import os import traceback import urllib import sys +import six from urlparse import parse_qs from saml2 import BINDING_HTTP_REDIRECT, class_name @@ -93,7 +94,7 @@ class Conversation(): raise CheckError def do_check(self, test, **kwargs): - if isinstance(test, basestring): + if isinstance(test, six.string_types): chk = self.check_factory(test)(**kwargs) else: chk = test(**kwargs) From 153de086452652dfb9a81f466e30f2ff305d371a Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 23:38:31 -0700 Subject: [PATCH 05/12] Fix OpenSSL string/bytes python3 issues OpenSSL works a bit differently in python3 which requires delicate handling. --- src/saml2/cert.py | 23 +++++++++++++++++------ tests/test_81_certificates.py | 2 +- 2 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/saml2/cert.py b/src/saml2/cert.py index 8249923..1a8d617 100644 --- a/src/saml2/cert.py +++ b/src/saml2/cert.py @@ -4,6 +4,7 @@ import base64 import datetime import dateutil.parser import pytz +import six from OpenSSL import crypto from os.path import join from os import remove @@ -154,10 +155,13 @@ class OpenSSLWrapper(object): tmp_cert = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) tmp_key = None if cipher_passphrase is not None: + passphrase = cipher_passphrase["passphrase"] + if isinstance(cipher_passphrase["passphrase"], + six.string_types): + passphrase = passphrase.encode('utf-8') tmp_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, k, cipher_passphrase["cipher"], - cipher_passphrase[ - "passphrase"]) + passphrase) else: tmp_key = crypto.dump_privatekey(crypto.FILETYPE_PEM, k) if write_to_file: @@ -190,7 +194,7 @@ class OpenSSLWrapper(object): f.close() def read_str_from_file(self, file, type="pem"): - f = open(file) + f = open(file, 'rt') str_data = f.read() f.close() @@ -257,7 +261,10 @@ class OpenSSLWrapper(object): cert.set_pubkey(req_cert.get_pubkey()) cert.sign(ca_key, hash_alg) - return crypto.dump_certificate(crypto.FILETYPE_PEM, cert) + cert_dump = crypto.dump_certificate(crypto.FILETYPE_PEM, cert) + if isinstance(cert_dump, six.string_types): + return cert_dump + return cert_dump.decode('utf-8') def verify_chain(self, cert_chain_str_list, cert_str): """ @@ -327,6 +334,8 @@ class OpenSSLWrapper(object): "signed certificate.") cert_algorithm = cert.get_signature_algorithm() + if six.PY3: + cert_algorithm = cert_algorithm.decode('ascii') cert_asn1 = crypto.dump_certificate(crypto.FILETYPE_ASN1, cert) @@ -342,7 +351,9 @@ class OpenSSLWrapper(object): signature_payload = cert_signature_decoded.payload - if signature_payload[0] != '\x00': + sig_pay0 = signature_payload[0] + if ((isinstance(sig_pay0, int) and sig_pay0 != 0) or + (isinstance(sig_pay0, str) and sig_pay0 != '\x00')): return (False, "The certificate should not contain any unused bits.") @@ -355,4 +366,4 @@ class OpenSSLWrapper(object): except crypto.Error as e: return False, "Certificate is incorrectly signed." except Exception as e: - return False, "Certificate is not valid for an unknown reason." + return False, "Certificate is not valid for an unknown reason. %s" % str(e) diff --git a/tests/test_81_certificates.py b/tests/test_81_certificates.py index 714a5c4..772211e 100644 --- a/tests/test_81_certificates.py +++ b/tests/test_81_certificates.py @@ -174,7 +174,7 @@ class TestGenerateCertificates(unittest.TestCase): request=True) cert_str = osw.create_cert_signed_certificate(ca_cert_str, ca_key_str, req_cert_str, - passphrase="qwerty") + passphrase=b"qwerty") valid = False try: From da21b279e2fa7d5c94ec370c02a7507f91d8ce43 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 23:42:53 -0700 Subject: [PATCH 06/12] Fix xml issues with python3 Some calls in etree will return bytes where they used to return a string type. --- src/saml2/__init__.py | 6 +++++- src/saml2/pack.py | 2 ++ src/saml2/response.py | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/saml2/__init__.py b/src/saml2/__init__.py index 87bfafa..5d3c351 100644 --- a/src/saml2/__init__.py +++ b/src/saml2/__init__.py @@ -674,7 +674,11 @@ class SamlBase(ExtensionContainer): return ElementTree.tostring(self._to_element_tree(), encoding="UTF-8") def __str__(self): - return self.to_string() + # Yes this is confusing. http://bugs.python.org/issue10942 + x = self.to_string() + if not isinstance(x, six.string_types): + x = x.decode('utf-8') + return x def keyswv(self): """ Return the keys of attributes or children that has values diff --git a/src/saml2/pack.py b/src/saml2/pack.py index 7c6b957..005b2a9 100644 --- a/src/saml2/pack.py +++ b/src/saml2/pack.py @@ -175,6 +175,8 @@ def make_soap_enveloped_saml_thingy(thingy, header_parts=None): _child.tag = '{%s}FuddleMuddle' % DUMMY_NAMESPACE body.append(_child) _str = ElementTree.tostring(envelope, encoding="UTF-8") + if isinstance(_str, six.binary_type): + _str = _str.decode('utf-8') logger.debug("SOAP precursor: %s" % _str) # find an remove the namespace definition i = _str.find(DUMMY_NAMESPACE) diff --git a/src/saml2/response.py b/src/saml2/response.py index aac8224..9c193e1 100644 --- a/src/saml2/response.py +++ b/src/saml2/response.py @@ -4,6 +4,7 @@ import calendar import logging +import six from saml2.samlp import STATUS_VERSION_MISMATCH from saml2.samlp import STATUS_AUTHN_FAILED from saml2.samlp import STATUS_INVALID_ATTR_NAME_OR_VALUE @@ -942,6 +943,8 @@ class AuthnResponse(StatusResponse): "not_on_or_after": nooa, "authn_info": self.authn_info()} def __str__(self): + if not isinstance(self.xmlstr, six.string_types): + return "%s" % self.xmlstr.decode("utf-8") return "%s" % self.xmlstr def verify_attesting_entity(self, address): From 29221328be15255035b9ba221c8efaac52a68431 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 23:53:25 -0700 Subject: [PATCH 07/12] Fix import issues with python3 Relative imports are different and some modules were renamed. --- src/saml2/assertion.py | 2 +- src/saml2/client_base.py | 6 +++--- src/saml2/ident.py | 4 ++-- src/saml2/request.py | 2 +- tests/test_30_mdstore.py | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/saml2/assertion.py b/src/saml2/assertion.py index 58d61ae..8c828b4 100644 --- a/src/saml2/assertion.py +++ b/src/saml2/assertion.py @@ -6,7 +6,7 @@ import logging import re from saml2.saml import NAME_FORMAT_URI import six -import xmlenc +from saml2 import xmlenc from saml2 import saml diff --git a/src/saml2/client_base.py b/src/saml2/client_base.py index 009d438..2661e71 100644 --- a/src/saml2/client_base.py +++ b/src/saml2/client_base.py @@ -6,8 +6,8 @@ to conclude its tasks. """ import threading -from urllib import urlencode -from urlparse import urlparse +from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import urlparse import six from saml2.entity import Entity @@ -25,7 +25,7 @@ import saml2 import time from saml2.soap import make_soap_enveloped_saml_thingy -from urlparse import parse_qs +from six.moves.urllib.parse import parse_qs from saml2.s_utils import signature, UnravelError, exception_trace from saml2.s_utils import do_attributes diff --git a/src/saml2/ident.py b/src/saml2/ident.py index c7090bc..49f8a63 100644 --- a/src/saml2/ident.py +++ b/src/saml2/ident.py @@ -4,8 +4,8 @@ import logging import six from hashlib import sha256 -from urllib import quote -from urllib import unquote +from six.moves.urllib.parse import quote +from six.moves.urllib.parse import unquote from saml2 import SAMLError from saml2.s_utils import rndstr from saml2.s_utils import PolicyError diff --git a/src/saml2/request.py b/src/saml2/request.py index 151de92..8f8cbb7 100644 --- a/src/saml2/request.py +++ b/src/saml2/request.py @@ -1,6 +1,6 @@ import logging -from attribute_converter import to_local +from saml2.attribute_converter import to_local from saml2 import time_util, BINDING_HTTP_REDIRECT from saml2.s_utils import OtherError diff --git a/tests/test_30_mdstore.py b/tests/test_30_mdstore.py index 14a2c09..2017b3d 100644 --- a/tests/test_30_mdstore.py +++ b/tests/test_30_mdstore.py @@ -2,7 +2,7 @@ # -*- coding: utf-8 -*- import datetime import re -from urllib import quote_plus +from six.moves.urllib.parse import quote_plus from saml2.httpbase import HTTPBase from saml2.mdstore import MetadataStore, MetaDataMDX From 1fde97df3914400a15bd5c94f599634c239da628 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Wed, 20 May 2015 23:59:20 -0700 Subject: [PATCH 08/12] Fix bytes/strings logical issues Hashes, and other calls, require bytes or strings in python3 where they were different in python 2.x. --- src/saml2/config.py | 3 ++- src/saml2/eptid.py | 15 +++++++++++++-- src/saml2/ident.py | 20 ++++++++++++++++++-- src/saml2/mdstore.py | 3 +++ src/saml2/mongo_store.py | 8 ++++---- src/saml2/s_utils.py | 9 +++++++++ 6 files changed, 49 insertions(+), 9 deletions(-) diff --git a/src/saml2/config.py b/src/saml2/config.py index d7953a0..bc934f0 100644 --- a/src/saml2/config.py +++ b/src/saml2/config.py @@ -8,6 +8,7 @@ import os import re import logging import logging.handlers +import six from importlib import import_module @@ -296,7 +297,7 @@ class Config(object): def unicode_convert(self, item): try: - return unicode(item, "utf-8") + return six.text_type(item, "utf-8") except TypeError: _uc = self.unicode_convert if isinstance(item, dict): diff --git a/src/saml2/eptid.py b/src/saml2/eptid.py index b2551b4..af5e992 100644 --- a/src/saml2/eptid.py +++ b/src/saml2/eptid.py @@ -8,6 +8,7 @@ import hashlib import shelve import logging +import six logger = logging.getLogger(__name__) @@ -21,13 +22,23 @@ class Eptid(object): md5 = hashlib.md5() for arg in args: md5.update(arg.encode("utf-8")) - md5.update(sp) - md5.update(self.secret) + if isinstance(sp, six.binary_type): + md5.update(sp) + else: + md5.update(sp.encode('utf-8')) + if isinstance(self.secret, six.binary_type): + md5.update(self.secret) + else: + md5.update(self.secret.encode('utf-8')) md5.digest() hashval = md5.hexdigest() + if isinstance(hashval, six.binary_type): + hashval = hashval.decode('ascii') return "!".join([idp, sp, hashval]) def __getitem__(self, key): + if six.PY3 and isinstance(key, six.binary_type): + key = key.decode('utf-8') return self._db[key] def __setitem__(self, key, value): diff --git a/src/saml2/ident.py b/src/saml2/ident.py index 49f8a63..c99a3bd 100644 --- a/src/saml2/ident.py +++ b/src/saml2/ident.py @@ -7,7 +7,7 @@ from hashlib import sha256 from six.moves.urllib.parse import quote from six.moves.urllib.parse import unquote from saml2 import SAMLError -from saml2.s_utils import rndstr +from saml2.s_utils import rndbytes from saml2.s_utils import PolicyError from saml2.saml import NameID from saml2.saml import NAMEID_FORMAT_PERSISTENT @@ -46,6 +46,16 @@ def code(item): return ",".join(_res) +def code_binary(item): + """ + Return a binary 'code' suitable for hashing. + """ + code_str = code(item) + if isinstance(code_str, six.string_types): + return code_str.encode('utf-8') + return code_str + + def decode(txt): """Turns a coded string by code() into a NameID class instance. @@ -75,11 +85,17 @@ class IdentDB(object): self.name_qualifier = name_qualifier def _create_id(self, nformat, name_qualifier="", sp_name_qualifier=""): - _id = sha256(rndstr(32)) + _id = sha256(rndbytes(32)) + if not isinstance(nformat, six.binary_type): + nformat = nformat.encode('utf-8') _id.update(nformat) if name_qualifier: + if not isinstance(name_qualifier, six.binary_type): + name_qualifier = name_qualifier.encode('utf-8') _id.update(name_qualifier) if sp_name_qualifier: + if not isinstance(sp_name_qualifier, six.binary_type): + sp_name_qualifier = sp_name_qualifier.encode('utf-8') _id.update(sp_name_qualifier) return _id.hexdigest() diff --git a/src/saml2/mdstore.py b/src/saml2/mdstore.py index d6db2bd..68206df 100644 --- a/src/saml2/mdstore.py +++ b/src/saml2/mdstore.py @@ -3,6 +3,7 @@ import logging import os import sys import json +import six from hashlib import sha1 from os.path import isfile, join @@ -487,6 +488,8 @@ class InMemoryMetaData(MetaData): try: for srv in ent[desc]: if "artifact_resolution_service" in srv: + if isinstance(eid, six.string_types): + eid = eid.encode('utf-8') s = sha1(eid) res[s.digest()] = ent except KeyError: diff --git a/src/saml2/mongo_store.py b/src/saml2/mongo_store.py index 7190882..ad9b072 100644 --- a/src/saml2/mongo_store.py +++ b/src/saml2/mongo_store.py @@ -9,7 +9,7 @@ from saml2.eptid import Eptid from saml2.mdstore import InMemoryMetaData from saml2.s_utils import PolicyError -from saml2.ident import code, IdentDB, Unknown +from saml2.ident import code_binary, IdentDB, Unknown from saml2.mdie import to_dict, from_dict from saml2 import md @@ -59,7 +59,7 @@ class SessionStorageMDB(object): def store_assertion(self, assertion, to_sign): name_id = assertion.subject.name_id - nkey = sha1(code(name_id)).hexdigest() + nkey = sha1(code_binary(name_id)).hexdigest() doc = { "name_id_key": nkey, @@ -94,7 +94,7 @@ class SessionStorageMDB(object): :return: """ result = [] - key = sha1(code(name_id)).hexdigest() + key = sha1(code_binary(name_id)).hexdigest() for item in self.assertion.find({"name_id_key": key}): assertion = from_dict(item["assertion"], ONTS, True) if session_index or requested_context: @@ -114,7 +114,7 @@ class SessionStorageMDB(object): def remove_authn_statements(self, name_id): logger.debug("remove authn about: %s" % name_id) - key = sha1(code(name_id)).hexdigest() + key = sha1(code_binary(name_id)).hexdigest() for item in self.assertion.find({"name_id_key": key}): self.assertion.remove(item["_id"]) diff --git a/src/saml2/s_utils.py b/src/saml2/s_utils.py index 9d2391e..271dea9 100644 --- a/src/saml2/s_utils.py +++ b/src/saml2/s_utils.py @@ -167,6 +167,15 @@ def rndstr(size=16, alphabet=""): alphabet = string.ascii_letters[0:52] + string.digits return type(alphabet)().join(rng.choice(alphabet) for _ in range(size)) +def rndbytes(size=16, alphabet=""): + """ + Returns rndstr always as a binary type + """ + x = rndstr(size, alphabet) + if isinstance(x, six.string_types): + return x.encode('utf-8') + return x + def sid(): """creates an unique SID for each session. From 4878c2386763516df1a98a035e2370a1d3f8d5f1 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 21 May 2015 00:04:08 -0700 Subject: [PATCH 09/12] Fix missing 'file' keyword for python3 This was an oversight, and accidental that 'file' would be missed, but 'filename' was seemingly intended. --- src/saml2/mdstore.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saml2/mdstore.py b/src/saml2/mdstore.py index 68206df..4cd8bdf 100644 --- a/src/saml2/mdstore.py +++ b/src/saml2/mdstore.py @@ -544,7 +544,7 @@ class MetaDataFile(InMemoryMetaData): """ def __init__(self, onts, attrc, filename=None, cert=None, **kwargs): super(MetaDataFile, self).__init__(onts, attrc, **kwargs) - if not file: + if not filename: raise SAMLError('No file specified.') self.filename = filename self.cert = cert From dcadf9e9f941a818f5a41b6727bde3d271921a57 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 21 May 2015 00:06:01 -0700 Subject: [PATCH 10/12] Fix shelve changes in python3 In python3 dbm works a little differently, as does shelve. Adapting the code for behavior instead of python version doesn't seem to work. --- src/saml2/eptid.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/saml2/eptid.py b/src/saml2/eptid.py index af5e992..d1a9dac 100644 --- a/src/saml2/eptid.py +++ b/src/saml2/eptid.py @@ -58,4 +58,7 @@ class Eptid(object): class EptidShelve(Eptid): def __init__(self, secret, filename): Eptid.__init__(self, secret) - self._db = shelve.open(filename, writeback=True) + if six.PY3: + if filename.endswith('.db'): + filename = filename.rsplit('.db', 1)[0] + self._db = shelve.open(filename, writeback=True, protocol=2) From bb24267e64128298225a73008e0240f516948f10 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 21 May 2015 00:08:10 -0700 Subject: [PATCH 11/12] Fix type comparison strictness fail in python3 In python3 types are more strictly compared sometimes. --- src/saml2/validate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/saml2/validate.py b/src/saml2/validate.py index 094c6c9..28b7f43 100644 --- a/src/saml2/validate.py +++ b/src/saml2/validate.py @@ -305,7 +305,7 @@ def validate_value_type(value, spec): {'base': 'string'} """ if "maxlen" in spec: - return len(value) <= spec["maxlen"] + return len(value) <= int(spec["maxlen"]) if spec["base"] == "string": if "enumeration" in spec: From 95ecd5196cff5dc9e63dd1d18b1a974049c11246 Mon Sep 17 00:00:00 2001 From: Clint Byrum Date: Thu, 21 May 2015 00:09:00 -0700 Subject: [PATCH 12/12] Fix comparison of view to list in python3 In python3, the items method returns a view which won't be equal to a list. One needs to cast out of the view to make it equal. --- tests/test_34_population.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_34_population.py b/tests/test_34_population.py index ebe084b..9abcf65 100644 --- a/tests/test_34_population.py +++ b/tests/test_34_population.py @@ -170,6 +170,6 @@ class TestPopulationMemoryBased(): "eduPersonEntitlement": "Anka"} info = self.population.get_info_from(nid, IDP_OTHER) - assert info.keys() == ["not_on_or_after", "name_id", "ava"] + assert list(info.keys()) == ["not_on_or_after", "name_id", "ava"] assert info["name_id"] == nid assert info["ava"] == {"eduPersonEntitlement": "Anka"}