Merge from upstream CVS HEAD

CHANGES

@@ -1,3 +1,20 @@
+----------------------------------------------------------------
+Released 2.4.21 2015-07-xx
+
+Changes since 2.4.20:
+
+Lib/
+* LDAPObject.read_s() now returns None instead of raising 
+  ldap.NO_SUCH_OBJECT in case the search operation returned emtpy result.
+* ldap.resiter.ResultProcessor.allresults() now takes new key-word
+  argument add_ctrls which is internally passed to LDAPObject.result4()
+  and lets the method also return response control along with the search 
+  results.
+* Added ldap.controls.deref implementing support for dereference control
+
+Tests/
+* Unit tests for module ldif (thanks to Petr Viktorin)
+
 ----------------------------------------------------------------
 Released 2.4.20 2015-07-07
 
@@ -1178,4 +1195,4 @@ Released 2.0.0pre02 2002-02-01
 ----------------------------------------------------------------
 Released 1.10alpha3 2000-09-19
 
-$Id: CHANGES,v 1.350 2015/07/07 13:21:42 stroeder Exp $
+$Id: CHANGES,v 1.355 2015/09/19 13:38:30 stroeder Exp $

Demo/pyasn1/derefcontrol.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python
+"""
+This sample script demonstrates the use of the dereference control
+(see https://tools.ietf.org/html/draft-masarati-ldap-deref)
+
+Requires module pyasn1 (see http://pyasn1.sourceforge.net/)
+"""
+
+import pprint,ldap,ldap.modlist,ldap.resiter
+
+from ldap.controls.deref import DereferenceControl
+
+uri = "ldap://ipa.demo1.freeipa.org"
+
+class MyLDAPObject(ldap.ldapobject.LDAPObject,ldap.resiter.ResultProcessor):
+  pass
+
+
+l = MyLDAPObject(uri,trace_level=0)
+l.simple_bind_s('uid=admin,cn=users,cn=accounts,dc=demo1,dc=freeipa,dc=org','Secret123')
+
+dc = DereferenceControl(
+  True,
+  {
+    'member':[
+      'uid',
+      'description',
+      'cn',
+      'mail',
+    ],
+  }
+)
+
+print 'pyasn1 output of request control:'
+print dc._derefSpecs().prettyPrint()
+
+msg_id = l.search_ext(
+  'dc=demo1,dc=freeipa,dc=org',
+  ldap.SCOPE_SUBTREE,
+  '(objectClass=groupOfNames)',
+  attrlist=['cn','objectClass','member','description'],
+  serverctrls = [dc]
+)
+
+for res_type,res_data,res_msgid,res_controls in l.allresults(msg_id,add_ctrls=1):
+  for dn,entry,deref_control in res_data:
+    # process dn and entry
+    print dn,entry['objectClass']
+    if deref_control:
+      pprint.pprint(deref_control[0].derefRes)

Lib/dsml.py

@@ -4,13 +4,13 @@ dsml - generate and parse DSMLv1 data
 
 See http://www.python-ldap.org/ for details.
 
-$Id: dsml.py,v 1.37 2015/06/05 21:04:58 stroeder Exp $
+$Id: dsml.py,v 1.38 2015/08/08 13:36:30 stroeder Exp $
 
 Python compability note:
 Tested with Python 2.0+.
 """
 
-__version__ = '2.4.20'
+__version__ = '2.4.21'
 
 import string,base64
 

Lib/ldap/__init__.py

@@ -3,12 +3,12 @@ ldap - base module
 
 See http://www.python-ldap.org/ for details.
 
-$Id: __init__.py,v 1.97 2015/06/05 21:04:58 stroeder Exp $
+$Id: __init__.py,v 1.98 2015/08/08 13:36:30 stroeder Exp $
 """
 
 # This is also the overall release version number
 
-__version__ = '2.4.20'
+__version__ = '2.4.21'
 
 import sys
 

Lib/ldap/controls/deref.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+"""
+ldap.controls.deref - classes for 
+(see https://tools.ietf.org/html/draft-masarati-ldap-deref)
+
+See http://www.python-ldap.org/ for project details.
+
+$Id: deref.py,v 1.2 2015/09/19 13:41:01 stroeder Exp $
+"""
+
+__all__ = [
+  'DEREF_CONTROL_OID',
+  'DereferenceControl',
+]
+
+import ldap.controls
+from ldap.controls import LDAPControl,KNOWN_RESPONSE_CONTROLS
+
+import pyasn1_modules.rfc2251
+from pyasn1.type import namedtype,univ,tag
+from pyasn1.codec.ber import encoder,decoder
+from pyasn1_modules.rfc2251 import LDAPDN,AttributeDescription,AttributeDescriptionList,AttributeValue
+
+
+DEREF_CONTROL_OID = '1.3.6.1.4.1.4203.666.5.16'
+
+
+# Request types
+#---------------------------------------------------------------------------
+
+# For compability with ASN.1 declaration in I-D
+AttributeList = AttributeDescriptionList
+
+class DerefSpec(univ.Sequence):
+  componentType = namedtype.NamedTypes(
+    namedtype.NamedType(
+      'derefAttr',
+      AttributeDescription()
+    ),
+    namedtype.NamedType(
+      'attributes',
+      AttributeList()
+    ),
+  )
+
+class DerefSpecs(univ.SequenceOf):
+  componentType = DerefSpec()
+
+# Response types
+#---------------------------------------------------------------------------
+
+
+class AttributeValues(univ.SetOf):
+    componentType = AttributeValue()
+
+
+class PartialAttribute(univ.Sequence):
+  componentType = namedtype.NamedTypes(
+    namedtype.NamedType('type', AttributeDescription()),
+    namedtype.NamedType('vals', AttributeValues()),
+  )
+
+
+class PartialAttributeList(univ.SequenceOf):
+  componentType = PartialAttribute()
+  tagSet = univ.Sequence.tagSet.tagImplicitly(
+    tag.Tag(tag.tagClassContext,tag.tagFormatConstructed,0)
+  )
+
+
+class DerefRes(univ.Sequence):
+  componentType = namedtype.NamedTypes(
+    namedtype.NamedType('derefAttr', AttributeDescription()),
+    namedtype.NamedType('derefVal', LDAPDN()),
+    namedtype.OptionalNamedType('attrVals', PartialAttributeList()),
+  )
+
+
+class DerefResultControlValue(univ.SequenceOf):
+    componentType = DerefRes()
+
+
+class DereferenceControl(LDAPControl):
+  controlType = DEREF_CONTROL_OID
+
+  def __init__(self,criticality=False,derefSpecs=None):
+    LDAPControl.__init__(self,self.controlType,criticality)
+    self.derefSpecs = derefSpecs or {}
+
+  def _derefSpecs(self):
+    deref_specs = DerefSpecs()
+    i = 0
+    for deref_attr,deref_attribute_names in self.derefSpecs.items():
+      deref_spec = DerefSpec()
+      deref_attributes = AttributeList()
+      for j in range(len(deref_attribute_names)):
+        deref_attributes.setComponentByPosition(j,deref_attribute_names[j])
+      deref_spec.setComponentByName('derefAttr',AttributeDescription(deref_attr))
+      deref_spec.setComponentByName('attributes',deref_attributes)
+      deref_specs.setComponentByPosition(i,deref_spec)
+      i += 1
+    return deref_specs
+
+  def encodeControlValue(self):
+    return encoder.encode(self._derefSpecs())
+
+  def decodeControlValue(self,encodedControlValue):
+    decodedValue,_ = decoder.decode(encodedControlValue,asn1Spec=DerefResultControlValue())
+    self.derefRes = {}
+    for deref_res in decodedValue:
+      deref_attr,deref_val,deref_vals = deref_res
+      partial_attrs_dict = dict([
+        (str(t),map(str,v))
+        for t,v in deref_vals or []
+      ])
+      try:
+        self.derefRes[str(deref_attr)].append((str(deref_val),partial_attrs_dict))
+      except KeyError:
+        self.derefRes[str(deref_attr)] = [(str(deref_val),partial_attrs_dict)]
+
+
+KNOWN_RESPONSE_CONTROLS[DereferenceControl.controlType] = DereferenceControl

Lib/ldap/controls/openldap.py

@@ -1,9 +1,10 @@
+# -*- coding: utf-8 -*-
 """
 ldap.controls.openldap - classes for OpenLDAP-specific controls
 
 See http://www.python-ldap.org/ for project details.
 
-$Id: openldap.py,v 1.3 2015/06/22 17:56:50 stroeder Exp $
+$Id: openldap.py,v 1.4 2015/09/18 17:24:39 stroeder Exp $
 """
 
 import ldap.controls

Lib/ldap/controls/readentry.py

@@ -1,11 +1,11 @@
-#!/usr/bin/env python
+# -*- coding: utf-8 -*-
 """
 ldap.controls.readentry - classes for the Read Entry controls
 (see RFC 4527)
 
 See http://www.python-ldap.org/ for project details.
 
-$Id: readentry.py,v 1.4 2011/07/28 08:57:12 stroeder Exp $
+$Id: readentry.py,v 1.5 2015/09/18 17:24:55 stroeder Exp $
 """
 
 import ldap

Lib/ldap/controls/sessiontrack.py

@@ -1,10 +1,11 @@
+# -*- coding: utf-8 -*-
 """
 ldap.controls.sessiontrack - class for session tracking control
 (see draft-wahl-ldap-session)
 
 See http://www.python-ldap.org/ for project details.
 
-$Id: sessiontrack.py,v 1.4 2013/07/04 16:20:06 stroeder Exp $
+$Id: sessiontrack.py,v 1.5 2015/09/18 17:25:07 stroeder Exp $
 """
 
 from ldap.controls import RequestControl

Lib/ldap/ldapobject.py

@@ -3,7 +3,7 @@ ldapobject.py - wraps class _ldap.LDAPObject
 
 See http://www.python-ldap.org/ for details.
 
-\$Id: ldapobject.py,v 1.146 2015/06/11 15:13:43 stroeder Exp $
+\$Id: ldapobject.py,v 1.147 2015/08/08 13:37:41 stroeder Exp $
 
 Compability:
 - Tested with Python 2.0+ but should work with Python 1.5.x
@@ -847,7 +847,7 @@ class SimpleLDAPObject:
     if r:
       return r[0][1]
     else:
-      raise ldap.NO_SUCH_OBJECT('Empty search result reading entry %s' % (repr(dn)))
+      return None
 
   def read_subschemasubentry_s(self,subschemasubentry_dn,attrs=None):
     """

Lib/ldap/resiter.py

@@ -3,7 +3,7 @@ ldap.resiter - processing LDAP results with iterators
 
 See http://www.python-ldap.org/ for details.
 
-\$Id: resiter.py,v 1.6 2011/07/28 08:23:32 stroeder Exp $
+\$Id: resiter.py,v 1.7 2015/09/18 20:20:32 stroeder Exp $
 
 Python compability note:
 Requires Python 2.3+
@@ -15,15 +15,15 @@ class ResultProcessor:
   Mix-in class used with ldap.ldapopbject.LDAPObject or derived classes.
   """
 
-  def allresults(self,msgid,timeout=-1):
+  def allresults(self,msgid,timeout=-1,add_ctrls=0):
     """
     Generator function which returns an iterator for processing all LDAP operation
     results of the given msgid retrieved with LDAPObject.result3() -> 4-tuple
     """
-    result_type,result_list,result_msgid,result_serverctrls = self.result3(msgid,0,timeout)
+    result_type,result_list,result_msgid,result_serverctrls,_,_ = self.result4(msgid,0,timeout,add_ctrls=add_ctrls)
     while result_type and result_list:
       # Loop over list of search results
       for result_item in result_list:
         yield (result_type,result_list,result_msgid,result_serverctrls)
-      result_type,result_list,result_msgid,result_serverctrls = self.result3(msgid,0,timeout)
+      result_type,result_list,result_msgid,result_serverctrls,_,_ = self.result4(msgid,0,timeout,add_ctrls=add_ctrls)
     return # allresults()

Lib/ldap/schema/subentry.py

@@ -3,7 +3,7 @@ ldap.schema.subentry -  subschema subentry handling
 
 See http://www.python-ldap.org/ for details.
 
-\$Id: subentry.py,v 1.35 2015/06/06 09:21:38 stroeder Exp $
+\$Id: subentry.py,v 1.36 2015/08/08 14:13:30 stroeder Exp $
 """
 
 import ldap.cidict,ldap.schema
@@ -481,6 +481,7 @@ def urlfetch(uri,trace_level=0,bytes_mode=None):
     subschemasubentry_dn,s_temp = ldif_parser.all_records[0]
   # Work-around for mixed-cased attribute names
   subschemasubentry_entry = ldap.cidict.cidict()
+  s_temp = s_temp or {}
   for at,av in s_temp.items():
     if at in SCHEMA_CLASS_MAPPING:
       try:

Lib/ldapurl.py

@@ -3,7 +3,7 @@ ldapurl - handling of LDAP URLs as described in RFC 4516
 
 See http://www.python-ldap.org/ for details.
 
-\$Id: ldapurl.py,v 1.72 2015/06/06 09:21:37 stroeder Exp $
+\$Id: ldapurl.py,v 1.73 2015/08/08 13:36:30 stroeder Exp $
 
 Python compability note:
 This module only works with Python 2.0+ since
@@ -11,7 +11,7 @@ This module only works with Python 2.0+ since
 2. list comprehensions are used.
 """
 
-__version__ = '2.4.20'
+__version__ = '2.4.21'
 
 __all__ = [
   # constants

Lib/ldif.py

@@ -3,13 +3,13 @@ ldif - generate and parse LDIF data (see RFC 2849)
 
 See http://www.python-ldap.org/ for details.
 
-$Id: ldif.py,v 1.82 2015/06/21 11:38:32 stroeder Exp $
+$Id: ldif.py,v 1.83 2015/08/08 13:36:30 stroeder Exp $
 
 Python compability note:
 Tested with Python 2.0+, but should work with Python 1.5.2+.
 """
 
-__version__ = '2.4.20'
+__version__ = '2.4.21'
 
 __all__ = [
   # constants

Tests/t_ldif.py

@@ -0,0 +1,132 @@
+# -*- coding: utf-8 -*-
+
+import unittest
+import textwrap
+
+import ldif
+
+
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
+
+
+class TestParse(unittest.TestCase):
+    maxDiff = None
+
+    def check_ldif_to_records(self, ldif_string, expected):
+        #import pdb; pdb.set_trace()
+        got = ldif.ParseLDIF(StringIO(ldif_string))
+        self.assertEqual(got, expected)
+
+    def check_records_to_ldif(self, records, expected):
+        f = StringIO()
+        ldif_writer = ldif.LDIFWriter(f)
+        for dn, attrs in records:
+            ldif_writer.unparse(dn, attrs)
+        got = f.getvalue()
+        self.assertEqual(got, expected)
+
+    def check_roundtrip(self, ldif_source, records, ldif_expected=None):
+        ldif_source = textwrap.dedent(ldif_source).lstrip() + '\n'
+        if ldif_expected is None:
+            ldif_expected = ldif_source
+        else:
+            ldif_expected = textwrap.dedent(ldif_expected).lstrip() + '\n'
+
+        self.check_ldif_to_records(ldif_source, records)
+        self.check_records_to_ldif(records, ldif_expected)
+        self.check_ldif_to_records(ldif_expected, records)
+
+    def test_simple(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                attrib: value
+                attrib: value2
+            """, [
+                ('cn=x,cn=y,cn=z', {'attrib': [b'value', b'value2']}),
+            ])
+
+    def test_multiple(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                a: v
+                attrib: value
+                attrib: value2
+
+                dn: cn=a,cn=b,cn=c
+                attrib: value2
+                attrib: value3
+                b: v
+            """, [
+                ('cn=x,cn=y,cn=z', {'attrib': [b'value', b'value2'], 'a': [b'v']}),
+                ('cn=a,cn=b,cn=c', {'attrib': [b'value2', b'value3'], 'b': [b'v']}),
+            ])
+
+    def test_folded(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                attrib: very
+                 long
+                  value
+                attrib2: %s
+            """ % ('asdf.' * 20), [
+                ('cn=x,cn=y,cn=z', {'attrib': [b'verylong value'],
+                                    'attrib2': [b'asdf.' * 20]}),
+            ], """
+                dn: cn=x,cn=y,cn=z
+                attrib: verylong value
+                attrib2: asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.asdf.as
+                 df.asdf.asdf.asdf.asdf.asdf.asdf.
+            """)
+
+    def test_empty(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                attrib: 
+                attrib: foo
+            """, [
+                ('cn=x,cn=y,cn=z', {'attrib': [b'', b'foo']}),
+            ])
+
+    def test_binary(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                attrib:: CQAKOiVA
+            """, [
+                ('cn=x,cn=y,cn=z', {'attrib': [b'\t\0\n:%@']}),
+            ])
+
+    def test_unicode(self):
+        self.check_roundtrip("""
+                dn: cn=Michael Stroeder,dc=stroeder,dc=com
+                lastname: Ströder
+            """, [
+                ('cn=Michael Stroeder,dc=stroeder,dc=com',
+                 {'lastname': [b'Str\303\266der']}),
+            ], """
+                dn: cn=Michael Stroeder,dc=stroeder,dc=com
+                lastname:: U3Ryw7ZkZXI=
+            """)
+
+    def test_sorted(self):
+        self.check_roundtrip("""
+                dn: cn=x,cn=y,cn=z
+                b: value_b
+                c: value_c
+                a: value_a
+            """, [
+                ('cn=x,cn=y,cn=z', {'a': [b'value_a'],
+                                    'b': [b'value_b'],
+                                    'c': [b'value_c']}),
+            ], """
+                dn: cn=x,cn=y,cn=z
+                a: value_a
+                b: value_b
+                c: value_c
+            """)
+
+
+if __name__ == '__main__':
+    unittest.main()

setup.py

@@ -3,7 +3,7 @@ setup.py - Setup package with the help Python's DistUtils
 
 See http://www.python-ldap.org/ for details.
 
-$Id: setup.py,v 1.72 2014/03/12 20:29:23 stroeder Exp $
+$Id: setup.py,v 1.73 2015/09/19 13:38:30 stroeder Exp $
 """
 
 import sys,os,string,time
@@ -164,6 +164,7 @@ setup(
     'ldap.async',
     'ldap.compat',
     'ldap.controls',
+    'ldap.controls.deref',
     'ldap.controls.libldap',
     'ldap.controls.openldap',
     'ldap.controls.ppolicy',