Fixes to utility library.

analyse.py

@@ -6,7 +6,7 @@ import sys
 import urllib
 
 import conf
-import utilities
+import utility
 
 
 if __name__ == '__main__':
@@ -17,7 +17,7 @@ if __name__ == '__main__':
 
     while day < datetime.datetime.now():
         print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
-        for line in utilities.read_remote_lines(
+        for line in utility.read_remote_lines(
               'http://www.rcbops.com/gerrit/merged/%s/%s/%s'
               % (day.year, day.month, day.day)):
             try:

scrapereviews.py

@@ -8,6 +8,7 @@ import time
 
 import feedutils
 import sql
+import utility
 
 
 def reviews():
@@ -18,9 +19,9 @@ def reviews():
 
     while day < datetime.datetime.now():
         print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
-        data = utilities.read_remote_file(
+        data = utility.read_remote_file(
               'http://www.rcbops.com/gerrit/merged/%s/%s/%s_reviews.json'
-              % (day.year, day.month, day.day)):
+              % (day.year, day.month, day.day))
         j = json.loads(data)
 
         for username in j:

utility.py

@@ -3,6 +3,7 @@
 import decimal
 import types
 import unicodedata
+import urllib
 
 
 def DisplayFriendlySize(bytes):
@@ -28,3 +29,31 @@ def Normalize(value):
   normalized = unicodedata.normalize('NFKD', unicode(value))
   normalized = normalized.encode('ascii', 'ignore')
   return normalized
+
+
+def read_remote_lines(url):
+    remote = urllib.urlopen(url)
+    data = ''
+    while True:
+        d = remote.read(100)
+        if not d:
+            break
+
+        data += d
+
+        if data.find('\n') != -1:
+            elems = data.split('\n')
+            for line in elems[:-1]:
+                yield line
+            data = elems[-1]
+
+    if data:
+        yield data
+
+
+def read_remote_file(url):
+    data = []
+    for line in read_remote_lines(url):
+        data.append(line)
+    return '\n'.join(data)
+