Fixes to utility library.
This commit is contained in:
parent
05b2273abe
commit
99aa2df294
@ -6,7 +6,7 @@ import sys
|
||||
import urllib
|
||||
|
||||
import conf
|
||||
import utilities
|
||||
import utility
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
@ -17,7 +17,7 @@ if __name__ == '__main__':
|
||||
|
||||
while day < datetime.datetime.now():
|
||||
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
||||
for line in utilities.read_remote_lines(
|
||||
for line in utility.read_remote_lines(
|
||||
'http://www.rcbops.com/gerrit/merged/%s/%s/%s'
|
||||
% (day.year, day.month, day.day)):
|
||||
try:
|
||||
|
@ -8,6 +8,7 @@ import time
|
||||
|
||||
import feedutils
|
||||
import sql
|
||||
import utility
|
||||
|
||||
|
||||
def reviews():
|
||||
@ -18,9 +19,9 @@ def reviews():
|
||||
|
||||
while day < datetime.datetime.now():
|
||||
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
||||
data = utilities.read_remote_file(
|
||||
data = utility.read_remote_file(
|
||||
'http://www.rcbops.com/gerrit/merged/%s/%s/%s_reviews.json'
|
||||
% (day.year, day.month, day.day)):
|
||||
% (day.year, day.month, day.day))
|
||||
j = json.loads(data)
|
||||
|
||||
for username in j:
|
||||
|
@ -3,6 +3,7 @@
|
||||
import decimal
|
||||
import types
|
||||
import unicodedata
|
||||
import urllib
|
||||
|
||||
|
||||
def DisplayFriendlySize(bytes):
|
||||
@ -28,3 +29,31 @@ def Normalize(value):
|
||||
normalized = unicodedata.normalize('NFKD', unicode(value))
|
||||
normalized = normalized.encode('ascii', 'ignore')
|
||||
return normalized
|
||||
|
||||
|
||||
def read_remote_lines(url):
|
||||
remote = urllib.urlopen(url)
|
||||
data = ''
|
||||
while True:
|
||||
d = remote.read(100)
|
||||
if not d:
|
||||
break
|
||||
|
||||
data += d
|
||||
|
||||
if data.find('\n') != -1:
|
||||
elems = data.split('\n')
|
||||
for line in elems[:-1]:
|
||||
yield line
|
||||
data = elems[-1]
|
||||
|
||||
if data:
|
||||
yield data
|
||||
|
||||
|
||||
def read_remote_file(url):
|
||||
data = []
|
||||
for line in read_remote_lines(url):
|
||||
data.append(line)
|
||||
return '\n'.join(data)
|
||||
|
Loading…
Reference in New Issue
Block a user