Fixes to utility library.
This commit is contained in:
parent
05b2273abe
commit
99aa2df294
@ -6,7 +6,7 @@ import sys
|
|||||||
import urllib
|
import urllib
|
||||||
|
|
||||||
import conf
|
import conf
|
||||||
import utilities
|
import utility
|
||||||
|
|
||||||
|
|
||||||
if __name__ == '__main__':
|
if __name__ == '__main__':
|
||||||
@ -17,7 +17,7 @@ if __name__ == '__main__':
|
|||||||
|
|
||||||
while day < datetime.datetime.now():
|
while day < datetime.datetime.now():
|
||||||
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
||||||
for line in utilities.read_remote_lines(
|
for line in utility.read_remote_lines(
|
||||||
'http://www.rcbops.com/gerrit/merged/%s/%s/%s'
|
'http://www.rcbops.com/gerrit/merged/%s/%s/%s'
|
||||||
% (day.year, day.month, day.day)):
|
% (day.year, day.month, day.day)):
|
||||||
try:
|
try:
|
||||||
|
@ -8,6 +8,7 @@ import time
|
|||||||
|
|
||||||
import feedutils
|
import feedutils
|
||||||
import sql
|
import sql
|
||||||
|
import utility
|
||||||
|
|
||||||
|
|
||||||
def reviews():
|
def reviews():
|
||||||
@ -18,9 +19,9 @@ def reviews():
|
|||||||
|
|
||||||
while day < datetime.datetime.now():
|
while day < datetime.datetime.now():
|
||||||
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
print 'Processing %s/%s/%s' % (day.year, day.month, day.day)
|
||||||
data = utilities.read_remote_file(
|
data = utility.read_remote_file(
|
||||||
'http://www.rcbops.com/gerrit/merged/%s/%s/%s_reviews.json'
|
'http://www.rcbops.com/gerrit/merged/%s/%s/%s_reviews.json'
|
||||||
% (day.year, day.month, day.day)):
|
% (day.year, day.month, day.day))
|
||||||
j = json.loads(data)
|
j = json.loads(data)
|
||||||
|
|
||||||
for username in j:
|
for username in j:
|
||||||
|
@ -3,6 +3,7 @@
|
|||||||
import decimal
|
import decimal
|
||||||
import types
|
import types
|
||||||
import unicodedata
|
import unicodedata
|
||||||
|
import urllib
|
||||||
|
|
||||||
|
|
||||||
def DisplayFriendlySize(bytes):
|
def DisplayFriendlySize(bytes):
|
||||||
@ -28,3 +29,31 @@ def Normalize(value):
|
|||||||
normalized = unicodedata.normalize('NFKD', unicode(value))
|
normalized = unicodedata.normalize('NFKD', unicode(value))
|
||||||
normalized = normalized.encode('ascii', 'ignore')
|
normalized = normalized.encode('ascii', 'ignore')
|
||||||
return normalized
|
return normalized
|
||||||
|
|
||||||
|
|
||||||
|
def read_remote_lines(url):
|
||||||
|
remote = urllib.urlopen(url)
|
||||||
|
data = ''
|
||||||
|
while True:
|
||||||
|
d = remote.read(100)
|
||||||
|
if not d:
|
||||||
|
break
|
||||||
|
|
||||||
|
data += d
|
||||||
|
|
||||||
|
if data.find('\n') != -1:
|
||||||
|
elems = data.split('\n')
|
||||||
|
for line in elems[:-1]:
|
||||||
|
yield line
|
||||||
|
data = elems[-1]
|
||||||
|
|
||||||
|
if data:
|
||||||
|
yield data
|
||||||
|
|
||||||
|
|
||||||
|
def read_remote_file(url):
|
||||||
|
data = []
|
||||||
|
for line in read_remote_lines(url):
|
||||||
|
data.append(line)
|
||||||
|
return '\n'.join(data)
|
||||||
|
|
Loading…
Reference in New Issue
Block a user