opendev/gerrit
Code Issues Proposed changes

Limit the total upload for a file to at most 1 MB

Browse Source 
Some files can compress very well, e.g. a 6 MB XML file may compress
as small as 300 KB.  The compressed size of the base file plus the
patch was well under the 1 MB limit, so we were able to upload the
data to GAE, but Gerrit couldn't expand the file in-memory as the
24 MB required (old + new image, 2x for split and unsplit versions)
was above the allowed memory usage.

Instead we need to base our limit decision here on the uncompressed
size.  We should have enough buffer space between the true 1 MB
limit and the MAX_DATA_SIZE that any zlib header/footers added
on uncompressable content will still let us fit in under the 1 MB
post limit.

Signed-off-by: Shawn O. Pearce <sop@google.com>
This commit is contained in:
Shawn O. Pearce
2008-10-22 09:40:50 -07:00
parent 6d9efb0f88
commit 080d760dd6
1 changed files with 8 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

29
mgrapp/src/com/google/codereview/manager/unpack/PatchSetUploader.java
View File

@@ -38,7 +38,6 @@ import java.io.Writer;
import java.security.MessageDigest;
import java.util.zip.Deflater;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.InflaterInputStream;
class PatchSetUploader implements Runnable {
@@ -179,8 +178,6 @@ class PatchSetUploader implements Runnable {
req.setFileName(diff.getFilename());
req.setStatus(diff.getStatus());
ByteString patchz = deflate(diff.getPatch());
if (!diff.isBinary() && !diff.isTruncated()) {
final ObjectId baseId = diff.getBaseId();
@@ -195,13 +192,12 @@ class PatchSetUploader implements Runnable {
throw new StopProcessingException("No " + baseId.name());
}
final ByteString basez = deflate(ldr.getCachedBytes());
if (basez.size() + patchz.size() > MAX_DATA_SIZE) {
final byte[] base = ldr.getCachedBytes();
if (base.length + diff.getPatchSize() > MAX_DATA_SIZE) {
diff.truncatePatch();
patchz = deflate(diff.getPatch());
} else {
req.setBaseId(baseId.name());
req.setBaseZ(basez);
req.setBaseZ(deflate(base));
}
} catch (IOException err) {
LOG.fatal(logkey() + " cannot read base " + baseId.name(), err);
@@ -219,8 +215,9 @@ class PatchSetUploader implements Runnable {
}
}
req.setPatchZ(patchz);
req.setPatchId(hashOfInflated(patchz));
final byte[] rawpatch = diff.getPatch();
req.setPatchZ(deflate(rawpatch));
req.setPatchId(hashOf(rawpatch));
return req.build();
}
@@ -238,19 +235,9 @@ class PatchSetUploader implements Runnable {
return r.toByteString();
}
private static String hashOfInflated(final ByteString in) {
private static String hashOf(final byte[] in) {
final MessageDigest md = Constants.newMessageDigest();
final byte[] tmp = new byte[512];
final InflaterInputStream iis = new InflaterInputStream(in.newInput());
int cnt;
try {
while ((cnt = iis.read(tmp)) > 0) {
md.update(tmp, 0, cnt);
}
} catch (IOException err) {
// This should not happen.
throw new StopProcessingException("Unexpected IO error", err);
}
md.update(in, 0, in.length);
return ObjectId.fromRaw(md.digest()).name();
}
}