From 378a0aa6a82a00c26d2bda239eb45cf3600db7ac Mon Sep 17 00:00:00 2001 From: Michael Comella Date: Wed, 17 Feb 2016 18:20:20 -0800 Subject: [PATCH] Bug 1244861 - Gzip outgoing telemetry pings. r=rnewman This commit adds the GzipNonChunkedCompressingEntity which is necessary because the telemetry servers don't support chunked uploading, which the built in GzipCompressingEntity does. I tested this on my local device and logs for successful uploads were sent for both the testing gzip server as well as the official telemetry server. My data correctly appears on the former and I did not check the latter. MozReview-Commit-ID: 4bCNiRYyqFD --HG-- extra : rebase_source : 28f877258fdf25a098bc5f1634c060b3fd53fe39 --- mobile/android/base/android-services.mozbuild | 1 + .../telemetry/TelemetryUploadService.java | 2 + .../mozilla/gecko/sync/net/BaseResource.java | 36 ++++++++ .../net/GzipNonChunkedCompressingEntity.java | 92 +++++++++++++++++++ 4 files changed, 131 insertions(+) create mode 100644 mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java diff --git a/mobile/android/base/android-services.mozbuild b/mobile/android/base/android-services.mozbuild index 0f7ebb4aea88..622409f21cff 100644 --- a/mobile/android/base/android-services.mozbuild +++ b/mobile/android/base/android-services.mozbuild @@ -912,6 +912,7 @@ sync_java_files = [TOPSRCDIR + '/mobile/android/services/src/main/java/org/mozil 'sync/net/BearerAuthHeaderProvider.java', 'sync/net/BrowserIDAuthHeaderProvider.java', 'sync/net/ConnectionMonitorThread.java', + 'sync/net/GzipNonChunkedCompressingEntity.java', 'sync/net/HandleProgressException.java', 'sync/net/HawkAuthHeaderProvider.java', 'sync/net/HMACAuthHeaderProvider.java', diff --git a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java index 54b960bc6e56..384870414c59 100644 --- a/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java +++ b/mobile/android/base/java/org/mozilla/gecko/telemetry/TelemetryUploadService.java @@ -200,6 +200,8 @@ public class TelemetryUploadService extends BackgroundService { delegate.setResource(resource); resource.delegate = delegate; + resource.setShouldCompressUploadedEntity(true); + resource.setShouldChunkUploadsHint(false); // Telemetry servers don't support chunking. // We're in a background thread so we don't have any reason to do this asynchronously. // If we tried, onStartCommand would return and IntentService might stop itself before we finish. diff --git a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java index 3bb90dbde7fa..60bbc86bbca7 100644 --- a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java +++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/BaseResource.java @@ -30,6 +30,7 @@ import ch.boye.httpclientandroidlib.HttpResponse; import ch.boye.httpclientandroidlib.HttpVersion; import ch.boye.httpclientandroidlib.client.AuthCache; import ch.boye.httpclientandroidlib.client.ClientProtocolException; +import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity; import ch.boye.httpclientandroidlib.client.methods.HttpDelete; import ch.boye.httpclientandroidlib.client.methods.HttpGet; import ch.boye.httpclientandroidlib.client.methods.HttpPatch; @@ -80,6 +81,10 @@ public class BaseResource implements Resource { protected HttpRequestBase request; public final String charset = "utf-8"; + private boolean shouldGzipCompress = false; + // A hint whether uploaded payloads are chunked. Default true to use GzipCompressingEntity, which is built-in functionality. + private boolean shouldChunkUploadsHint = true; + /** * We have very few writes (observers tend to be installed around sync * sessions) and many iterations (every HTTP request iterates observers), so @@ -162,6 +167,34 @@ public class BaseResource implements Resource { return this.getURI().getHost(); } + /** + * Causes the Resource to compress the uploaded entity payload in requests with payloads (e.g. post, put) + * @param shouldCompress true if the entity should be compressed, false otherwise + */ + public void setShouldCompressUploadedEntity(final boolean shouldCompress) { + shouldGzipCompress = shouldCompress; + } + + /** + * Causes the Resource to chunk the uploaded entity payload in requests with payloads (e.g. post, put). + * Note: this flag is only a hint - chunking is not guaranteed. + * + * Chunking is currently supported with gzip compression. + * + * @param shouldChunk true if the transfer should be chunked, false otherwise + */ + public void setShouldChunkUploadsHint(final boolean shouldChunk) { + shouldChunkUploadsHint = shouldChunk; + } + + private HttpEntity getMaybeCompressedEntity(final HttpEntity entity) { + if (!shouldGzipCompress) { + return entity; + } + + return shouldChunkUploadsHint ? new GzipCompressingEntity(entity) : new GzipNonChunkedCompressingEntity(entity); + } + /** * This shuts up HttpClient, which will otherwise debug log about there * being no auth cache in the context. @@ -365,6 +398,7 @@ public class BaseResource implements Resource { @Override public void post(HttpEntity body) { Logger.debug(LOG_TAG, "HTTP POST " + this.uri.toASCIIString()); + body = getMaybeCompressedEntity(body); HttpPost request = new HttpPost(this.uri); request.setEntity(body); this.go(request); @@ -373,6 +407,7 @@ public class BaseResource implements Resource { @Override public void patch(HttpEntity body) { Logger.debug(LOG_TAG, "HTTP PATCH " + this.uri.toASCIIString()); + body = getMaybeCompressedEntity(body); HttpPatch request = new HttpPatch(this.uri); request.setEntity(body); this.go(request); @@ -381,6 +416,7 @@ public class BaseResource implements Resource { @Override public void put(HttpEntity body) { Logger.debug(LOG_TAG, "HTTP PUT " + this.uri.toASCIIString()); + body = getMaybeCompressedEntity(body); HttpPut request = new HttpPut(this.uri); request.setEntity(body); this.go(request); diff --git a/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java new file mode 100644 index 000000000000..1e238c022a4c --- /dev/null +++ b/mobile/android/services/src/main/java/org/mozilla/gecko/sync/net/GzipNonChunkedCompressingEntity.java @@ -0,0 +1,92 @@ +/* This Source Code Form is subject to the terms of the Mozilla Public + * License, v. 2.0. If a copy of the MPL was not distributed with this + * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ + +package org.mozilla.gecko.sync.net; + +import ch.boye.httpclientandroidlib.HttpEntity; +import ch.boye.httpclientandroidlib.client.entity.GzipCompressingEntity; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; + +/** + * Wrapping entity that compresses content when {@link #writeTo writing}. + * + * This differs from {@link GzipCompressingEntity} in that it does not chunk + * the sent data, therefore replacing the "Transfer-Encoding" HTTP header with + * the "Content-Length" header required by some servers. + * + * However, to measure the content length, the gzipped content will be temporarily + * stored in memory so be careful what content you send! + */ +public class GzipNonChunkedCompressingEntity extends GzipCompressingEntity { + final int MAX_BUFFER_SIZE_BYTES = 10 * 1000 * 1000; // 10 MB. + + private byte[] gzippedContent; + + public GzipNonChunkedCompressingEntity(final HttpEntity entity) { + super(entity); + } + + /** + * @return content length for gzipped content or -1 if there is an error + */ + @Override + public long getContentLength() { + try { + initBuffer(); + } catch (final IOException e) { + // GzipCompressingEntity always returns -1 in which case a 'Content-Length' header is omitted. + // Presumably, without it the request will fail (either client-side or server-side). + return -1; + } + return gzippedContent.length; + } + + @Override + public boolean isChunked() { + // "Content-Length" & chunked encoding are mutually exclusive: + // https://en.wikipedia.org/wiki/Chunked_transfer_encoding + return false; + } + + @Override + public InputStream getContent() throws IOException { + initBuffer(); + return new ByteArrayInputStream(gzippedContent); + } + + @Override + public void writeTo(final OutputStream outstream) throws IOException { + initBuffer(); + outstream.write(gzippedContent); + } + + private void initBuffer() throws IOException { + if (gzippedContent != null) { + return; + } + + final long unzippedContentLength = wrappedEntity.getContentLength(); + if (unzippedContentLength > MAX_BUFFER_SIZE_BYTES) { + throw new IOException( + "Wrapped entity content length, " + unzippedContentLength + " bytes, exceeds max: " + MAX_BUFFER_SIZE_BYTES); + } + + // The buffer size needed by the gzipped content should be smaller than this, + // but it's more efficient just to allocate one larger buffer than allocate + // twice if the gzipped content is too large for the default buffer. + final ByteArrayOutputStream s = new ByteArrayOutputStream((int) unzippedContentLength); + try { + super.writeTo(s); + } finally { + s.close(); + } + + gzippedContent = s.toByteArray(); + } +}