Bug 1311398 - Load public suffix list from disk and extract activity stream labels asynchronously. r=Grisha

Not really happy about the change but it addresses the problem.

* Loading the list from disk is much faster than parsing the string (2500ms vs. 30-50ms on a Nexus 6P)
* Because of (potential) disk I/O we are required to extract the label asynchronously on a background thread
* We load the list only once and then keep it in memory - like we did before the change.

MozReview-Commit-ID: 9MPGbmIGRnS

--HG--
extra : rebase_source : 8b31f852c6bb90fd57baeb07ba0066421d5a6e46
This commit is contained in:
Sebastian Kaspari 2016-11-02 13:56:43 +01:00
Родитель b25f505550
Коммит 2130ff7058
10 изменённых файлов: 8649 добавлений и 235 удалений

Разница между файлами не показана из-за своего большого размера Загрузить разницу

Просмотреть файл

@ -7,6 +7,7 @@ package org.mozilla.gecko.activitystream;
import android.content.Context;
import android.net.Uri;
import android.os.AsyncTask;
import android.text.TextUtils;
import com.keepsafe.switchboard.SwitchBoard;
@ -94,41 +95,55 @@ public class ActivityStream {
*
* @param usePath Use the path of the URL to extract a label (if suitable)
*/
public static String extractLabel(String url, boolean usePath) {
if (TextUtils.isEmpty(url)) {
return "";
}
public static void extractLabel(final Context context, final String url, final boolean usePath, final LabelCallback callback) {
new AsyncTask<Void, Void, String>() {
@Override
protected String doInBackground(Void... params) {
if (TextUtils.isEmpty(url)) {
return "";
}
final Uri uri = Uri.parse(url);
final Uri uri = Uri.parse(url);
// Use last path segment if suitable
if (usePath) {
final String segment = uri.getLastPathSegment();
if (!TextUtils.isEmpty(segment)
&& !UNDESIRED_LABELS.contains(segment)
&& !segment.matches("^[0-9]+$")) {
// Use last path segment if suitable
if (usePath) {
final String segment = uri.getLastPathSegment();
if (!TextUtils.isEmpty(segment)
&& !UNDESIRED_LABELS.contains(segment)
&& !segment.matches("^[0-9]+$")) {
boolean hasUndesiredPrefix = false;
for (int i = 0; i < UNDESIRED_LABEL_PREFIXES.size(); i++) {
if (segment.startsWith(UNDESIRED_LABEL_PREFIXES.get(i))) {
hasUndesiredPrefix = true;
break;
boolean hasUndesiredPrefix = false;
for (int i = 0; i < UNDESIRED_LABEL_PREFIXES.size(); i++) {
if (segment.startsWith(UNDESIRED_LABEL_PREFIXES.get(i))) {
hasUndesiredPrefix = true;
break;
}
}
if (!hasUndesiredPrefix) {
return segment;
}
}
}
if (!hasUndesiredPrefix) {
return segment;
// If no usable path segment was found then use the host without public suffix and common subdomains
final String host = uri.getHost();
if (TextUtils.isEmpty(host)) {
return url;
}
return StringUtils.stripCommonSubdomains(
PublicSuffix.stripPublicSuffix(context, host));
}
}
// If no usable path segment was found then use the host without public suffix and common subdomains
final String host = uri.getHost();
if (TextUtils.isEmpty(host)) {
return url;
}
@Override
protected void onPostExecute(String label) {
callback.onLabelExtracted(label);
}
}.execute();
}
return StringUtils.stripCommonSubdomains(
PublicSuffix.stripPublicSuffix(host));
public abstract static class LabelCallback {
public abstract void onLabelExtracted(String label);
}
}

Просмотреть файл

@ -17,6 +17,7 @@ import android.widget.ImageView;
import android.widget.TextView;
import org.mozilla.gecko.R;
import org.mozilla.gecko.activitystream.ActivityStream.LabelCallback;
import org.mozilla.gecko.db.BrowserContract;
import org.mozilla.gecko.home.HomePager;
import org.mozilla.gecko.home.activitystream.menu.ActivityStreamContextMenu;
@ -165,14 +166,13 @@ public abstract class StreamItem extends RecyclerView.ViewHolder {
vSourceView.setText(vSourceView.getText());
}
private void updatePage(String url) {
final String label = extractLabel(url, false);
if (!TextUtils.isEmpty(label)) {
vPageView.setText(label);
} else {
vPageView.setText(url);
}
private void updatePage(final String url) {
extractLabel(itemView.getContext(), url, false, new LabelCallback() {
@Override
public void onLabelExtracted(String label) {
vPageView.setText(TextUtils.isEmpty(label) ? url : label);
}
});
}
@Override

Просмотреть файл

@ -107,6 +107,7 @@ public class StreamRecyclerAdapter extends RecyclerView.Adapter<StreamItem> impl
@Override
public int getItemCount() {
final int highlightsCount;
if (highlightsCursor != null) {
highlightsCount = highlightsCursor.getCount();
} else {

Просмотреть файл

@ -22,6 +22,7 @@ import org.mozilla.gecko.IntentHelper;
import org.mozilla.gecko.R;
import org.mozilla.gecko.Telemetry;
import org.mozilla.gecko.TelemetryContract;
import org.mozilla.gecko.activitystream.ActivityStream;
import org.mozilla.gecko.db.BrowserDB;
import org.mozilla.gecko.home.HomePager;
import org.mozilla.gecko.icons.IconCallback;
@ -76,8 +77,11 @@ public class ActivityStreamContextMenu
setContentView(content);
((TextView) findViewById(R.id.title)).setText(title);
final String label = extractLabel(url, false);
((TextView) findViewById(R.id.url)).setText(label);
extractLabel(context, url, false, new ActivityStream.LabelCallback() {
public void onLabelExtracted(String label) {
((TextView) findViewById(R.id.url)).setText(label);
}
});
// Copy layouted parameters from the Highlights / TopSites items to ensure consistency
final FaviconView faviconView = (FaviconView) findViewById(R.id.icon);

Просмотреть файл

@ -56,8 +56,12 @@ class TopSitesCard extends RecyclerView.ViewHolder
}
void bind(final TopSitesPageAdapter.TopSite topSite) {
final String label = ActivityStream.extractLabel(topSite.url, true);
title.setText(label);
ActivityStream.extractLabel(itemView.getContext(), topSite.url, true, new ActivityStream.LabelCallback() {
@Override
public void onLabelExtracted(String label) {
title.setText(label);
}
});
this.url = topSite.url;

Просмотреть файл

@ -4,15 +4,16 @@
package org.mozilla.gecko.util.publicsuffix;
import android.content.Context;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.text.TextUtils;
import android.support.annotation.WorkerThread;
import org.mozilla.gecko.util.StringUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Set;
/**
* Helper methods for the public suffix part of a domain.
@ -34,12 +35,13 @@ public class PublicSuffix {
* independent.co.uk -> independent
*/
@NonNull
public static String stripPublicSuffix(@NonNull String domain) {
@WorkerThread // This method might need to load data from disk
public static String stripPublicSuffix(Context context, @NonNull String domain) {
if (domain.length() == 0) {
return domain;
}
final int index = findPublicSuffixIndex(domain);
final int index = findPublicSuffixIndex(context, domain);
if (index == -1) {
return domain;
}
@ -50,14 +52,16 @@ public class PublicSuffix {
/**
* Returns the index of the leftmost part of the public suffix, or -1 if not found.
*/
private static int findPublicSuffixIndex(String domain) {
@WorkerThread
private static int findPublicSuffixIndex(Context context, String domain) {
final List<String> parts = normalizeAndSplit(domain);
final int partsSize = parts.size();
final Set<String> exact = PublicSuffixPatterns.getExactSet(context);
for (int i = 0; i < partsSize; i++) {
String ancestorName = StringUtils.join(".", parts.subList(i, partsSize));
if (PublicSuffixPatterns.EXACT.contains(ancestorName)) {
if (exact.contains(ancestorName)) {
return joinIndex(parts, i);
}

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

Просмотреть файл

@ -3,9 +3,14 @@
package org.mozilla.gecko.activitystream;
import android.os.SystemClock;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mozilla.gecko.background.testhelpers.TestRunner;
import org.robolectric.Robolectric;
import org.robolectric.RuntimeEnvironment;
import org.robolectric.shadows.ShadowLooper;
import static org.junit.Assert.assertEquals;
@ -20,94 +25,61 @@ public class TestActivityStream {
@Test
public void testExtractLabelWithPath() {
// Empty values
assertEquals("", ActivityStream.extractLabel("", true));
assertEquals("", ActivityStream.extractLabel(null, true));
assertLabelEquals("", "", true);
assertLabelEquals("", null, true);
// Without path
assertEquals("news.ycombinator",
ActivityStream.extractLabel("https://news.ycombinator.com/", true));
assertEquals("sql.telemetry.mozilla",
ActivityStream.extractLabel("https://sql.telemetry.mozilla.org/", true));
assertEquals("sso.mozilla",
ActivityStream.extractLabel("http://sso.mozilla.com/", true));
assertEquals("youtube",
ActivityStream.extractLabel("http://youtube.com/", true));
assertEquals("images.google",
ActivityStream.extractLabel("http://images.google.com/", true));
assertEquals("smile.amazon",
ActivityStream.extractLabel("http://smile.amazon.com/", true));
assertEquals("localhost",
ActivityStream.extractLabel("http://localhost:5000/", true));
assertEquals("independent",
ActivityStream.extractLabel("http://www.independent.co.uk/", true));
assertLabelEquals("news.ycombinator", "https://news.ycombinator.com/", true);
assertLabelEquals("sql.telemetry.mozilla", "https://sql.telemetry.mozilla.org/", true);
assertLabelEquals("sso.mozilla", "http://sso.mozilla.com/", true);
assertLabelEquals("youtube", "http://youtube.com/", true);
assertLabelEquals("images.google", "http://images.google.com/", true);
assertLabelEquals("smile.amazon", "http://smile.amazon.com/", true);
assertLabelEquals("localhost", "http://localhost:5000/", true);
assertLabelEquals("independent", "http://www.independent.co.uk/", true);
// With path
assertEquals("firefox",
ActivityStream.extractLabel("https://addons.mozilla.org/en-US/firefox/", true));
assertEquals("activity-stream",
ActivityStream.extractLabel("https://trello.com/b/KX3hV8XS/activity-stream", true));
assertEquals("activity-stream",
ActivityStream.extractLabel("https://github.com/mozilla/activity-stream", true));
assertEquals("sidekiq",
ActivityStream.extractLabel("https://dispatch-news.herokuapp.com/sidekiq", true));
assertEquals("nchapman",
ActivityStream.extractLabel("https://github.com/nchapman/", true));
assertLabelEquals("firefox", "https://addons.mozilla.org/en-US/firefox/", true);
assertLabelEquals("activity-stream", "https://trello.com/b/KX3hV8XS/activity-stream", true);
assertLabelEquals("activity-stream", "https://github.com/mozilla/activity-stream", true);
assertLabelEquals("sidekiq", "https://dispatch-news.herokuapp.com/sidekiq", true);
assertLabelEquals("nchapman", "https://github.com/nchapman/", true);
// Unusable paths
assertEquals("phonebook.mozilla", // instead of "login"
ActivityStream.extractLabel("https://phonebook.mozilla.org/mellon/login?ReturnTo=https%3A%2F%2Fphonebook.mozilla.org%2F&IdP=http%3A%2F%2Fwww.okta.com", true));
assertEquals("ipay.adp", // instead of "index.jsf"
ActivityStream.extractLabel("https://ipay.adp.com/iPay/index.jsf", true));
assertEquals("calendar.google", // instead of "render"
ActivityStream.extractLabel("https://calendar.google.com/calendar/render?pli=1#main_7", true));
assertEquals("myworkday", // instead of "home.htmld"
ActivityStream.extractLabel("https://www.myworkday.com/vhr_mozilla/d/home.htmld", true));
assertEquals("mail.google", // instead of "1"
ActivityStream.extractLabel("https://mail.google.com/mail/u/1/#inbox", true));
assertEquals("docs.google", // instead of "edit"
ActivityStream.extractLabel("https://docs.google.com/presentation/d/11cyrcwhKTmBdEBIZ3szLO0-_Imrx2CGV2B9_LZHDrds/edit#slide=id.g15d41bb0f3_0_82", true));
assertLabelEquals("phonebook.mozilla","https://phonebook.mozilla.org/mellon/login?ReturnTo=https%3A%2F%2Fphonebook.mozilla.org%2F&IdP=http%3A%2F%2Fwww.okta.com", true);
assertLabelEquals("ipay.adp", "https://ipay.adp.com/iPay/index.jsf", true);
assertLabelEquals("calendar.google", "https://calendar.google.com/calendar/render?pli=1#main_7", true);
assertLabelEquals("myworkday", "https://www.myworkday.com/vhr_mozilla/d/home.htmld", true);
assertLabelEquals("mail.google", "https://mail.google.com/mail/u/1/#inbox", true);
assertLabelEquals("docs.google", "https://docs.google.com/presentation/d/11cyrcwhKTmBdEBIZ3szLO0-_Imrx2CGV2B9_LZHDrds/edit#slide=id.g15d41bb0f3_0_82", true);
// Special cases
assertEquals("irccloud.mozilla",
ActivityStream.extractLabel("https://irccloud.mozilla.com/#!/ircs://irc1.dmz.scl3.mozilla.com:6697/%23universal-search", true));
assertLabelEquals("irccloud.mozilla", "https://irccloud.mozilla.com/#!/ircs://irc1.dmz.scl3.mozilla.com:6697/%23universal-search", true);
}
@Test
public void testExtractLabelWithoutPath() {
assertEquals("addons.mozilla",
ActivityStream.extractLabel("https://addons.mozilla.org/en-US/firefox/", false));
assertLabelEquals("addons.mozilla", "https://addons.mozilla.org/en-US/firefox/", false);
assertLabelEquals("trello", "https://trello.com/b/KX3hV8XS/activity-stream", false);
assertLabelEquals("github", "https://github.com/mozilla/activity-stream", false);
assertLabelEquals("dispatch-news", "https://dispatch-news.herokuapp.com/sidekiq", false);
assertLabelEquals("github", "https://github.com/nchapman/", false);
}
assertEquals("trello",
ActivityStream.extractLabel("https://trello.com/b/KX3hV8XS/activity-stream", false));
private void assertLabelEquals(String expectedLabel, String url, boolean usePath) {
final String[] actualLabel = new String[1];
assertEquals("github",
ActivityStream.extractLabel("https://github.com/mozilla/activity-stream", false));
ActivityStream.LabelCallback callback = new ActivityStream.LabelCallback() {
@Override
public void onLabelExtracted(String label) {
actualLabel[0] = label;
}
};
assertEquals("dispatch-news",
ActivityStream.extractLabel("https://dispatch-news.herokuapp.com/sidekiq", false));
ActivityStream.extractLabel(RuntimeEnvironment.application, url, usePath, callback);
assertEquals("github",
ActivityStream.extractLabel("https://github.com/nchapman/", false));
ShadowLooper.runUiThreadTasks();
assertEquals(expectedLabel, actualLabel[0]);
}
}

Просмотреть файл

@ -4,42 +4,59 @@ import org.junit.Assert;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mozilla.gecko.background.testhelpers.TestRunner;
import org.robolectric.RuntimeEnvironment;
@RunWith(TestRunner.class)
public class TestPublicSuffix {
@Test
public void testStripPublicSuffix() {
// Test empty value
Assert.assertEquals("", PublicSuffix.stripPublicSuffix(""));
Assert.assertEquals("",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, ""));
// Test domains with public suffix
Assert.assertEquals("www.mozilla", PublicSuffix.stripPublicSuffix("www.mozilla.org"));
Assert.assertEquals("www.google", PublicSuffix.stripPublicSuffix("www.google.com"));
Assert.assertEquals("foobar", PublicSuffix.stripPublicSuffix("foobar.blogspot.com"));
Assert.assertEquals("independent", PublicSuffix.stripPublicSuffix("independent.co.uk"));
Assert.assertEquals("biz", PublicSuffix.stripPublicSuffix("biz.com.ua"));
Assert.assertEquals("example", PublicSuffix.stripPublicSuffix("example.org"));
Assert.assertEquals("example", PublicSuffix.stripPublicSuffix("example.pvt.k12.ma.us"));
Assert.assertEquals("www.mozilla",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "www.mozilla.org"));
Assert.assertEquals("www.google",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "www.google.com"));
Assert.assertEquals("foobar",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "foobar.blogspot.com"));
Assert.assertEquals("independent",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "independent.co.uk"));
Assert.assertEquals("biz",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "biz.com.ua"));
Assert.assertEquals("example",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "example.org"));
Assert.assertEquals("example",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "example.pvt.k12.ma.us"));
// Test domain without public suffix
Assert.assertEquals("localhost", PublicSuffix.stripPublicSuffix("localhost"));
Assert.assertEquals("firefox.mozilla", PublicSuffix.stripPublicSuffix("firefox.mozilla"));
Assert.assertEquals("localhost",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "localhost"));
Assert.assertEquals("firefox.mozilla",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "firefox.mozilla"));
// IDN domains
Assert.assertEquals("ουτοπία.δπθ", PublicSuffix.stripPublicSuffix("ουτοπία.δπθ.gr"));
Assert.assertEquals("a网络A", PublicSuffix.stripPublicSuffix("a网络A.网络.Cn"));
Assert.assertEquals("ουτοπία.δπθ",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "ουτοπία.δπθ.gr"));
Assert.assertEquals("a网络A",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "a网络A.网络.Cn"));
// Other non-domain values
Assert.assertEquals("192.168.0.1", PublicSuffix.stripPublicSuffix("192.168.0.1"));
Assert.assertEquals("asdflkj9uahsd", PublicSuffix.stripPublicSuffix("asdflkj9uahsd"));
Assert.assertEquals("192.168.0.1",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "192.168.0.1"));
Assert.assertEquals("asdflkj9uahsd",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "asdflkj9uahsd"));
// Other trailing and other types of dots
Assert.assertEquals("www.mozilla。homeexample", PublicSuffix.stripPublicSuffix("www.mozilla。homeexample。org"));
Assert.assertEquals("example", PublicSuffix.stripPublicSuffix("example.org"));
Assert.assertEquals("www.mozilla。homeexample",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "www.mozilla。homeexample。org"));
Assert.assertEquals("example",
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, "example.org"));
}
@Test(expected = NullPointerException.class)
public void testStripPublicSuffixThrowsException() {
PublicSuffix.stripPublicSuffix(null);
PublicSuffix.stripPublicSuffix(RuntimeEnvironment.application, null);
}
}