From 717e7c409726993ff3a3fe666cfcee47b182edf8 Mon Sep 17 00:00:00 2001
From: Ricky Rosario <rickyrosario@gmail.com>
Date: Tue, 19 Sep 2017 06:54:12 -0400
Subject: [PATCH] feat (sections): #3423 implement deduping for sections

---
 system-addon/lib/SectionsManager.jsm          | 33 ++++++++++++++++---
 .../test/unit/lib/SectionsManager.test.js     | 23 +++++++++++++
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/system-addon/lib/SectionsManager.jsm b/system-addon/lib/SectionsManager.jsm
index 59badc13c..b13d85315 100644
--- a/system-addon/lib/SectionsManager.jsm
+++ b/system-addon/lib/SectionsManager.jsm
@@ -4,9 +4,10 @@
 "use strict";
 
 const {utils: Cu} = Components;
+Cu.import("resource://gre/modules/EventEmitter.jsm");
 Cu.import("resource://gre/modules/Services.jsm");
 const {actionCreators: ac, actionTypes: at} = Cu.import("resource://activity-stream/common/Actions.jsm", {});
-Cu.import("resource://gre/modules/EventEmitter.jsm");
+const {Dedupe} = Cu.import("resource://activity-stream/common/Dedupe.jsm", {});
 
 /*
  * Generators for built in sections, keyed by the pref name for their feed.
@@ -36,7 +37,8 @@ const BUILT_IN_SECTIONS = {
       icon: "check"
     },
     shouldSendImpressionStats: true,
-    order: 0
+    order: 0,
+    dedupeFrom: ["highlights"]
   }),
   "feeds.section.highlights": options => ({
     id: "highlights",
@@ -73,6 +75,8 @@ const SectionsManager = {
     Object.keys(this.CONTEXT_MENU_PREFS).forEach(k =>
       Services.prefs.addObserver(this.CONTEXT_MENU_PREFS[k], this));
 
+    this.dedupe = new Dedupe(site => site && site.url);
+
     this.initialized = true;
     this.emit(this.INIT);
   },
@@ -123,10 +127,31 @@ const SectionsManager = {
     this.updateSectionContextMenuOptions(options);
 
     if (this.sections.has(id)) {
-      this.sections.set(id, Object.assign(this.sections.get(id), options));
-      this.emit(this.UPDATE_SECTION, id, options, shouldBroadcast);
+      const dedupedOptions = this.dedupeRows(id, options);
+      this.sections.set(id, Object.assign(this.sections.get(id), dedupedOptions));
+      this.emit(this.UPDATE_SECTION, id, dedupedOptions, shouldBroadcast);
+
+      // Update any sections that dedupe from the updated section
+      this.sections.forEach(section => {
+        if (section.dedupeFrom && section.dedupeFrom.includes(id)) {
+          this.updateSection(section.id, section, shouldBroadcast);
+        }
+      });
     }
   },
+  dedupeRows(id, options) {
+    const dedupeFrom = this.sections.get(id).dedupeFrom;
+    if (dedupeFrom && dedupeFrom.length > 0 && options.rows) {
+      for (const sectionId of dedupeFrom) {
+        const section = this.sections.get(sectionId);
+        if (section && section.rows) {
+          const [, newRows] = this.dedupe.group(section.rows, options.rows);
+          options.rows = newRows;
+        }
+      }
+    }
+    return options;
+  },
 
   /**
    * Sets the section's context menu options. These are all available context menu
diff --git a/system-addon/test/unit/lib/SectionsManager.test.js b/system-addon/test/unit/lib/SectionsManager.test.js
index 7487857e3..71ef8efa6 100644
--- a/system-addon/test/unit/lib/SectionsManager.test.js
+++ b/system-addon/test/unit/lib/SectionsManager.test.js
@@ -146,6 +146,7 @@ describe("SectionsManager", () => {
     });
     it("should update all sections", () => {
       SectionsManager.sections.clear();
+      const updateSectionOrig = SectionsManager.updateSection;
       SectionsManager.updateSection = sinon.spy();
 
       SectionsManager.addSection("ID1", {title: "FAKE_TITLE_1"});
@@ -155,6 +156,7 @@ describe("SectionsManager", () => {
       assert.calledTwice(SectionsManager.updateSection);
       assert.calledWith(SectionsManager.updateSection, "ID1", {title: "FAKE_TITLE_1"}, true);
       assert.calledWith(SectionsManager.updateSection, "ID2", {title: "FAKE_TITLE_2"}, true);
+      SectionsManager.updateSection = updateSectionOrig;
     });
     it("context menu pref change should update sections", () => {
       let observer;
@@ -203,6 +205,27 @@ describe("SectionsManager", () => {
       assert.notCalled(spy);
     });
   });
+  describe("#dedupe", () => {
+    it("should dedupe stories from highlights", () => {
+      SectionsManager.init();
+      // Add some rows to highlights
+      SectionsManager.updateSection("highlights", {rows: [{url: "https://highlight.com/abc"}, {url: "https://shared.com/def"}]});
+      // Add some rows to top stories
+      SectionsManager.updateSection("topstories", {rows: [{url: "https://topstory.com/ghi"}, {url: "https://shared.com/def"}]});
+      // Verify deduping
+      assert.deepEqual(SectionsManager.sections.get("topstories").rows, [{url: "https://topstory.com/ghi"}]);
+    });
+    it("should dedupe stories from highlights when updating highlights", () => {
+      SectionsManager.init();
+      // Add some rows to top stories
+      SectionsManager.updateSection("topstories", {rows: [{url: "https://topstory.com/ghi"}, {url: "https://shared.com/def"}]});
+      assert.deepEqual(SectionsManager.sections.get("topstories").rows, [{url: "https://topstory.com/ghi"}, {url: "https://shared.com/def"}]);
+      // Add some rows to highlights
+      SectionsManager.updateSection("highlights", {rows: [{url: "https://highlight.com/abc"}, {url: "https://shared.com/def"}]});
+      // Verify deduping
+      assert.deepEqual(SectionsManager.sections.get("topstories").rows, [{url: "https://topstory.com/ghi"}]);
+    });
+  });
 });
 
 describe("SectionsFeed", () => {