Fixes bug 1518921 - Discovery Stream Spocs (#4649)

* Fixes bug 1518921 - Discovery Stream Spocs

common/Actions.jsm

@@ -45,6 +45,8 @@ for (const type of [
   "DISCOVERY_STREAM_FEEDS_UPDATE",
   "DISCOVERY_STREAM_LAYOUT_RESET",
   "DISCOVERY_STREAM_LAYOUT_UPDATE",
+  "DISCOVERY_STREAM_SPOCS_ENDPOINT",
+  "DISCOVERY_STREAM_SPOCS_UPDATE",
   "DOWNLOAD_CHANGED",
   "FILL_SEARCH_TERM",
   "FOCUS_SEARCH",

common/Reducers.jsm

@@ -56,6 +56,11 @@ const INITIAL_STATE = {
     feeds: {
       // "https://foo.com/feed1": {lastUpdated: 123, data: []}
     },
+    spocs: {
+      spocs_endpoint: "",
+      lastUpdated: null,
+      data: [],
+    },
   },
   Search: {
     // Pretend the search box is focused after handing off to AwesomeBar.
@@ -457,6 +462,26 @@ function DiscoveryStream(prevState = INITIAL_STATE.DiscoveryStream, action) {
       return {...prevState, lastUpdated: INITIAL_STATE.DiscoveryStream.lastUpdated, layout: INITIAL_STATE.DiscoveryStream.layout};
     case at.DISCOVERY_STREAM_FEEDS_UPDATE:
       return {...prevState, feeds: action.data || prevState.feeds};
+    case at.DISCOVERY_STREAM_SPOCS_ENDPOINT:
+      return {
+        ...prevState,
+        spocs: {
+          ...INITIAL_STATE.DiscoveryStream.spocs,
+          spocs_endpoint: action.data || INITIAL_STATE.DiscoveryStream.spocs.spocs_endpoint,
+        },
+      };
+    case at.DISCOVERY_STREAM_SPOCS_UPDATE:
+      if (action.data) {
+        return {
+          ...prevState,
+          spocs: {
+            ...prevState.spocs,
+            lastUpdated: action.data.lastUpdated,
+            data: action.data.spocs,
+          },
+        };
+      }
+      return prevState;
     default:
       return prevState;
   }

lib/DiscoveryStreamFeed.jsm

@@ -13,6 +13,7 @@ const {PersistentCache} = ChromeUtils.import("resource://activity-stream/lib/Per
 const CACHE_KEY = "discovery_stream";
 const LAYOUT_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
 const COMPONENT_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
+const SPOCS_FEEDS_UPDATE_TIME = 30 * 60 * 1000; // 30 minutes
 const CONFIG_PREF_NAME = "browser.newtabpage.activity-stream.discoverystream.config";
 
 this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
@@ -81,6 +82,28 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
     return null;
   }
 
+  async fetchSpocs() {
+    const {DiscoveryStream} = this.store.getState();
+    const endpoint = DiscoveryStream.spocs.spocs_endpoint;
+    if (!endpoint) {
+      Cu.reportError("No endpoint configured for pocket, so could not fetch spocs");
+      return null;
+    }
+    try {
+      const response = await fetch(endpoint, {credentials: "omit"});
+      if (!response.ok) {
+        // istanbul ignore next
+        throw new Error(`Spocs endpoint returned unexpected status: ${response.status}`);
+      }
+      return response.json();
+    } catch (error) {
+      // istanbul ignore next
+      Cu.reportError(`Failed to fetch spocs: ${error.message}`);
+    }
+    // istanbul ignore next
+    return null;
+  }
+
   async loadLayout() {
     const cachedData = await this.cache.get() || {};
     let {layout: layoutResponse} = cachedData;
@@ -103,6 +126,12 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
         },
       }));
     }
+    if (layoutResponse && layoutResponse.spocs && layoutResponse.spocs.url) {
+      this.store.dispatch(ac.BroadcastToContent({
+        type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT,
+        data: layoutResponse.spocs.url,
+      }));
+    }
   }
 
   async loadComponentFeeds() {
@@ -126,6 +155,33 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
     }
   }
 
+  async loadSpocs() {
+    const cachedData = await this.cache.get() || {};
+    let {spocs} = cachedData;
+    if (!spocs || !(Date.now() - spocs.lastUpdated < SPOCS_FEEDS_UPDATE_TIME)) {
+      const spocsResponse = await this.fetchSpocs();
+      if (spocsResponse) {
+        spocs = {
+          lastUpdated: Date.now(),
+          data: spocsResponse,
+        };
+        await this.cache.set("spocs", spocs);
+      } else {
+        Cu.reportError("No response for spocs_endpoint prop");
+      }
+    }
+
+    if (spocs) {
+      this.store.dispatch(ac.BroadcastToContent({
+        type: at.DISCOVERY_STREAM_SPOCS_UPDATE,
+        data: {
+          lastUpdated: spocs.lastUpdated,
+          spocs: spocs.data,
+        },
+      }));
+    }
+  }
+
   async getComponentFeed(feedUrl) {
     const cachedData = await this.cache.get() || {};
     const {feeds} = cachedData;
@@ -164,6 +220,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
   async enable() {
     await this.loadLayout();
     await this.loadComponentFeeds();
+    await this.loadSpocs();
     this.loaded = true;
   }
 
@@ -177,6 +234,7 @@ this.DiscoveryStreamFeed = class DiscoveryStreamFeed {
   async clearCache() {
     await this.cache.set("layout", {});
     await this.cache.set("feeds", {});
+    await this.cache.set("spocs", {});
   }
 
   async onPrefChange() {

test/unit/common/Reducers.test.js

@@ -668,6 +668,27 @@ describe("Reducers", () => {
       const state = DiscoveryStream(undefined, {type: at.DISCOVERY_STREAM_CONFIG_CHANGE, data: {enabled: true}});
       assert.deepEqual(state.config, {enabled: true});
     });
+    it("should set spoc_endpoint with DISCOVERY_STREAM_SPOCS_ENDPOINT", () => {
+      const state = DiscoveryStream(undefined, {type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT, data: "foo.com"});
+      assert.equal(state.spocs.spocs_endpoint, "foo.com");
+    });
+    it("should set spocs with DISCOVERY_STREAM_SPOCS_UPDATE", () => {
+      const data = {
+        lastUpdated: 123,
+        spocs: [1, 2, 3],
+      };
+      const state = DiscoveryStream(undefined, {type: at.DISCOVERY_STREAM_SPOCS_UPDATE, data});
+      assert.deepEqual(state.spocs, {
+        spocs_endpoint: "",
+        data: [1, 2, 3],
+        lastUpdated: 123,
+      });
+    });
+    it("should handle no data from DISCOVERY_STREAM_SPOCS_UPDATE", () => {
+      const data = null;
+      const state = DiscoveryStream(undefined, {type: at.DISCOVERY_STREAM_SPOCS_UPDATE, data});
+      assert.deepEqual(state.spocs, INITIAL_STATE.DiscoveryStream.spocs);
+    });
   });
   describe("Search", () => {
     it("should return INITIAL_STATE by default", () => {

test/unit/lib/DiscoveryStreamFeed.test.js

@@ -1,5 +1,5 @@
+import {actionCreators as ac, actionTypes as at} from "common/Actions.jsm";
 import {combineReducers, createStore} from "redux";
-import {actionTypes as at} from "common/Actions.jsm";
 import {DiscoveryStreamFeed} from "lib/DiscoveryStreamFeed.jsm";
 import {reducers} from "common/Reducers.jsm";
 
@@ -87,6 +87,18 @@ describe("DiscoveryStreamFeed", () => {
       assert.notCalled(fetchStub);
       assert.notCalled(feed.cache.set);
     });
+    it("should set spocs_endpoint from layout", async () => {
+      const resp = {layout: ["foo", "bar"], spocs: {url: "foo.com"}};
+      const fakeCache = {};
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve(fakeCache));
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+
+      fetchStub.resolves({ok: true, json: () => Promise.resolve(resp)});
+
+      await feed.loadLayout();
+
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.spocs_endpoint, "foo.com");
+    });
   });
 
   describe("#loadComponentFeeds", () => {
@@ -113,7 +125,7 @@ describe("DiscoveryStreamFeed", () => {
 
       const feedResp = await feed.getComponentFeed("foo.com");
 
-      assert.deepEqual(feedResp.data, "data");
+      assert.equal(feedResp.data, "data");
     });
     it("should fetch fresh data if cache is old", async () => {
       const fakeCache = {feeds: {"foo.com": {lastUpdated: Date.now()}}};
@@ -160,17 +172,89 @@ describe("DiscoveryStreamFeed", () => {
     });
   });
 
+  describe("#loadSpocs", () => {
+    it("should fetch fresh data if cache is empty", async () => {
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve());
+      sandbox.stub(feed, "fetchSpocs").returns(Promise.resolve("data"));
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+
+      await feed.loadSpocs();
+
+      assert.calledWith(feed.cache.set, "spocs", {"data": "data", "lastUpdated": 0});
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.data, "data");
+    });
+    it("should fetch fresh data if cache is old", async () => {
+      const cachedSpoc = {"data": "old", "lastUpdated": Date.now()};
+      const cachedData = {"spocs": cachedSpoc};
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve(cachedData));
+      sandbox.stub(feed, "fetchSpocs").returns(Promise.resolve("new"));
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+      clock.tick(THIRTY_MINUTES + 1);
+
+      await feed.loadSpocs();
+
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.data, "new");
+    });
+    it("should return data from cache if it is fresh", async () => {
+      const cachedSpoc = {"data": "old", "lastUpdated": Date.now()};
+      const cachedData = {"spocs": cachedSpoc};
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve(cachedData));
+      sandbox.stub(feed, "fetchSpocs").returns(Promise.resolve("new"));
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+      clock.tick(THIRTY_MINUTES - 1);
+
+      await feed.loadSpocs();
+
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.data, "old");
+    });
+  });
+
+  describe("#fetchSpocs", () => {
+    it("should return old spocs if fetch failed", async () => {
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+      feed.store.dispatch(ac.BroadcastToContent({
+        type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT,
+        data: "foo.com",
+      }));
+      fetchStub.resolves({ok: false, json: () => Promise.resolve({})});
+      const fakeCache = {spocs: {lastUpdated: Date.now(), data: "old data"}};
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve(fakeCache));
+      clock.tick(THIRTY_MINUTES + 1);
+
+      await feed.loadSpocs();
+
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.data, "old data");
+    });
+    it("should return new spocs if fetch succeeds", async () => {
+      sandbox.stub(feed.cache, "set").returns(Promise.resolve());
+      feed.store.dispatch(ac.BroadcastToContent({
+        type: at.DISCOVERY_STREAM_SPOCS_ENDPOINT,
+        data: "foo.com",
+      }));
+      fetchStub.resolves({ok: true, json: () => Promise.resolve("new data")});
+      const fakeCache = {spocs: {lastUpdated: Date.now(), data: "old data"}};
+      sandbox.stub(feed.cache, "get").returns(Promise.resolve(fakeCache));
+      clock.tick(THIRTY_MINUTES + 1);
+
+      await feed.loadSpocs();
+
+      assert.equal(feed.store.getState().DiscoveryStream.spocs.data, "new data");
+    });
+  });
+
   describe("#clearCache", () => {
-    it("should set .layout and .feeds to {}", async () => {
+    it("should set .layout, .feeds and .spocs to {}", async () => {
       sandbox.stub(feed.cache, "set").returns(Promise.resolve());
 
       await feed.clearCache();
 
-      assert.calledTwice(feed.cache.set);
+      assert.calledThrice(feed.cache.set);
       const {firstCall} = feed.cache.set;
       const {secondCall} = feed.cache.set;
+      const {thirdCall} = feed.cache.set;
       assert.deepEqual(firstCall.args, ["layout", {}]);
       assert.deepEqual(secondCall.args, ["feeds", {}]);
+      assert.deepEqual(thirdCall.args, ["spocs", {}]);
     });
   });