diff --git a/package.json b/package.json index a1737ce..0c7e97c 100644 --- a/package.json +++ b/package.json @@ -10,7 +10,7 @@ "export": "node analytics_export/startExport.js", "lint": "eslint analytics_export/", "lint-fix": "eslint --fix analytics_export/", - "pretty": "prettier --write analytics_export", + "pretty": "prettier --write analytics_export/ test/", "test": "mocha" }, "dependencies": { @@ -28,6 +28,8 @@ "eslint-config-prettier": "^6.11.0", "eslint-plugin-import": "^2.21.2", "mocha": "^8.0.1", - "prettier": "^2.0.5" + "prettier": "^2.0.5", + "proxyquire": "^2.1.3", + "sinon": "^9.0.2" } } diff --git a/test/testBigqueryClient.js b/test/testBigqueryClient.js new file mode 100644 index 0000000..51207fc --- /dev/null +++ b/test/testBigqueryClient.js @@ -0,0 +1,196 @@ +"use strict"; + + +const proxyquire = require("proxyquire"); +const { assert } = require("chai"); +import { describe, it } from "mocha"; +const { stub } = require("sinon"); + +const { BigqueryClient } = require("../analytics_export/bigqueryClient") + +describe("BigqueryClient", () => { + describe("factory", () => { + const mockBigqueryClient = (mockDataset) => ( + proxyquire( + "../analytics_export/bigqueryClient.js", + { + "@google-cloud/bigquery": { + BigQuery: stub().callsFake(() => ({ + dataset: stub().returns(mockDataset), + })), + }, + }, + ) + ); + + it("should create the dataset if it does not exist", async () => { + const mockDataset = { + exists: stub().returns([false]), + create: stub(), + }; + + const bigqueryClient = mockBigqueryClient(mockDataset); + + await bigqueryClient.BigqueryClient.createClient( + "fake-project", + "test_app_store", + ); + + assert.isTrue(mockDataset.exists.calledOnce); + assert.isTrue(mockDataset.create.calledOnce); + }); + + it("should skip dataset creation if dataset exists", async () => { + const mockDataset = { + exists: stub().returns([true]), + create: stub(), + }; + + const bigqueryClient = mockBigqueryClient(mockDataset); + + await bigqueryClient.BigqueryClient.createClient( + "fake-project", + "test_app_store", + ); + + assert.isTrue(mockDataset.exists.calledOnce); + assert.isTrue(mockDataset.create.notCalled); + }); + }); + + describe("create table", () => { + const tableName = "tabby"; + + it("should create table if it does not exist", async () => { + const mockTable = { + exists: stub().returns([false]), + }; + const mockDataset = { + table: stub().returns(mockTable), + createTable: stub().returns([tableName]), + }; + const bigqueryClient = new BigqueryClient(mockDataset); + + await bigqueryClient.createTableIfNotExists(tableName, [], ""); + + assert.isTrue(mockTable.exists.calledOnce); + assert.isTrue(mockDataset.table.calledOnce); + assert.isTrue(mockDataset.createTable.calledOnce); + assert.isTrue(mockDataset.createTable.calledWith(tableName)); + }); + + it("should skip table creation if table exists", async () => { + const mockTable = { + exists: stub().returns([true]), + }; + const mockDataset = { + table: stub().returns(mockTable), + createTable: stub().returns([tableName]), + }; + + const bigqueryClient = new BigqueryClient(mockDataset); + + await bigqueryClient.createTableIfNotExists(tableName, [], ""); + + assert.isTrue(mockTable.exists.calledOnce); + assert.isTrue(mockDataset.table.calledOnce); + assert.isTrue(mockDataset.createTable.notCalled); + }); + }); + + describe("write data", () => { + let mockBigqueryClient; + let mockTable; + let mockDataset; + let mockFs; + + beforeEach(() => { + mockTable = { + exists: stub().returns([true]), + load: stub(), + }; + mockDataset = { + table: stub().returns(mockTable), + }; + + mockFs = { + writeFileSync: stub(), + } + + mockBigqueryClient = proxyquire( + "../analytics_export/bigqueryClient.js", + { + fs: mockFs, + tempy: { + file: () => "file.csv", + }, + }, + ); + }); + + it("should give the correct table name based on measure and dimension", async () => { + const bqClient = new mockBigqueryClient.BigqueryClient(mockDataset); + + const tableName = await bqClient.writeData( + "impressionsTotal", + "region", + "2020-07-01", + [], + true, + ); + + assert.equal(tableName, "impressions_by_region"); + }); + + it("should give the correct table name for null dimensions", async () => { + const bqClient = new mockBigqueryClient.BigqueryClient(mockDataset); + + const tableName = await bqClient.writeData( + "impressionsTotal", + null, + "2020-07-01", + [], + true, + ); + + assert.equal(tableName, "impressions_total"); + }); + + it("should write the correct data to a temporary file", async () => { + const bqClient = new mockBigqueryClient.BigqueryClient(mockDataset); + + const data = [ + { date: "2020-07-07", app_name: "Firefox", value: 1, region: "a" }, + { date: "2020-07-07", app_name: "Firefox", value: 2, region: "a" }, + ] + + const tableName = await bqClient.writeData( + "impressionsTotal", + "region", + "2020-07-01", + data, + true, + ); + + const writtenData = "2020-07-07\tFirefox\t1\ta\n2020-07-07\tFirefox\t2\ta"; + assert.isTrue(mockFs.writeFileSync.calledWith("file.csv", writtenData)); + }); + + it("should write to a partition of the table", async () => { + const bqClient = new mockBigqueryClient.BigqueryClient(mockDataset); + bqClient.createTableIfNotExists = stub(); + + const tableName = await bqClient.writeData( + "impressionsTotal", + "region", + "2020-07-01", + [], + true, + ); + + assert.isTrue(mockDataset.table.calledOnce); + assert.isTrue(mockDataset.table.calledOnceWithExactly(`${tableName}$20200701`)); + }); + + }); +}); diff --git a/test/testExport.js b/test/testExport.js index 50b2fc2..507d2d6 100644 --- a/test/testExport.js +++ b/test/testExport.js @@ -1,13 +1,10 @@ "use strict"; +import { describe, it } from "mocha"; + const analyticsExport = require("../analytics_export/analyticsExport.js"); -const chai = require("chai"); - -chai.should(); - -describe("Export Functions", function () { - it("should pass test", function () { - analyticsExport.startExport().should.equal("test"); - }); +describe("getAllowedDimensionsPerMeasure", () => { + it("should fail if given date before data start date", () => { + }); }); diff --git a/yarn.lock b/yarn.lock index a9335cc..238f421 100644 --- a/yarn.lock +++ b/yarn.lock @@ -73,6 +73,42 @@ resolved "https://registry.yarnpkg.com/@google-cloud/promisify/-/promisify-2.0.1.tgz#79f722463a5779197267c6870362b1d7927081f7" integrity sha512-82EQzwrNauw1fkbUSr3f+50Bcq7g4h0XvLOk8C5e9ABkXYHei7ZPi9tiMMD7Vh3SfcdH97d1ibJ3KBWp2o1J+w== +"@sinonjs/commons@^1", "@sinonjs/commons@^1.6.0", "@sinonjs/commons@^1.7.0", "@sinonjs/commons@^1.7.2": + version "1.8.0" + resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.0.tgz#c8d68821a854c555bba172f3b06959a0039b236d" + integrity sha512-wEj54PfsZ5jGSwMX68G8ZXFawcSglQSXqCftWX3ec8MDUzQdHgcKvw97awHbY0efQEL5iKUOAmmVtoYgmrSG4Q== + dependencies: + type-detect "4.0.8" + +"@sinonjs/fake-timers@^6.0.0", "@sinonjs/fake-timers@^6.0.1": + version "6.0.1" + resolved "https://registry.yarnpkg.com/@sinonjs/fake-timers/-/fake-timers-6.0.1.tgz#293674fccb3262ac782c7aadfdeca86b10c75c40" + integrity sha512-MZPUxrmFubI36XS1DI3qmI0YdN1gks62JtFZvxR67ljjSNCeK6U08Zx4msEWOXuofgqUt6zPHSi1H9fbjR/NRA== + dependencies: + "@sinonjs/commons" "^1.7.0" + +"@sinonjs/formatio@^5.0.1": + version "5.0.1" + resolved "https://registry.yarnpkg.com/@sinonjs/formatio/-/formatio-5.0.1.tgz#f13e713cb3313b1ab965901b01b0828ea6b77089" + integrity sha512-KaiQ5pBf1MpS09MuA0kp6KBQt2JUOQycqVG1NZXvzeaXe5LGFqAKueIS0bw4w0P9r7KuBSVdUk5QjXsUdu2CxQ== + dependencies: + "@sinonjs/commons" "^1" + "@sinonjs/samsam" "^5.0.2" + +"@sinonjs/samsam@^5.0.2", "@sinonjs/samsam@^5.0.3": + version "5.0.3" + resolved "https://registry.yarnpkg.com/@sinonjs/samsam/-/samsam-5.0.3.tgz#86f21bdb3d52480faf0892a480c9906aa5a52938" + integrity sha512-QucHkc2uMJ0pFGjJUDP3F9dq5dx8QIaqISl9QgwLOh6P9yv877uONPGXh/OH/0zmM3tW1JjuJltAZV2l7zU+uQ== + dependencies: + "@sinonjs/commons" "^1.6.0" + lodash.get "^4.4.2" + type-detect "^4.0.8" + +"@sinonjs/text-encoding@^0.7.1": + version "0.7.1" + resolved "https://registry.yarnpkg.com/@sinonjs/text-encoding/-/text-encoding-0.7.1.tgz#8da5c6530915653f3a1f38fd5f101d8c3f8079c5" + integrity sha512-+iTbntw2IZPb/anVDbypzfQa+ay64MW0Zo8aJ8gZPWMMK6/OubMVb6lUPMagqjOPnmtauXnFCACVl3O7ogjeqQ== + "@tootallnate/once@1": version "1.1.2" resolved "https://registry.yarnpkg.com/@tootallnate/once/-/once-1.1.2.tgz#ccb91445360179a04e7fe6aff78c00ffc1eeaf82" @@ -512,7 +548,7 @@ delayed-stream@~1.0.0: resolved "https://registry.yarnpkg.com/delayed-stream/-/delayed-stream-1.0.0.tgz#df3ae199acadfb7d440aaae0b29e2272b24ec619" integrity sha1-3zrhmayt+31ECqrgsp4icrJOxhk= -diff@4.0.2: +diff@4.0.2, diff@^4.0.2: version "4.0.2" resolved "https://registry.yarnpkg.com/diff/-/diff-4.0.2.tgz#60f3aecb89d5fae520c11aa19efc2bb982aade7d" integrity sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A== @@ -845,6 +881,14 @@ file-entry-cache@^5.0.1: dependencies: flat-cache "^2.0.1" +fill-keys@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/fill-keys/-/fill-keys-1.0.2.tgz#9a8fa36f4e8ad634e3bf6b4f3c8882551452eb20" + integrity sha1-mo+jb06K1jTjv2tPPIiCVRRS6yA= + dependencies: + is-object "~1.0.1" + merge-descriptors "~1.0.0" + fill-range@^7.0.1: version "7.0.1" resolved "https://registry.yarnpkg.com/fill-range/-/fill-range-7.0.1.tgz#1919a6a7c75fe38b2c7c77e5198535da9acdda40" @@ -1204,6 +1248,11 @@ is-number@^7.0.0: resolved "https://registry.yarnpkg.com/is-number/-/is-number-7.0.0.tgz#7535345b896734d5f80c4d06c50955527a14f12b" integrity sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng== +is-object@~1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/is-object/-/is-object-1.0.1.tgz#8952688c5ec2ffd6b03ecc85e769e02903083470" + integrity sha1-iVJojF7C/9awPsyF52ngKQMINHA= + is-regex@^1.1.0: version "1.1.0" resolved "https://registry.yarnpkg.com/is-regex/-/is-regex-1.1.0.tgz#ece38e389e490df0dc21caea2bd596f987f767ff" @@ -1243,6 +1292,11 @@ is@^3.3.0: resolved "https://registry.yarnpkg.com/is/-/is-3.3.0.tgz#61cff6dd3c4193db94a3d62582072b44e5645d79" integrity sha512-nW24QBoPcFGGHJGUwnfpI7Yc5CdqWNdsyHQszVE/z2pKHXzh7FZ5GWhJqSyaQ9wMkQnsTx+kAI8bHlCX4tKdbg== +isarray@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/isarray/-/isarray-0.0.1.tgz#8a18acfca9a8f4177e09abfc6038939b05d1eedf" + integrity sha1-ihis/Kmo9Bd+Cav8YDiTmwXR7t8= + isarray@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" @@ -1355,6 +1409,11 @@ jsprim@^1.2.2: json-schema "0.2.3" verror "1.10.0" +just-extend@^4.0.2: + version "4.1.0" + resolved "https://registry.yarnpkg.com/just-extend/-/just-extend-4.1.0.tgz#7278a4027d889601640ee0ce0e5a00b992467da4" + integrity sha512-ApcjaOdVTJ7y4r08xI5wIqpvwS48Q0PBG4DJROcEkH1f8MdAiNFyFxz3xoL0LWAVwjrwPYZdVHHxhRHcx/uGLA== + jwa@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/jwa/-/jwa-2.0.0.tgz#a7e9c3f29dae94027ebcaf49975c9345593410fc" @@ -1413,6 +1472,11 @@ locate-path@^5.0.0: dependencies: p-locate "^4.1.0" +lodash.get@^4.4.2: + version "4.4.2" + resolved "https://registry.yarnpkg.com/lodash.get/-/lodash.get-4.4.2.tgz#2d177f652fa31e939b4438d5341499dfa3825e99" + integrity sha1-LRd/ZS+jHpObRDjVNBSZ36OCXpk= + lodash@^4.17.14, lodash@^4.17.15: version "4.17.15" resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.15.tgz#b447f6670a0455bbfeedd11392eff330ea097548" @@ -1432,6 +1496,11 @@ lru-cache@^5.0.0: dependencies: yallist "^3.0.2" +merge-descriptors@~1.0.0: + version "1.0.1" + resolved "https://registry.yarnpkg.com/merge-descriptors/-/merge-descriptors-1.0.1.tgz#b00aaa556dd8b44568150ec9d1b953f3f90cbb61" + integrity sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E= + mime-db@1.44.0: version "1.44.0" resolved "https://registry.yarnpkg.com/mime-db/-/mime-db-1.44.0.tgz#fa11c5eb0aca1334b4233cb4d52f10c5a6272f92" @@ -1499,6 +1568,11 @@ mocha@^8.0.1: yargs-parser "13.1.2" yargs-unparser "1.6.0" +module-not-found-error@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/module-not-found-error/-/module-not-found-error-1.0.1.tgz#cf8b4ff4f29640674d6cdd02b0e3bc523c2bbdc0" + integrity sha1-z4tP9PKWQGdNbN0CsOO8UjwrvcA= + moment@^2.24.0: version "2.27.0" resolved "https://registry.yarnpkg.com/moment/-/moment-2.27.0.tgz#8bff4e3e26a236220dfe3e36de756b6ebaa0105d" @@ -1519,6 +1593,17 @@ natural-compare@^1.4.0: resolved "https://registry.yarnpkg.com/natural-compare/-/natural-compare-1.4.0.tgz#4abebfeed7541f2c27acfb29bdbbd15c8d5ba4f7" integrity sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc= +nise@^4.0.1: + version "4.0.4" + resolved "https://registry.yarnpkg.com/nise/-/nise-4.0.4.tgz#d73dea3e5731e6561992b8f570be9e363c4512dd" + integrity sha512-bTTRUNlemx6deJa+ZyoCUTRvH3liK5+N6VQZ4NIw90AgDXY6iPnsqplNFf6STcj+ePk0H/xqxnP75Lr0J0Fq3A== + dependencies: + "@sinonjs/commons" "^1.7.0" + "@sinonjs/fake-timers" "^6.0.0" + "@sinonjs/text-encoding" "^0.7.1" + just-extend "^4.0.2" + path-to-regexp "^1.7.0" + node-fetch@^2.2.0, node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" @@ -1710,6 +1795,13 @@ path-parse@^1.0.6: resolved "https://registry.yarnpkg.com/path-parse/-/path-parse-1.0.6.tgz#d62dbb5679405d72c4737ec58600e9ddcf06d24c" integrity sha512-GSmOT2EbHrINBf9SR7CDELwlJ8AENk3Qn7OikK4nFYAu3Ote2+JYNVvkpAEQm3/TLNEJFD/xZJjzyxg3KBWOzw== +path-to-regexp@^1.7.0: + version "1.8.0" + resolved "https://registry.yarnpkg.com/path-to-regexp/-/path-to-regexp-1.8.0.tgz#887b3ba9d84393e87a0a0b9f4cb756198b53548a" + integrity sha512-n43JRhlUKUAlibEJhPeir1ncUID16QnEjNpwzNdO3Lm4ywrBpBZ5oLD0I6br9evr1Y9JTqwRtAh7JLoOzAQdVA== + dependencies: + isarray "0.0.1" + path-type@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/path-type/-/path-type-2.0.0.tgz#f012ccb8415b7096fc2daa1054c3d72389594c73" @@ -1770,6 +1862,15 @@ promise.allsettled@1.0.2: function-bind "^1.1.1" iterate-value "^1.0.0" +proxyquire@^2.1.3: + version "2.1.3" + resolved "https://registry.yarnpkg.com/proxyquire/-/proxyquire-2.1.3.tgz#2049a7eefa10a9a953346a18e54aab2b4268df39" + integrity sha512-BQWfCqYM+QINd+yawJz23tbBM40VIGXOdDw3X344KcclI/gtBbdWF6SlQ4nK/bYhF9d27KYug9WzljHC6B9Ysg== + dependencies: + fill-keys "^1.0.2" + module-not-found-error "^1.0.1" + resolve "^1.11.1" + psl@^1.1.28: version "1.8.0" resolved "https://registry.yarnpkg.com/psl/-/psl-1.8.0.tgz#9326f8bcfb013adcc005fdff056acce020e51c24" @@ -1880,7 +1981,7 @@ resolve-from@^4.0.0: resolved "https://registry.yarnpkg.com/resolve-from/-/resolve-from-4.0.0.tgz#4abcd852ad32dd7baabfe9b40e00a36db5f392e6" integrity sha512-pb/MYmXstAkysRFx8piNI1tGFNQIFA3vkE3Gq4EuA1dF6gHp/+vgZqsCGJapvy8N3Q+4o7FwvquPJcnZ7RYy4g== -resolve@^1.10.0, resolve@^1.13.1, resolve@^1.17.0: +resolve@^1.10.0, resolve@^1.11.1, resolve@^1.13.1, resolve@^1.17.0: version "1.17.0" resolved "https://registry.yarnpkg.com/resolve/-/resolve-1.17.0.tgz#b25941b54968231cc2d1bb76a79cb7f2c0bf8444" integrity sha512-ic+7JYiV8Vi2yzQGFWOkiZD5Z9z7O2Zhm9XMaTxdJExKasieFCr+yXZ/WmXsckHiKl12ar0y6XiXDx3m4RHn1w== @@ -1944,6 +2045,19 @@ shebang-regex@^3.0.0: resolved "https://registry.yarnpkg.com/shebang-regex/-/shebang-regex-3.0.0.tgz#ae16f1644d873ecad843b0307b143362d4c42172" integrity sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A== +sinon@^9.0.2: + version "9.0.2" + resolved "https://registry.yarnpkg.com/sinon/-/sinon-9.0.2.tgz#b9017e24633f4b1c98dfb6e784a5f0509f5fd85d" + integrity sha512-0uF8Q/QHkizNUmbK3LRFqx5cpTttEVXudywY9Uwzy8bTfZUhljZ7ARzSxnRHWYWtVTeh4Cw+tTb3iU21FQVO9A== + dependencies: + "@sinonjs/commons" "^1.7.2" + "@sinonjs/fake-timers" "^6.0.1" + "@sinonjs/formatio" "^5.0.1" + "@sinonjs/samsam" "^5.0.3" + diff "^4.0.2" + nise "^4.0.1" + supports-color "^7.1.0" + slice-ansi@^2.1.0: version "2.1.0" resolved "https://registry.yarnpkg.com/slice-ansi/-/slice-ansi-2.1.0.tgz#cacd7693461a637a5788d92a7dd4fba068e81636" @@ -2213,7 +2327,7 @@ type-check@^0.4.0, type-check@~0.4.0: dependencies: prelude-ls "^1.2.1" -type-detect@^4.0.0, type-detect@^4.0.5: +type-detect@4.0.8, type-detect@^4.0.0, type-detect@^4.0.5, type-detect@^4.0.8: version "4.0.8" resolved "https://registry.yarnpkg.com/type-detect/-/type-detect-4.0.8.tgz#7646fb5f18871cfbb7749e69bd39a6388eb7450c" integrity sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==