From e0f2782c8b605eb8cd7867b2588f51111ab7ba26 Mon Sep 17 00:00:00 2001 From: Claire Kuang Date: Thu, 7 Oct 2021 12:01:26 +0100 Subject: [PATCH 1/6] removes closures and datachunks --- Speckle/Speckle.pq | 7 ++++++- Speckle/Speckle.query.pq | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index 005b036..b131dec 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -143,7 +143,12 @@ Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) => #"JSON" = Json.Document(Source), objects = if (IsCommitObject) then #"JSON"[data][stream][object][children][objects] else {#"JSON"[data][stream][object][data]}, - objectsTable = Table.FromRecords(objects) + + // remove closures from records, and remove DataChunk records + removeClosureField = List.Transform(objects, each Record.RemoveFields(_, "__closure", MissingField.Ignore)), + removeDatachunkRecords = List.RemoveItems(removeClosureField, List.FindText(removeClosureField, "Speckle.Core.Models.DataChunk")), + + objectsTable = Table.FromRecords(removeDatachunkRecords) in objectsTable; diff --git a/Speckle/Speckle.query.pq b/Speckle/Speckle.query.pq index 204e494..41325ae 100644 --- a/Speckle/Speckle.query.pq +++ b/Speckle/Speckle.query.pq @@ -1,5 +1,5 @@ // Use this file to write queries to test your data connector let - result = Speckle.Contents("https://speckle.xyz/streams/5dfbeb49c9/objects/ed4748572b27cfe008f2592a44ab85f1") + result = Speckle.Contents("https://speckle.xyz/streams/5b97b37b8b") in result \ No newline at end of file From 9e5c204c8f666b024230c07188ebdaa6ea44b4e9 Mon Sep 17 00:00:00 2001 From: Alan Rynne Date: Thu, 7 Oct 2021 18:51:26 +0200 Subject: [PATCH 2/6] metrics: First attempt --- Speckle/Speckle.pq | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index b131dec..c411bad 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -69,6 +69,18 @@ shared CommitTable = (url) as table => in commitTable; +Speckle.LogToMatomo = () => + let + Source = Web.Contents("https://rynne.matomo.cloud/matomo.php?idsite=1&rec=1&apiv=1&uid=powerbiuser&action_name=receive%2Fmanual&url=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&urlref=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&_cvar=%7B%22hostApplication%22:%20%22PowerBI%22%7D&e_c=Power%20BI&e_a=receive%2Fmanual", + [ + Headers=[ + #"Method"="POST" + ], + Content=Text.ToBinary("") + ]) + in + Source; + Speckle.GetObjectFromStream = (server, streamId) => let branchName = "main", @@ -128,6 +140,7 @@ shared Speckle.GetObjectFromCommit = (server, streamId, commitId) => Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) => let + Matomo = Speckle.LogToMatomo(), query = if (IsCommitObject) then "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { children { objects { data } } } } }""}" else "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { data } } }""}", Source = Web.Contents( From 70eb1059d2bad7cc33e76151ddcbe71deb3c0a51 Mon Sep 17 00:00:00 2001 From: Alan Rynne Date: Thu, 7 Oct 2021 19:29:14 +0200 Subject: [PATCH 3/6] metrics: second attempt --- Speckle/Speckle.pq | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index c411bad..2591873 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -48,14 +48,16 @@ Speckle.GetBranchAsList = (branchRecord) => in list; */ - [DataSource.Kind="Speckle", Publish="Speckle.Publish"] shared Speckle.Contents = Value.ReplaceType(CommitTable, type function (StreamUrl as Uri.Type) as any); +/* INFO: Variables will not be instantiated (or any code run) until they are used */ shared CommitTable = (url) as table => let // Get server and streamId, and branchName / commitId / objectid from the input url - server = Text.Combine({"https://", Uri.Parts(url)[Host]}), + s = Text.Combine({"https://", Uri.Parts(url)[Host]}), + server = Speckle.LogToMatomo(s), + segments = Text.Split(Text.AfterDelimiter(Uri.Parts(url)[Path], "/", 0), "/"), streamId = segments{1}, branchName = if( List.Count(segments) = 4 and segments{2} = "branches" ) then segments{3} else null, @@ -69,17 +71,19 @@ shared CommitTable = (url) as table => in commitTable; -Speckle.LogToMatomo = () => +Speckle.LogToMatomo = (server) => let - Source = Web.Contents("https://rynne.matomo.cloud/matomo.php?idsite=1&rec=1&apiv=1&uid=powerbiuser&action_name=receive%2Fmanual&url=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&urlref=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&_cvar=%7B%22hostApplication%22:%20%22PowerBI%22%7D&e_c=Power%20BI&e_a=receive%2Fmanual", + Source = Web.Contents("https://rynne.matomo.cloud/matomo.php?idsite=1&rec=1&apiv=1&uid=powerxyz&action_name=receive%2Fmanual&url=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&urlref=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&_cvar=%7B%22hostApplication%22:%20%22PowerBI%22%7D&e_c=Power%20BI&e_a=receive%2Fmanual", [ Headers=[ #"Method"="POST" ], - Content=Text.ToBinary("") - ]) + Content=Text.ToBinary(server) + ]), + Result = Text.FromBinary(Source), + return = if(Result = "some string") then Result else server in - Source; + server; Speckle.GetObjectFromStream = (server, streamId) => let @@ -140,7 +144,6 @@ shared Speckle.GetObjectFromCommit = (server, streamId, commitId) => Speckle.GetObjectFromObject = (server, streamId, objectId, IsCommitObject) => let - Matomo = Speckle.LogToMatomo(), query = if (IsCommitObject) then "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { children { objects { data } } } } }""}" else "{""query"": ""query { stream( id: \"""&streamId&"\"" ) { object (id: \"""&objectId&"\"") { data } } }""}", Source = Web.Contents( From de43e34c11ffb1b6cce04a9feb56036ddf91bb33 Mon Sep 17 00:00:00 2001 From: Alan Rynne Date: Thu, 7 Oct 2021 19:41:53 +0200 Subject: [PATCH 4/6] =?UTF-8?q?metrics:=20third's=20the=20charm=20?= =?UTF-8?q?=F0=9F=AA=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Speckle/Speckle.pq | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index 2591873..1905124 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -81,9 +81,10 @@ Speckle.LogToMatomo = (server) => Content=Text.ToBinary(server) ]), Result = Text.FromBinary(Source), - return = if(Result = "some string") then Result else server + Combined = Text.Combine({server,Result},"___"), + Split = Text.Split(Combined,"___"){0} in - server; + Split; Speckle.GetObjectFromStream = (server, streamId) => let From fc2ffa75ba871002402a9efbcde85e780d39d24a Mon Sep 17 00:00:00 2001 From: Alan Rynne Date: Thu, 7 Oct 2021 19:43:09 +0200 Subject: [PATCH 5/6] metrics: Added explanation of how logToMatomo works --- Speckle/Speckle.pq | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index 1905124..e49713d 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -71,6 +71,8 @@ shared CommitTable = (url) as table => in commitTable; + +/* Since everything is lazily evaluated, we must join and split the result of the matomo call with the server, and spit back the server url for PowerBI to actually log the calls to Matomo */ Speckle.LogToMatomo = (server) => let Source = Web.Contents("https://rynne.matomo.cloud/matomo.php?idsite=1&rec=1&apiv=1&uid=powerxyz&action_name=receive%2Fmanual&url=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&urlref=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&_cvar=%7B%22hostApplication%22:%20%22PowerBI%22%7D&e_c=Power%20BI&e_a=receive%2Fmanual", From c753e13859526c035486fdaac022cc461106e234 Mon Sep 17 00:00:00 2001 From: Alan Rynne Date: Fri, 8 Oct 2021 12:46:58 +0200 Subject: [PATCH 6/6] LogToMatomo() cleanup --- Speckle/Speckle.pq | 39 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/Speckle/Speckle.pq b/Speckle/Speckle.pq index e49713d..4e60a2c 100644 --- a/Speckle/Speckle.pq +++ b/Speckle/Speckle.pq @@ -75,20 +75,55 @@ shared CommitTable = (url) as table => /* Since everything is lazily evaluated, we must join and split the result of the matomo call with the server, and spit back the server url for PowerBI to actually log the calls to Matomo */ Speckle.LogToMatomo = (server) => let - Source = Web.Contents("https://rynne.matomo.cloud/matomo.php?idsite=1&rec=1&apiv=1&uid=powerxyz&action_name=receive%2Fmanual&url=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&urlref=http:%2F%2Fconnectors%2FPowerBI%2Freceive%2Fmanual&_cvar=%7B%22hostApplication%22:%20%22PowerBI%22%7D&e_c=Power%20BI&e_a=receive%2Fmanual", + matomoUrl = "https://speckle.matomo.cloud/matomo.php", + action = "receive/manual", + appName = "Power BI", + userId = "powerBIuser", + + params = [ + idsite = "2", + rec = "1", + apiv = "1", + uid = userId, + action_name = action, + url = Text.Combine({"http://connectors/PowerBI/", action}), + urlref = Text.Combine({"http://connectors/PowerBI/", action}), + _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName])) + ], + visitQuery = Uri.BuildQueryString(params), + visitRes = Web.Contents(Text.Combine({matomoUrl, "?", visitQuery}), [ Headers=[ #"Method"="POST" ], Content=Text.ToBinary(server) ]), - Result = Text.FromBinary(Source), + + eventParams = [ + idsite = "2", + rec = "1", + apiv = "1", + uid = userId, + _cvar = Text.FromBinary(Json.FromValue([hostApplication = appName])), + e_c = appName, + e_a = action + ], + eventQuery = Uri.BuildQueryString(eventParams), + eventRes = Web.Contents(Text.Combine({ matomoUrl, "?", eventQuery}), + [ + Headers=[ + #"Method"="POST" + ], + Content=Text.ToBinary(server) + ]), + Result = Text.FromBinary(visitRes) & Text.FromBinary(eventRes), Combined = Text.Combine({server,Result},"___"), Split = Text.Split(Combined,"___"){0} in Split; Speckle.GetObjectFromStream = (server, streamId) => + let branchName = "main", Source = Web.Contents(