microsoft
/
ghinsights
зеркало из https://github.com/microsoft/ghinsights.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

GHInsights Incremental Pipeline Completed 

-Completed Incremental changes for ghinsightsms
-For the EventId field, changed the GetInteger method to GetString
This commit is contained in:
t-shche 2016-07-08 17:36:35 -07:00
Родитель ba10634ad6
Коммит c18731e9f2
21 изменённых файлов: 157 добавлений и 39 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

4
ghinsights/DataFactory/DataFactory.dfproj
Просмотреть файл

@ -27,6 +27,10 @@
<Folder Include="Dependencies\" />
</ItemGroup>
<ItemGroup>
<Script Include="GHInsightsMSDataLakeStore.json" />
<Script Include="ProcessedDataMS.json" />
<Script Include="StagedDataMS.json" />
<Script Include="GHInsightsMSUSQL.json" />
<Script Include="MSGHTorrentAzureStorage.json" />
<Script Include="MSMongoDbDump.json" />
<Script Include="MSPrestagedEventsAzureStorage.json" />

15
ghinsights/DataFactory/GHInsightsMSDataLakeStore.json Normal file
Просмотреть файл

@ -0,0 +1,15 @@
{
"$schema": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.LinkedService.json",
"name": "GHInsightsMSDataLakeStore",
"properties": {
"type": "AzureDataLakeStore",
"typeProperties": {
"authorization": "",
"sessionId": "",
"dataLakeStoreUri": "",
"accountName": "",
"subscriptionId": "",
"resourceGroupName": ""
}
}
}

14
ghinsights/DataFactory/GHInsightsMSUSQL.json Normal file
Просмотреть файл

@ -0,0 +1,14 @@
{
"$schema": "http://datafactories.schema.management.azure.com/schemas/2015-09-01/Microsoft.DataFactory.LinkedService.json",
"name": "GHInsightsMSUSQL",
"properties": {
"type": "AzureDataLakeAnalytics",
"typeProperties": {
"authorization": "",
"sessionId": "",
"accountName": "",
"subscriptionId": "",
"resourceGroupName": ""
}
}
}

2
ghinsights/DataFactory/GHTorrentEventDetailPipeline.json
Просмотреть файл

@ -40,7 +40,6 @@
"typeProperties": {
"priority": 0,
"degreeOfParallelism": 5,
"scriptLinkedService": "GHInsightsAzureStorage",
"script": "GHInsights.dbo.StageData(@Year, @Month, @Day);",
"parameters": {
"Year": "$$Text.Format('{0:yyyy}',SliceStart)",
@ -68,7 +67,6 @@
"typeProperties": {
"priority": 0,
"degreeOfParallelism": 200,
"scriptLinkedService": "GHInsightsAzureStorage",
"script": "GHInsights.dbo.ProcessDaily(@Year, @Month, @Day);",
"parameters": {
"Year": "$$Text.Format('{0:yyyy}',SliceStart)",

55
ghinsights/DataFactory/MSGHTEventDetailPipeline.json
Просмотреть файл

@ -31,8 +31,63 @@
"frequency": "Day",
"interval": 1
}
},
{
"name": "StageUSQLActivity",
"type": "DataLakeAnalyticsU-SQL",
"inputs": [ { "name": "MSEventDetail" } ],
"outputs": [ { "name": "StageDataMS" } ],
"typeProperties": {
"priority": 0,
"degreeOfParallelism": 5,
"script": "GHInsights.dbo.StageData(@Year, @Month, @Day);",
"parameters": {
"Year": "$$Text.Format('{0:yyyy}',SliceStart)",
"Month": "$$Text.Format('{0:MM}',SliceStart)",
"Day": "$$Text.Format('{0:dd}',SliceStart)"
}
},
"linkedServiceName": "GHInsightsMSUSQL",
"policy": {
"concurrency": 6,
"executionPriorityOrder": "OldestFirst",
"retry": 0,
"timeout": "04:00:00"
},
"scheduler": {
"frequency": "Day",
"interval": 1
}
},
{
"name": "ProcessUSQLActivity",
"type": "DataLakeAnalyticsU-SQL",
"inputs": [ { "name": "StageDataMS" } ],
"outputs": [ { "name": "ProcessedDataMS" } ],
"typeProperties": {
"priority": 0,
"degreeOfParallelism": 20,
"script": "GHInsights.dbo.ProcessDaily(@Year, @Month, @Day);",
"parameters": {
"Year": "$$Text.Format('{0:yyyy}',SliceStart)",
"Month": "$$Text.Format('{0:MM}',SliceStart)",
"Day": "$$Text.Format('{0:dd}',SliceStart)"
}
},
"linkedServiceName": "GHInsightsMSUSQL",
"policy": {
"concurrency": 1,
"executionPriorityOrder": "OldestFirst",
"retry": 0,
"timeout": "04:00:00"
},
"scheduler": {
"frequency": "Day",
"interval": 1
}
}
],
"isPaused": false,
"start": "2016-06-13T00:00:00Z",
"end": "9999-09-09T00:00:00Z"
}

13
ghinsights/DataFactory/ProcessedDataMS.json Normal file
Просмотреть файл

@ -0,0 +1,13 @@
{
"name": "ProcessedDataMS",
"properties": {
"published": false,
"type": "CustomDataset",
"linkedServiceName": "GHInsightsMSDataLakeStore",
"typeProperties": { },
"availability": {
"frequency": "Day",
"interval": 1
}
}
}

13
ghinsights/DataFactory/StagedDataMS.json Normal file
Просмотреть файл

@ -0,0 +1,13 @@
{
"name": "StageDataMS",
"properties": {
"published": false,
"type": "CustomDataset",
"linkedServiceName": "GHInsightsMSDataLakeStore",
"typeProperties": { },
"availability": {
"frequency": "Day",
"interval": 1
}
}
}

34
ghinsights/USQL/ProcessDaily.usql
Просмотреть файл

@ -268,7 +268,7 @@ WHERE RowNumber == 1;
// 60
@NewRawEvent =
SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
SELECT GHInsights.USql.Utility.GetString(Data, "id") AS EventId
,Data
,GHInsights.USql.Utility.GetString(Data, "_id") AS EtlSourceId
,DateTime.Parse(IngestDate, null, System.Globalization.DateTimeStyles.AssumeUniversal).ToUniversalTime() AS EtlIngestDate
@ -303,11 +303,11 @@ SELECT EventId
,GHInsights.USql.Utility.GetString(Data, "payload.comment.user.login") AS PayloadCommentUserLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.comment.user.site_admin") AS PayloadCommentUserSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.comment.user.type") AS PayloadCommentUserType
,GHInsights.USql.Utility.GetString(Data, "payload.description") AS PayloadDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.description") AS PayloadDescription
,GHInsights.USql.Utility.GetInteger(Data, "payload.distinct_size") AS PayloadDistinctSize
,GHInsights.USql.Utility.GetDateTime(Data, "payload.forkee.created_at") AS PayloadForkeeCreatedAt
,GHInsights.USql.Utility.GetString(Data, "payload.forkee.default_branch") AS PayloadForkeeDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "payload.forkee.description") AS PayloadForkeeDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.forkee.description") AS PayloadForkeeDescription
,GHInsights.USql.Utility.GetBoolean(Data, "payload.forkee.fork") AS PayloadForkeeFork
,GHInsights.USql.Utility.GetInteger(Data, "payload.forkee.forks") AS PayloadForkeeForks
,GHInsights.USql.Utility.GetInteger(Data, "payload.forkee.forks_count") AS PayloadForkeeForksCount
@ -352,7 +352,7 @@ SELECT EventId
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.creator.login") AS PayloadIssueMilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.issue.milestone.creator.site_admin") AS PayloadIssueMilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.creator.type") AS PayloadIssueMilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.description") AS PayloadIssueMilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.issue.milestone.description") AS PayloadIssueMilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "payload.issue.milestone.due_on") AS PayloadIssueMilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "payload.issue.milestone.id") AS PayloadIssueMilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "payload.issue.milestone.number") AS PayloadIssueMilestoneNumber
@ -387,7 +387,7 @@ SELECT EventId
,GHInsights.USql.Utility.GetBoolean(Data, "payload.release.draft") AS PayloadReleaseDraft
,GHInsights.USql.Utility.GetInteger(Data, "payload.release.id") AS PayloadReleaseId
,GHInsights.USql.Utility.GetString(Data, "payload.release.name") AS PayloadReleaseName
,GHInsights.USql.Utility.GetBoolean(Data, "payload.release.prerelease") AS PayloadReleasePrerelease
,GHInsights.USql.Utility.GetBoolean(Data, "payload.rlemeease.prerelease") AS PayloadReleasePrerelease
,GHInsights.USql.Utility.GetDateTime(Data, "payload.release.published_at") AS PayloadReleasePublishedAt
,GHInsights.USql.Utility.GetString(Data, "payload.release.tag_name") AS PayloadReleaseTagName
,GHInsights.USql.Utility.GetString(Data, "payload.release.target_commitish") AS PayloadReleaseTargetCommitish
@ -785,7 +785,7 @@ SELECT EventId
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.ref") AS HeadRef
,GHInsights.USql.Utility.GetDateTime(Data, "payload.pull_request.head.repo.created_at") AS HeadRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.repo.default_branch") AS HeadRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.pull_request.head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "payload.pull_request.head.repo.fork") AS HeadRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.head.repo.forks") AS HeadRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.head.repo.forks_count") AS HeadRepoForksCount
@ -833,7 +833,7 @@ SELECT EventId
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.pull_request.milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.pull_request.milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "payload.pull_request.milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.milestone.number") AS MilestoneNumber
@ -1081,7 +1081,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetInteger(Data, "id") AS ForkId
,GHInsights.USql.Utility.GetDateTime(Data, "created_at") AS CreatedAt
,GHInsights.USql.Utility.GetString(Data, "default_branch") AS DefaultBranch
,GHInsights.USql.Utility.GetString(Data, "description") AS Description
,GHInsights.USql.Utility.GetUSqlString(Data, "description") AS Description
,GHInsights.USql.Utility.GetBoolean(Data, "fork") AS Fork
,GHInsights.USql.Utility.GetInteger(Data, "forks") AS Forks
,GHInsights.USql.Utility.GetInteger(Data, "forks_count") AS ForksCount
@ -1097,7 +1097,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetInteger(Data, "open_issues_count") AS OpenIssuesCount
,GHInsights.USql.Utility.GetDateTime(Data, "parent.created_at") AS ParentCreatedAt
,GHInsights.USql.Utility.GetString(Data, "parent.default_branch") AS ParentDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetBoolean(Data, "parent.fork") AS ParentFork
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks") AS ParentForks
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks_count") AS ParentForksCount
@ -1130,7 +1130,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetInteger(Data, "size") AS Size
,GHInsights.USql.Utility.GetDateTime(Data, "source.created_at") AS SourceCreatedAt
,GHInsights.USql.Utility.GetString(Data, "source.default_branch") AS SourceDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetBoolean(Data, "source.fork") AS SourceFork
,GHInsights.USql.Utility.GetInteger(Data, "source.forks") AS SourceForks
,GHInsights.USql.Utility.GetInteger(Data, "source.forks_count") AS SourceForksCount
@ -1359,7 +1359,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "milestone.number") AS MilestoneNumber
@ -1698,7 +1698,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "base.ref") AS BaseRef
,GHInsights.USql.Utility.GetDateTime(Data, "base.repo.created_at") AS BaseRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "base.repo.default_branch") AS BaseRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "base.repo.fork") AS BaseRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "base.repo.forks") AS BaseRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "base.repo.forks_count") AS BaseRepoForksCount
@ -1740,7 +1740,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "head.ref") AS HeadRef
,GHInsights.USql.Utility.GetDateTime(Data, "head.repo.created_at") AS HeadRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "head.repo.default_branch") AS HeadRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "head.repo.fork") AS HeadRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "head.repo.forks") AS HeadRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "head.repo.forks_count") AS HeadRepoForksCount
@ -1788,7 +1788,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "milestone.number") AS MilestoneNumber
@ -2041,7 +2041,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetString(Data, "owner.login") AS OwnerLogin
,GHInsights.USql.Utility.GetDateTime(Data, "created_at") AS CreatedAt
,GHInsights.USql.Utility.GetString(Data, "default_branch") AS DefaultBranch
,GHInsights.USql.Utility.GetString(Data, "description") AS Description
,GHInsights.USql.Utility.GetUSqlString(Data, "description") AS Description
,GHInsights.USql.Utility.GetBoolean(Data, "fork") AS Fork
,GHInsights.USql.Utility.GetInteger(Data, "forks") AS Forks
,GHInsights.USql.Utility.GetInteger(Data, "forks_count") AS ForksCount
@ -2065,7 +2065,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetString(Data, "owner.type") AS OwnerType
,GHInsights.USql.Utility.GetDateTime(Data, "parent.created_at") AS ParentCreatedAt
,GHInsights.USql.Utility.GetString(Data, "parent.default_branch") AS ParentDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetBoolean(Data, "parent.fork") AS ParentFork
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks") AS ParentForks
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks_count") AS ParentForksCount
@ -2099,7 +2099,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetInteger(Data, "size") AS Size
,GHInsights.USql.Utility.GetDateTime(Data, "source.created_at") AS SourceCreatedAt
,GHInsights.USql.Utility.GetString(Data, "source.default_branch") AS SourceDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetBoolean(Data, "source.fork") AS SourceFork
,GHInsights.USql.Utility.GetInteger(Data, "source.forks") AS SourceForks
,GHInsights.USql.Utility.GetInteger(Data, "source.forks_count") AS SourceForksCount

8
ghinsights/USQL/ProcessEvent.usql
Просмотреть файл

@ -3,7 +3,7 @@ REFERENCE ASSEMBLY [GHInsights.USql];
REFERENCE ASSEMBLY [Newtonsoft.Json];
@Event =
SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
SELECT GHInsights.USql.Utility.GetString(Data, "id") AS EventId
,GHInsights.USql.Utility.GetInteger(Data, "actor.id") AS ActorId
,GHInsights.USql.Utility.GetString(Data, "actor.login") AS ActorLogin
,GHInsights.USql.Utility.GetDateTime(Data, "created_at") AS CreatedAt
@ -29,11 +29,11 @@ SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
,GHInsights.USql.Utility.GetString(Data, "payload.comment.user.login") AS PayloadCommentUserLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.comment.user.site_admin") AS PayloadCommentUserSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.comment.user.type") AS PayloadCommentUserType
,GHInsights.USql.Utility.GetString(Data, "payload.description") AS PayloadDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.description") AS PayloadDescription
,GHInsights.USql.Utility.GetInteger(Data, "payload.distinct_size") AS PayloadDistinctSize
,GHInsights.USql.Utility.GetDateTime(Data, "payload.forkee.created_at") AS PayloadForkeeCreatedAt
,GHInsights.USql.Utility.GetString(Data, "payload.forkee.default_branch") AS PayloadForkeeDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "payload.forkee.description") AS PayloadForkeeDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.forkee.description") AS PayloadForkeeDescription
,GHInsights.USql.Utility.GetBoolean(Data, "payload.forkee.fork") AS PayloadForkeeFork
,GHInsights.USql.Utility.GetInteger(Data, "payload.forkee.forks") AS PayloadForkeeForks
,GHInsights.USql.Utility.GetInteger(Data, "payload.forkee.forks_count") AS PayloadForkeeForksCount
@ -78,7 +78,7 @@ SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.creator.login") AS PayloadIssueMilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.issue.milestone.creator.site_admin") AS PayloadIssueMilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.creator.type") AS PayloadIssueMilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "payload.issue.milestone.description") AS PayloadIssueMilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.issue.milestone.description") AS PayloadIssueMilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "payload.issue.milestone.due_on") AS PayloadIssueMilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "payload.issue.milestone.id") AS PayloadIssueMilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "payload.issue.milestone.number") AS PayloadIssueMilestoneNumber

Двоичные данные
ghinsights/USQL/ProcessEventPayloadCommit.usql
Просмотреть файл

Двоичный файл не отображается.

Двоичные данные
ghinsights/USQL/ProcessEventPayloadIssueLabel.usql
Просмотреть файл

Двоичный файл не отображается.

6
ghinsights/USQL/ProcessEventPayloadIssueLabel.usql.cs Normal file
Просмотреть файл

@ -0,0 +1,6 @@
using Microsoft.Analytics.Interfaces;
using Microsoft.Analytics.Types;
using System;
using System.Collections.Generic;
using System.IO;
using System.Text;

Двоичные данные
ghinsights/USQL/ProcessEventPayloadPage.usql
Просмотреть файл

Двоичный файл не отображается.

8
ghinsights/USQL/ProcessEventPayloadPullRequest.usql
Просмотреть файл

@ -3,7 +3,7 @@ REFERENCE ASSEMBLY [GHInsights.USql];
REFERENCE ASSEMBLY [Newtonsoft.Json];
@Event =
SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
SELECT GHInsights.USql.Utility.GetString(Data, "id") AS EventId
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.id") AS EventPayloadPullRequestId
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.additions") AS Additions
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.assignee.id") AS AssigneeId
@ -14,7 +14,7 @@ SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.base.ref") AS BaseRef
,GHInsights.USql.Utility.GetDateTime(Data, "payload.pull_request.base.repo.created_at") AS BaseRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.base.repo.default_branch") AS BaseRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.pull_request.base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "payload.pull_request.base.repo.fork") AS BaseRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.base.repo.forks") AS BaseRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.base.repo.forks_count") AS BaseRepoForksCount
@ -56,7 +56,7 @@ SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.ref") AS HeadRef
,GHInsights.USql.Utility.GetDateTime(Data, "payload.pull_request.head.repo.created_at") AS HeadRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.repo.default_branch") AS HeadRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.pull_request.head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "payload.pull_request.head.repo.fork") AS HeadRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.head.repo.forks") AS HeadRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.head.repo.forks_count") AS HeadRepoForksCount
@ -104,7 +104,7 @@ SELECT GHInsights.USql.Utility.GetInteger(Data, "id") AS EventId
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "payload.pull_request.milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "payload.pull_request.milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "payload.pull_request.milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "payload.pull_request.milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "payload.pull_request.milestone.number") AS MilestoneNumber

Двоичные данные
ghinsights/USQL/ProcessEventPayloadReleaseAsset.usql
Просмотреть файл

Двоичный файл не отображается.

2
ghinsights/USQL/ProcessFork.usql
Просмотреть файл

@ -10,7 +10,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetInteger(Data, "id") AS ForkId
,GHInsights.USql.Utility.GetDateTime(Data, "created_at") AS CreatedAt
,GHInsights.USql.Utility.GetString(Data, "default_branch") AS DefaultBranch
,GHInsights.USql.Utility.GetString(Data, "description") AS Description
,GHInsights.USql.Utility.GetUSqlString(Data, "description") AS Description
,GHInsights.USql.Utility.GetBoolean(Data, "fork") AS Fork
,GHInsights.USql.Utility.GetInteger(Data, "forks") AS Forks
,GHInsights.USql.Utility.GetInteger(Data, "forks_count") AS ForksCount

2
ghinsights/USQL/ProcessIssue.usql
Просмотреть файл

@ -29,7 +29,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "milestone.number") AS MilestoneNumber

6
ghinsights/USQL/ProcessPullRequest.usql
Просмотреть файл

@ -24,7 +24,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "base.ref") AS BaseRef
,GHInsights.USql.Utility.GetDateTime(Data, "base.repo.created_at") AS BaseRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "base.repo.default_branch") AS BaseRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "base.repo.description") AS BaseRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "base.repo.fork") AS BaseRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "base.repo.forks") AS BaseRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "base.repo.forks_count") AS BaseRepoForksCount
@ -66,7 +66,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "head.ref") AS HeadRef
,GHInsights.USql.Utility.GetDateTime(Data, "head.repo.created_at") AS HeadRepoCreatedAt
,GHInsights.USql.Utility.GetString(Data, "head.repo.default_branch") AS HeadRepoDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "head.repo.description") AS HeadRepoDescription
,GHInsights.USql.Utility.GetBoolean(Data, "head.repo.fork") AS HeadRepoFork
,GHInsights.USql.Utility.GetInteger(Data, "head.repo.forks") AS HeadRepoForks
,GHInsights.USql.Utility.GetInteger(Data, "head.repo.forks_count") AS HeadRepoForksCount
@ -114,7 +114,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "repo") AS Repo
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.login") AS MilestoneCreatorLogin
,GHInsights.USql.Utility.GetBoolean(Data, "milestone.creator.site_admin") AS MilestoneCreatorSiteAdmin
,GHInsights.USql.Utility.GetString(Data, "milestone.creator.type") AS MilestoneCreatorType
,GHInsights.USql.Utility.GetString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "milestone.description") AS MilestoneDescription
,GHInsights.USql.Utility.GetDateTime(Data, "milestone.due_on") AS MilestoneDueOn
,GHInsights.USql.Utility.GetInteger(Data, "milestone.id") AS MilestoneId
,GHInsights.USql.Utility.GetInteger(Data, "milestone.number") AS MilestoneNumber

6
ghinsights/USQL/ProcessRepo.usql
Просмотреть файл

@ -9,7 +9,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetString(Data, "owner.login") AS OwnerLogin
,GHInsights.USql.Utility.GetDateTime(Data, "created_at") AS CreatedAt
,GHInsights.USql.Utility.GetString(Data, "default_branch") AS DefaultBranch
,GHInsights.USql.Utility.GetString(Data, "description") AS Description
,GHInsights.USql.Utility.GetUSqlString(Data, "description") AS Description
,GHInsights.USql.Utility.GetBoolean(Data, "fork") AS Fork
,GHInsights.USql.Utility.GetInteger(Data, "forks") AS Forks
,GHInsights.USql.Utility.GetInteger(Data, "forks_count") AS ForksCount
@ -32,7 +32,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetString(Data, "owner.type") AS OwnerType
,GHInsights.USql.Utility.GetDateTime(Data, "parent.created_at") AS ParentCreatedAt
,GHInsights.USql.Utility.GetString(Data, "parent.default_branch") AS ParentDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "parent.description") AS ParentDescription
,GHInsights.USql.Utility.GetBoolean(Data, "parent.fork") AS ParentFork
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks") AS ParentForks
,GHInsights.USql.Utility.GetInteger(Data, "parent.forks_count") AS ParentForksCount
@ -66,7 +66,7 @@ SELECT GHInsights.USql.Utility.GetString(Data, "name") AS Name
,GHInsights.USql.Utility.GetInteger(Data, "size") AS Size
,GHInsights.USql.Utility.GetDateTime(Data, "source.created_at") AS SourceCreatedAt
,GHInsights.USql.Utility.GetString(Data, "source.default_branch") AS SourceDefaultBranch
,GHInsights.USql.Utility.GetString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetUSqlString(Data, "source.description") AS SourceDescription
,GHInsights.USql.Utility.GetBoolean(Data, "source.fork") AS SourceFork
,GHInsights.USql.Utility.GetInteger(Data, "source.forks") AS SourceForks
,GHInsights.USql.Utility.GetInteger(Data, "source.forks_count") AS SourceForksCount

2
ghinsights/USQL/StageData.usql
Просмотреть файл

@ -17,7 +17,7 @@ DECLARE @IngestDatePartition = @IngestDate.ToString("yyyy-MM-dd");
EXTRACT EventName string
,IngestDate DateTime
,Data SqlMap<string, byte[]>
FROM "wasb://raw@ghinsightspublic.blob.core.windows.net/{EventName:*}/v1/{IngestDate:yyyy}/{IngestDate:MM}/{EventName:*}_{IngestDate:yyyy}_{IngestDate:MM}_{IngestDate:dd}.json.gz"
FROM "wasb://raw@ghinsightsms.blob.core.windows.net/{EventName:*}/v1/{IngestDate:yyyy}/{IngestDate:MM}/{EventName:*}_{IngestDate:yyyy}_{IngestDate:MM}_{IngestDate:dd}.json.gz"
USING GHInsights.USql.Extractors.FlatJson(outputColumnName: "Data");
ALTER TABLE Staging.GHTorrent.GitHubData DROP IF EXISTS PARTITION (@IngestDatePartition);

6
ghinsights/USQL/USQL.usqlproj
Просмотреть файл

@ -247,7 +247,7 @@
<ClusterDatabase>GHInsights</ClusterDatabase>
<ClusterSchema>dbo</ClusterSchema>
</Script>
<Script Include="EventKeys.usql" />
<Script Include="QueryDates.usql" />
<Script Include="StageData.usql">
<UseLocalComputeAccount>true</UseLocalComputeAccount>
<LocalComputeAccount>localcomputeaccount</LocalComputeAccount>
@ -259,8 +259,8 @@
</Script>
</ItemGroup>
<ItemGroup>
<ScriptCode Include="EventKeys.usql.cs">
<DependentUpon>EventKeys.usql</DependentUpon>
<ScriptCode Include="QueryDates.usql.cs">
<DependentUpon>QueryDates.usql</DependentUpon>
</ScriptCode>
</ItemGroup>
<Import Project="$(AppData)\Microsoft\DataLake\MsBuild\1.0\Usql.targets" />