зеркало из https://github.com/microsoft/mwt-ds.git
added VideoIndexer/Ooyala integration
This commit is contained in:
Markus Cozowicz
Родитель
49299a931c
Коммит
f0c24a1de9
|
|
@ -51,7 +51,8 @@ namespace Microsoft.DecisionService.Crawl
|
|||
if (topicRemoteRaw != null)
|
||||
blobContent.Output.Add(new JProperty("topics", topicRemoteRaw.Value<string>().Split(',').Select(float.Parse).ToArray()));
|
||||
},
|
||||
cancellationToken);
|
||||
isPost: true,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public class StringTable
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
{
|
||||
private readonly string queryParams;
|
||||
|
||||
public CognitiveService(string containerName, string queryParams = null) : base(containerName)
|
||||
public CognitiveService(string containerName, string queryParams = null, string apiKey = null) : base(containerName, apiKey)
|
||||
{
|
||||
this.queryParams = queryParams;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ using System.Linq;
|
|||
using System.Collections.Generic;
|
||||
using System.Threading;
|
||||
using Microsoft.DecisionService.Crawl.Data;
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
|
|
@ -28,7 +29,17 @@ namespace Microsoft.DecisionService.Crawl
|
|||
public static Task<HttpResponseMessage> Run(HttpRequestMessage req, TraceWriter log, CancellationToken cancellationToken)
|
||||
{
|
||||
return cogService.InvokeAsync(req, log,
|
||||
reqBody => new UrlHolder { Url = reqBody.Image },
|
||||
reqBody =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reqBody.Image))
|
||||
return null;
|
||||
|
||||
Uri uri;
|
||||
if (!Uri.TryCreate(reqBody.Image, UriKind.Absolute, out uri))
|
||||
return null;
|
||||
|
||||
return new UrlHolder { Url = reqBody.Image };
|
||||
},
|
||||
(reqBody, blobContent) =>
|
||||
{
|
||||
var visionResponse = JsonConvert.DeserializeObject<Face[]>(blobContent.Value);
|
||||
|
|
@ -45,7 +56,8 @@ namespace Microsoft.DecisionService.Crawl
|
|||
}
|
||||
}
|
||||
},
|
||||
cancellationToken);
|
||||
isPost: true,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public class Face
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ namespace Microsoft.DecisionService.Crawl
|
|||
if (!string.IsNullOrEmpty(reqBody.Article))
|
||||
textBuilder.AppendLine(reqBody.Article);
|
||||
|
||||
if (textBuilder.Length == 0)
|
||||
return null;
|
||||
|
||||
return Services.Limit(textBuilder.ToString(), 10240);
|
||||
},
|
||||
(reqBody, blobContent) =>
|
||||
|
|
@ -52,7 +55,8 @@ namespace Microsoft.DecisionService.Crawl
|
|||
blobContent.Output.Add("Tags", new JObject(q));
|
||||
}
|
||||
},
|
||||
cancellationToken);
|
||||
isPost: true,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public class EntityResponse
|
||||
|
|
|
|||
|
|
@ -36,6 +36,9 @@ namespace Microsoft.DecisionService.Crawl
|
|||
if (!string.IsNullOrEmpty(reqBody.Article))
|
||||
textBuilder.AppendLine(reqBody.Article);
|
||||
|
||||
if (textBuilder.Length == 0)
|
||||
return null;
|
||||
|
||||
var text = textBuilder.ToString();
|
||||
|
||||
// Based on email thread with Arvind Krishnaa Jagannathan <arjagann@microsoft.com>
|
||||
|
|
@ -63,7 +66,8 @@ namespace Microsoft.DecisionService.Crawl
|
|||
if (responseObj?.Documents?.Length == 1)
|
||||
blobContent.Output.Add(new JProperty("XSentiment", responseObj.Documents[0].Score));
|
||||
},
|
||||
cancellationToken);
|
||||
isPost: true,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public class TextAnalyticRequest
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ using Newtonsoft.Json.Linq;
|
|||
using System.Linq;
|
||||
using System.Threading;
|
||||
using Microsoft.DecisionService.Crawl.Data;
|
||||
using System;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
|
|
@ -27,7 +28,17 @@ namespace Microsoft.DecisionService.Crawl
|
|||
public static async Task<HttpResponseMessage> Run(HttpRequestMessage req, TraceWriter log, CancellationToken cancellationToken)
|
||||
{
|
||||
return await cogService.InvokeAsync(req, log,
|
||||
reqBody => new UrlHolder { Url = reqBody.Image },
|
||||
reqBody =>
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reqBody.Image))
|
||||
return null;
|
||||
|
||||
Uri uri;
|
||||
if (!Uri.TryCreate(reqBody.Image, UriKind.Absolute, out uri))
|
||||
return null;
|
||||
|
||||
return new UrlHolder { Url = reqBody.Image };
|
||||
},
|
||||
(reqBody, blobContent) =>
|
||||
{
|
||||
var visionResponse = JsonConvert.DeserializeObject<VisionResponse>(blobContent.Value);
|
||||
|
|
@ -69,7 +80,8 @@ namespace Microsoft.DecisionService.Crawl
|
|||
celebs.Select(t => new JProperty(t.Key, t.Max(x => x.Confidence))))));
|
||||
}
|
||||
},
|
||||
cancellationToken);
|
||||
isPost: true,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
|
||||
public class VisionResponse
|
||||
|
|
|
|||
|
|
@ -137,6 +137,23 @@
|
|||
<Reference Include="System.Xml.Linq" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdown.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownAudioEffects.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownBreakdown.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownContentModeration.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownFace.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownInsights.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownSentiment.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownSummarizedInsights.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownTopic.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownTranscriptBlock.cs" />
|
||||
<Compile Include="VideoIndexer\Breakdown\VideoBreakdownTranscriptBlockLine.cs" />
|
||||
<Compile Include="VideoIndexer\Ooyala\Ooyala.cs" />
|
||||
<Compile Include="VideoIndexer\Ooyala\OoyalaVideo.cs" />
|
||||
<Compile Include="VideoIndexer\VideoIndexer.cs" />
|
||||
<Compile Include="VideoIndexer\VideoIndexerFeaturizer.cs" />
|
||||
<Compile Include="VideoIndexer\Search\VideoIndexerSearchResult.cs" />
|
||||
<Compile Include="VideoIndexer\Search\VideoIndexerSearchResultItem.cs" />
|
||||
<Content Include="packages.config">
|
||||
<SubType>Designer</SubType>
|
||||
</Content>
|
||||
|
|
@ -151,6 +168,8 @@
|
|||
<Compile Include="CognitiveServiceEntityLinking\CognitiveServiceEntityLinking.cs" />
|
||||
<Compile Include="CognitiveServiceTextAnalytics\CognitiveServiceTextAnalytics.cs" />
|
||||
<Compile Include="HttpCachedService.cs" />
|
||||
<Compile Include="VideoIndexer\Ooyala\JSON.cs" />
|
||||
<Compile Include="VideoIndexer\Ooyala\OoyalaAPI.cs" />
|
||||
<Compile Include="RawStringConverter.cs" />
|
||||
<Compile Include="RSS\RSS.cs" />
|
||||
<Content Include="CognitiveServiceVision\function.json" />
|
||||
|
|
@ -159,6 +178,7 @@
|
|||
<Content Include="CognitiveServiceTextAnalytics\function.json" />
|
||||
<Content Include="AzureMLTopic\function.json" />
|
||||
<None Include="Properties\PublishProfiles\DevProfile.pubxml" />
|
||||
<Content Include="VideoIndexer\function.json" />
|
||||
<None Include="Web.Debug.config">
|
||||
<DependentUpon>Web.config</DependentUpon>
|
||||
</None>
|
||||
|
|
|
|||
|
|
@ -46,5 +46,8 @@ namespace Microsoft.DecisionService.Crawl.Data
|
|||
|
||||
[JsonProperty("forceRefresh")]
|
||||
public bool ForceRefresh { get; set; } = false;
|
||||
|
||||
[JsonProperty("video", NullValueHandling = NullValueHandling.Ignore)]
|
||||
public string Video { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -32,17 +32,33 @@ namespace Microsoft.DecisionService.Crawl
|
|||
internal string apiKey;
|
||||
internal string storageConnectionString;
|
||||
|
||||
public HttpCachedService(string containerName)
|
||||
public HttpCachedService(string containerName, string apiKey = null)
|
||||
{
|
||||
// limit due to Azure Storage container name
|
||||
if (containerName.Length > 24 - 6 /* yyyyMM */)
|
||||
throw new ArgumentException($"{nameof(containerName)}: '{containerName}' is too long. Must be {24 - 6} characters at most.");
|
||||
this.containerName = containerName;
|
||||
|
||||
this.apiKey = apiKey;
|
||||
}
|
||||
|
||||
protected virtual void Initialize()
|
||||
{ }
|
||||
|
||||
public async Task<string> GetAzureStorageConnectionStringAsync()
|
||||
{
|
||||
await this.InitializeAsync();
|
||||
|
||||
return this.storageConnectionString;
|
||||
}
|
||||
|
||||
public async Task<HttpClient> GetHttpClientAsync()
|
||||
{
|
||||
await this.InitializeAsync();
|
||||
|
||||
return this.client;
|
||||
}
|
||||
|
||||
private async Task InitializeAsync()
|
||||
{
|
||||
if (this.client != null)
|
||||
|
|
@ -60,15 +76,19 @@ namespace Microsoft.DecisionService.Crawl
|
|||
var keyVault = new KeyVaultClient(new KeyVaultClient.AuthenticationCallback(keyVaultHelper.GetAccessToken));
|
||||
|
||||
this.endpoint = (await keyVault.GetSecretAsync(keyVaultUrl, containerName + "Endpoint").ConfigureAwait(false)).Value;
|
||||
this.apiKey = (await keyVault.GetSecretAsync(keyVaultUrl, containerName + "Key").ConfigureAwait(false)).Value;
|
||||
this.storageConnectionString = (await keyVault.GetSecretAsync(keyVaultUrl, "StorageConnectionString").ConfigureAwait(false)).Value;
|
||||
|
||||
if (string.IsNullOrEmpty(this.apiKey))
|
||||
this.apiKey = (await keyVault.GetSecretAsync(keyVaultUrl, containerName + "Key").ConfigureAwait(false)).Value;
|
||||
}
|
||||
else
|
||||
{
|
||||
// fallback to local settings
|
||||
this.endpoint = ConfigurationManager.AppSettings[containerName + "Endpoint"];
|
||||
this.apiKey = ConfigurationManager.AppSettings[containerName + "Key"];
|
||||
this.storageConnectionString = ConfigurationManager.AppSettings["StorageConnectionString"];
|
||||
|
||||
if (string.IsNullOrEmpty(this.apiKey))
|
||||
this.apiKey = ConfigurationManager.AppSettings[containerName + "Key"];
|
||||
}
|
||||
|
||||
this.client = new HttpClient()
|
||||
|
|
@ -79,7 +99,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
this.Initialize();
|
||||
}
|
||||
|
||||
public async Task<BlobContent> PostAsync(TraceWriter log, string site, string id, object request, bool forceRefresh, CancellationToken cancellationToken)
|
||||
public async Task<BlobContent> RequestAsync(TraceWriter log, string site, string id, object request, bool forceRefresh, bool isPost, CancellationToken cancellationToken)
|
||||
{
|
||||
await this.InitializeAsync();
|
||||
|
||||
|
|
@ -123,13 +143,22 @@ namespace Microsoft.DecisionService.Crawl
|
|||
|
||||
var stopwatchReqeust = Stopwatch.StartNew();
|
||||
|
||||
// make the actual HTTP request
|
||||
responseMessage = await this.client.PostAsync(
|
||||
string.Empty,
|
||||
new StringContent(
|
||||
body,
|
||||
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
|
||||
contentType));
|
||||
if (isPost)
|
||||
{
|
||||
// make the actual HTTP request
|
||||
responseMessage = await this.client.PostAsync(
|
||||
string.Empty,
|
||||
new StringContent(
|
||||
body,
|
||||
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
|
||||
contentType),
|
||||
cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
// make the actual HTTP request
|
||||
responseMessage = await this.client.GetAsync(body, cancellationToken);
|
||||
}
|
||||
|
||||
Services.TelemetryClient.TrackDependency(this.containerName, this.endpoint, this.containerName, null,
|
||||
DateTime.UtcNow, stopwatchReqeust.Elapsed,
|
||||
|
|
@ -191,6 +220,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
public async Task<HttpResponseMessage> InvokeAsync(HttpRequestMessage req, TraceWriter log,
|
||||
Func<CrawlResponse, object> requestBodyFunc,
|
||||
Action<CrawlResponse, BlobContent> responseAction,
|
||||
bool isPost,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
log.Info("Crawl." + this.containerName);
|
||||
|
|
@ -212,12 +242,26 @@ namespace Microsoft.DecisionService.Crawl
|
|||
operation.Telemetry.Properties.Add("AppId", reqBody.Site);
|
||||
operation.Telemetry.Properties.Add("ActionId", reqBody.Id);
|
||||
|
||||
blobContent = await this.PostAsync(
|
||||
var serviceRequestBody = requestBodyFunc(reqBody);
|
||||
|
||||
if (serviceRequestBody == null)
|
||||
{
|
||||
return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
string.Empty,
|
||||
new UTF8Encoding(encoderShouldEmitUTF8Identifier: false),
|
||||
"application/json")
|
||||
};
|
||||
}
|
||||
|
||||
blobContent = await this.RequestAsync(
|
||||
log,
|
||||
reqBody.Site,
|
||||
reqBody.Id,
|
||||
requestBodyFunc(reqBody),
|
||||
serviceRequestBody,
|
||||
reqBody.ForceRefresh,
|
||||
isPost,
|
||||
cancellationToken);
|
||||
|
||||
if (blobContent != null)
|
||||
|
|
@ -232,7 +276,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
}
|
||||
}
|
||||
|
||||
return req.CreateResponse(blobContent);
|
||||
return Services.CreateResponse(blobContent);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
|
|
|||
|
|
@ -80,7 +80,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
// The order of the items allows customers to specify their base-line policy
|
||||
.OrderBy(elem => elem.index)
|
||||
.Select(x => x.elem);
|
||||
|
||||
|
||||
var actions = items.Select(x => new
|
||||
{
|
||||
ids = new[] { new { id = x.Descendants("link").FirstOrDefault()?.Value } },
|
||||
|
|
@ -92,6 +92,10 @@ namespace Microsoft.DecisionService.Crawl
|
|||
{
|
||||
// TODO: properly support 4.2.6. The "atom:id" Element
|
||||
new { guid = x.Descendants("guid").FirstOrDefault()?.Value }
|
||||
},
|
||||
privateDetails = new[]
|
||||
{
|
||||
new { video = x.Descendants("content").FirstOrDefault()?.Attribute("url")?.Value }
|
||||
}
|
||||
}).ToList();
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
return text.Substring(length);
|
||||
}
|
||||
|
||||
public static HttpResponseMessage CreateResponse(this HttpRequestMessage req, BlobContent blobContent)
|
||||
public static HttpResponseMessage CreateResponse(BlobContent blobContent)
|
||||
{
|
||||
blobContent.Output?.Add(new JProperty("_expires", blobContent.Expires));
|
||||
|
||||
|
|
@ -65,7 +65,7 @@ namespace Microsoft.DecisionService.Crawl
|
|||
return response;
|
||||
}
|
||||
|
||||
public static void TrackException(Exception ex, HttpRequestMessage req, TraceWriter log, string reqBodyStr, CrawlResponse reqBody, BlobContent blobContent)
|
||||
public static void TrackException(Exception ex, HttpRequestMessage req, TraceWriter log, string reqBodyStr, CrawlResponse reqBody, BlobContent blobContent = null)
|
||||
{
|
||||
var props = new Dictionary<string, string>
|
||||
{
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResult
|
||||
{
|
||||
[JsonProperty("state")] // processed
|
||||
public string State { get; set; }
|
||||
|
||||
[JsonProperty("durationInSeconds")]
|
||||
public int DurationInSeconds { get; set; }
|
||||
|
||||
[JsonProperty("breakdowns")]
|
||||
public VideoBreakdownResultBreakdown[] Breakdowns { get; set; }
|
||||
|
||||
|
||||
[JsonProperty("summarizedInsights")]
|
||||
public VideoBreakdownResultSummarizedInsights SummarizedInsights { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultAudioEffects
|
||||
{
|
||||
[JsonProperty("audioEffectKey")]
|
||||
public string AudioEffectKey { get; set; }
|
||||
|
||||
[JsonProperty("seenDuration")]
|
||||
public float SeenDuration { get; set; }
|
||||
|
||||
[JsonProperty("seenDurationRatio")]
|
||||
public float SeenDurationRatio { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultBreakdown
|
||||
{
|
||||
[JsonProperty("externalId")]
|
||||
public string ExternalId { get; set; }
|
||||
|
||||
[JsonProperty("insights")]
|
||||
public VideoBreakdownResultInsights Insight { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultContentModeration
|
||||
{
|
||||
[JsonProperty("adultClassifierValue")]
|
||||
public float AdultClassifierValue { get; set; }
|
||||
|
||||
[JsonProperty("bannedWordsCount")]
|
||||
public int BannedWordsCount { get; set; }
|
||||
|
||||
[JsonProperty("isSuspectedAsAdult")]
|
||||
public bool IsSuspectedAsAdult { get; set; }
|
||||
|
||||
[JsonProperty("isAdult")]
|
||||
public bool IsAdult { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultFace
|
||||
{
|
||||
[JsonProperty("name")]
|
||||
public string Name { get; set; } // // Unknown #1
|
||||
|
||||
[JsonProperty("seenDuration")]
|
||||
public float SeenDuration { get; set; }
|
||||
|
||||
[JsonProperty("seenDurationRatio")]
|
||||
public float SeenDurationRatio { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultInsights
|
||||
{
|
||||
[JsonProperty("contentModeration")]
|
||||
public VideoBreakdownResultContentModeration ContentModeration { get; set; }
|
||||
|
||||
[JsonProperty("viewToken")]
|
||||
public string ViewToken { get; set; }
|
||||
|
||||
[JsonProperty("language")]
|
||||
public string Language { get; set; }
|
||||
|
||||
[JsonProperty("transcriptBlocks")]
|
||||
public VideoBreakdownResultTranscriptBlock[] TranscriptBlocks { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultSentiment
|
||||
{
|
||||
[JsonProperty("sentimentKey")]
|
||||
public string SentimentKey { get; set; }
|
||||
|
||||
[JsonProperty("seenDurationRatio")]
|
||||
public float SeenDurationRatio { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,16 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultSummarizedInsights
|
||||
{
|
||||
[JsonProperty("faces")]
|
||||
public VideoBreakdownResultFace[] Faces { get; set; }
|
||||
|
||||
[JsonProperty("topics")]
|
||||
public VideoBreakdownResultTopic[] Topics { get; set; }
|
||||
|
||||
[JsonProperty("sentiments")]
|
||||
public VideoBreakdownResultSentiment[] Sentiments { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultTopic
|
||||
{
|
||||
[JsonProperty("name")]
|
||||
public string Name { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultTranscriptBlock
|
||||
{
|
||||
[JsonProperty("lines")]
|
||||
public VideoBreakdownResultTranscriptBlockLine[] Lines { get; set; }
|
||||
|
||||
[JsonProperty("sentiment")]
|
||||
public float Sentiment { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoBreakdownResultTranscriptBlockLine
|
||||
{
|
||||
[JsonProperty("text")]
|
||||
public string Text { get; set; }
|
||||
|
||||
[JsonProperty("sentiment")]
|
||||
public float Confidence { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,501 @@
|
|||
using System;
|
||||
using System.Collections;
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
|
||||
namespace Procurios.Public
|
||||
{
|
||||
/// <summary>
|
||||
/// This class encodes and decodes JSON strings.
|
||||
/// Spec. details, see http://www.json.org/
|
||||
///
|
||||
/// JSON uses Arrays and Objects. These correspond here to the datatypes ArrayList and Hashtable.
|
||||
/// All numbers are parsed to doubles.
|
||||
/// </summary>
|
||||
public class JSON
|
||||
{
|
||||
public const int TOKEN_NONE = 0;
|
||||
public const int TOKEN_CURLY_OPEN = 1;
|
||||
public const int TOKEN_CURLY_CLOSE = 2;
|
||||
public const int TOKEN_SQUARED_OPEN = 3;
|
||||
public const int TOKEN_SQUARED_CLOSE = 4;
|
||||
public const int TOKEN_COLON = 5;
|
||||
public const int TOKEN_COMMA = 6;
|
||||
public const int TOKEN_STRING = 7;
|
||||
public const int TOKEN_NUMBER = 8;
|
||||
public const int TOKEN_TRUE = 9;
|
||||
public const int TOKEN_FALSE = 10;
|
||||
public const int TOKEN_NULL = 11;
|
||||
|
||||
private const int BUILDER_CAPACITY = 2000;
|
||||
|
||||
/// <summary>
|
||||
/// Parses the string json into a value
|
||||
/// </summary>
|
||||
/// <param name="json">A JSON string.</param>
|
||||
/// <returns>An ArrayList, a Hashtable, a double, a string, null, true, or false</returns>
|
||||
public static object JsonDecode(string json)
|
||||
{
|
||||
bool success = true;
|
||||
|
||||
return JsonDecode(json, ref success);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Parses the string json into a value; and fills 'success' with the successfullness of the parse.
|
||||
/// </summary>
|
||||
/// <param name="json">A JSON string.</param>
|
||||
/// <param name="success">Successful parse?</param>
|
||||
/// <returns>An ArrayList, a Hashtable, a double, a string, null, true, or false</returns>
|
||||
public static object JsonDecode(string json, ref bool success)
|
||||
{
|
||||
success = true;
|
||||
if (json != null) {
|
||||
char[] charArray = json.ToCharArray();
|
||||
int index = 0;
|
||||
object value = ParseValue(charArray, ref index, ref success);
|
||||
return value;
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a Hashtable / ArrayList object into a JSON string
|
||||
/// </summary>
|
||||
/// <param name="json">A Hashtable / ArrayList</param>
|
||||
/// <returns>A JSON encoded string, or null if object 'json' is not serializable</returns>
|
||||
public static string JsonEncode(object json)
|
||||
{
|
||||
StringBuilder builder = new StringBuilder(BUILDER_CAPACITY);
|
||||
bool success = SerializeValue(json, builder);
|
||||
return (success ? builder.ToString() : null);
|
||||
}
|
||||
|
||||
protected static Hashtable ParseObject(char[] json, ref int index, ref bool success)
|
||||
{
|
||||
Hashtable table = new Hashtable();
|
||||
int token;
|
||||
|
||||
// {
|
||||
NextToken(json, ref index);
|
||||
|
||||
bool done = false;
|
||||
while (!done) {
|
||||
token = LookAhead(json, index);
|
||||
if (token == JSON.TOKEN_NONE) {
|
||||
success = false;
|
||||
return null;
|
||||
} else if (token == JSON.TOKEN_COMMA) {
|
||||
NextToken(json, ref index);
|
||||
} else if (token == JSON.TOKEN_CURLY_CLOSE) {
|
||||
NextToken(json, ref index);
|
||||
return table;
|
||||
} else {
|
||||
|
||||
// name
|
||||
string name = ParseString(json, ref index, ref success);
|
||||
if (!success) {
|
||||
success = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
// :
|
||||
token = NextToken(json, ref index);
|
||||
if (token != JSON.TOKEN_COLON) {
|
||||
success = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
// value
|
||||
object value = ParseValue(json, ref index, ref success);
|
||||
if (!success) {
|
||||
success = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
table[name] = value;
|
||||
}
|
||||
}
|
||||
|
||||
return table;
|
||||
}
|
||||
|
||||
protected static ArrayList ParseArray(char[] json, ref int index, ref bool success)
|
||||
{
|
||||
ArrayList array = new ArrayList();
|
||||
|
||||
// [
|
||||
NextToken(json, ref index);
|
||||
|
||||
bool done = false;
|
||||
while (!done) {
|
||||
int token = LookAhead(json, index);
|
||||
if (token == JSON.TOKEN_NONE) {
|
||||
success = false;
|
||||
return null;
|
||||
} else if (token == JSON.TOKEN_COMMA) {
|
||||
NextToken(json, ref index);
|
||||
} else if (token == JSON.TOKEN_SQUARED_CLOSE) {
|
||||
NextToken(json, ref index);
|
||||
break;
|
||||
} else {
|
||||
object value = ParseValue(json, ref index, ref success);
|
||||
if (!success) {
|
||||
return null;
|
||||
}
|
||||
|
||||
array.Add(value);
|
||||
}
|
||||
}
|
||||
|
||||
return array;
|
||||
}
|
||||
|
||||
protected static object ParseValue(char[] json, ref int index, ref bool success)
|
||||
{
|
||||
switch (LookAhead(json, index)) {
|
||||
case JSON.TOKEN_STRING:
|
||||
return ParseString(json, ref index, ref success);
|
||||
case JSON.TOKEN_NUMBER:
|
||||
return ParseNumber(json, ref index, ref success);
|
||||
case JSON.TOKEN_CURLY_OPEN:
|
||||
return ParseObject(json, ref index, ref success);
|
||||
case JSON.TOKEN_SQUARED_OPEN:
|
||||
return ParseArray(json, ref index, ref success);
|
||||
case JSON.TOKEN_TRUE:
|
||||
NextToken(json, ref index);
|
||||
return true;
|
||||
case JSON.TOKEN_FALSE:
|
||||
NextToken(json, ref index);
|
||||
return false;
|
||||
case JSON.TOKEN_NULL:
|
||||
NextToken(json, ref index);
|
||||
return null;
|
||||
case JSON.TOKEN_NONE:
|
||||
break;
|
||||
}
|
||||
|
||||
success = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
protected static string ParseString(char[] json, ref int index, ref bool success)
|
||||
{
|
||||
StringBuilder s = new StringBuilder(BUILDER_CAPACITY);
|
||||
char c;
|
||||
|
||||
EatWhitespace(json, ref index);
|
||||
|
||||
// "
|
||||
c = json[index++];
|
||||
|
||||
bool complete = false;
|
||||
while (!complete) {
|
||||
|
||||
if (index == json.Length) {
|
||||
break;
|
||||
}
|
||||
|
||||
c = json[index++];
|
||||
if (c == '"') {
|
||||
complete = true;
|
||||
break;
|
||||
} else if (c == '\\') {
|
||||
|
||||
if (index == json.Length) {
|
||||
break;
|
||||
}
|
||||
c = json[index++];
|
||||
if (c == '"') {
|
||||
s.Append('"');
|
||||
} else if (c == '\\') {
|
||||
s.Append('\\');
|
||||
} else if (c == '/') {
|
||||
s.Append('/');
|
||||
} else if (c == 'b') {
|
||||
s.Append('\b');
|
||||
} else if (c == 'f') {
|
||||
s.Append('\f');
|
||||
} else if (c == 'n') {
|
||||
s.Append('\n');
|
||||
} else if (c == 'r') {
|
||||
s.Append('\r');
|
||||
} else if (c == 't') {
|
||||
s.Append('\t');
|
||||
} else if (c == 'u') {
|
||||
int remainingLength = json.Length - index;
|
||||
if (remainingLength >= 4) {
|
||||
// parse the 32 bit hex into an integer codepoint
|
||||
uint codePoint;
|
||||
if (!(success = UInt32.TryParse(new string(json, index, 4), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out codePoint))) {
|
||||
return "";
|
||||
}
|
||||
// convert the integer codepoint to a unicode char and add to string
|
||||
s.Append(Char.ConvertFromUtf32((int)codePoint));
|
||||
// skip 4 chars
|
||||
index += 4;
|
||||
} else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
} else {
|
||||
s.Append(c);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (!complete) {
|
||||
success = false;
|
||||
return null;
|
||||
}
|
||||
|
||||
return s.ToString();
|
||||
}
|
||||
|
||||
protected static double ParseNumber(char[] json, ref int index, ref bool success)
|
||||
{
|
||||
EatWhitespace(json, ref index);
|
||||
|
||||
int lastIndex = GetLastIndexOfNumber(json, index);
|
||||
int charLength = (lastIndex - index) + 1;
|
||||
|
||||
double number;
|
||||
success = Double.TryParse(new string(json, index, charLength), NumberStyles.Any, CultureInfo.InvariantCulture, out number);
|
||||
|
||||
index = lastIndex + 1;
|
||||
return number;
|
||||
}
|
||||
|
||||
protected static int GetLastIndexOfNumber(char[] json, int index)
|
||||
{
|
||||
int lastIndex;
|
||||
|
||||
for (lastIndex = index; lastIndex < json.Length; lastIndex++) {
|
||||
if ("0123456789+-.eE".IndexOf(json[lastIndex]) == -1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
return lastIndex - 1;
|
||||
}
|
||||
|
||||
protected static void EatWhitespace(char[] json, ref int index)
|
||||
{
|
||||
for (; index < json.Length; index++) {
|
||||
if (" \t\n\r".IndexOf(json[index]) == -1) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected static int LookAhead(char[] json, int index)
|
||||
{
|
||||
int saveIndex = index;
|
||||
return NextToken(json, ref saveIndex);
|
||||
}
|
||||
|
||||
protected static int NextToken(char[] json, ref int index)
|
||||
{
|
||||
EatWhitespace(json, ref index);
|
||||
|
||||
if (index == json.Length) {
|
||||
return JSON.TOKEN_NONE;
|
||||
}
|
||||
|
||||
char c = json[index];
|
||||
index++;
|
||||
switch (c) {
|
||||
case '{':
|
||||
return JSON.TOKEN_CURLY_OPEN;
|
||||
case '}':
|
||||
return JSON.TOKEN_CURLY_CLOSE;
|
||||
case '[':
|
||||
return JSON.TOKEN_SQUARED_OPEN;
|
||||
case ']':
|
||||
return JSON.TOKEN_SQUARED_CLOSE;
|
||||
case ',':
|
||||
return JSON.TOKEN_COMMA;
|
||||
case '"':
|
||||
return JSON.TOKEN_STRING;
|
||||
case '0': case '1': case '2': case '3': case '4':
|
||||
case '5': case '6': case '7': case '8': case '9':
|
||||
case '-':
|
||||
return JSON.TOKEN_NUMBER;
|
||||
case ':':
|
||||
return JSON.TOKEN_COLON;
|
||||
}
|
||||
index--;
|
||||
|
||||
int remainingLength = json.Length - index;
|
||||
|
||||
// false
|
||||
if (remainingLength >= 5) {
|
||||
if (json[index] == 'f' &&
|
||||
json[index + 1] == 'a' &&
|
||||
json[index + 2] == 'l' &&
|
||||
json[index + 3] == 's' &&
|
||||
json[index + 4] == 'e') {
|
||||
index += 5;
|
||||
return JSON.TOKEN_FALSE;
|
||||
}
|
||||
}
|
||||
|
||||
// true
|
||||
if (remainingLength >= 4) {
|
||||
if (json[index] == 't' &&
|
||||
json[index + 1] == 'r' &&
|
||||
json[index + 2] == 'u' &&
|
||||
json[index + 3] == 'e') {
|
||||
index += 4;
|
||||
return JSON.TOKEN_TRUE;
|
||||
}
|
||||
}
|
||||
|
||||
// null
|
||||
if (remainingLength >= 4) {
|
||||
if (json[index] == 'n' &&
|
||||
json[index + 1] == 'u' &&
|
||||
json[index + 2] == 'l' &&
|
||||
json[index + 3] == 'l') {
|
||||
index += 4;
|
||||
return JSON.TOKEN_NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return JSON.TOKEN_NONE;
|
||||
}
|
||||
|
||||
protected static bool SerializeValue(object value, StringBuilder builder)
|
||||
{
|
||||
bool success = true;
|
||||
|
||||
if (value is string) {
|
||||
success = SerializeString((string)value, builder);
|
||||
} else if (value is Hashtable) {
|
||||
success = SerializeObject((Hashtable)value, builder);
|
||||
} else if (value is ArrayList) {
|
||||
success = SerializeArray((ArrayList)value, builder);
|
||||
} else if (value != null && value.GetType().IsArray){
|
||||
var tmp = new ArrayList();
|
||||
tmp.AddRange((ICollection)value);
|
||||
success = SerializeArray(tmp, builder);
|
||||
} else if (IsNumeric(value)) {
|
||||
success = SerializeNumber(Convert.ToDouble(value), builder);
|
||||
} else if ((value is Boolean) && ((Boolean)value == true)) {
|
||||
builder.Append("true");
|
||||
} else if ((value is Boolean) && ((Boolean)value == false)) {
|
||||
builder.Append("false");
|
||||
} else if (value == null) {
|
||||
builder.Append("null");
|
||||
} else {
|
||||
success = false;
|
||||
}
|
||||
return success;
|
||||
}
|
||||
|
||||
protected static bool SerializeObject(Hashtable anObject, StringBuilder builder)
|
||||
{
|
||||
builder.Append("{");
|
||||
|
||||
IDictionaryEnumerator e = anObject.GetEnumerator();
|
||||
bool first = true;
|
||||
while (e.MoveNext()) {
|
||||
string key = e.Key.ToString();
|
||||
object value = e.Value;
|
||||
|
||||
if (!first) {
|
||||
builder.Append(", ");
|
||||
}
|
||||
|
||||
SerializeString(key, builder);
|
||||
builder.Append(":");
|
||||
if (!SerializeValue(value, builder)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
first = false;
|
||||
}
|
||||
|
||||
builder.Append("}");
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static bool SerializeArray(ArrayList anArray, StringBuilder builder)
|
||||
{
|
||||
builder.Append("[");
|
||||
|
||||
bool first = true;
|
||||
for (int i = 0; i < anArray.Count; i++) {
|
||||
object value = anArray[i];
|
||||
|
||||
if (!first) {
|
||||
builder.Append(", ");
|
||||
}
|
||||
|
||||
if (!SerializeValue(value, builder)) {
|
||||
return false;
|
||||
}
|
||||
|
||||
first = false;
|
||||
}
|
||||
|
||||
builder.Append("]");
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static bool SerializeString(string aString, StringBuilder builder)
|
||||
{
|
||||
builder.Append("\"");
|
||||
|
||||
char[] charArray = aString.ToCharArray();
|
||||
for (int i = 0; i < charArray.Length; i++) {
|
||||
char c = charArray[i];
|
||||
if (c == '"') {
|
||||
builder.Append("\\\"");
|
||||
} else if (c == '\\') {
|
||||
builder.Append("\\\\");
|
||||
} else if (c == '\b') {
|
||||
builder.Append("\\b");
|
||||
} else if (c == '\f') {
|
||||
builder.Append("\\f");
|
||||
} else if (c == '\n') {
|
||||
builder.Append("\\n");
|
||||
} else if (c == '\r') {
|
||||
builder.Append("\\r");
|
||||
} else if (c == '\t') {
|
||||
builder.Append("\\t");
|
||||
} else {
|
||||
int codepoint = Convert.ToInt32(c);
|
||||
if ((codepoint >= 32) && (codepoint <= 126)) {
|
||||
builder.Append(c);
|
||||
} else {
|
||||
builder.Append("\\u" + Convert.ToString(codepoint, 16).PadLeft(4, '0'));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
builder.Append("\"");
|
||||
return true;
|
||||
}
|
||||
|
||||
protected static bool SerializeNumber(double number, StringBuilder builder)
|
||||
{
|
||||
builder.Append(Convert.ToString(number, CultureInfo.InvariantCulture));
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if a given object is numeric in any way
|
||||
/// (can be integer, double, null, etc).
|
||||
///
|
||||
/// Thanks to mtighe for pointing out Double.TryParse to me.
|
||||
/// </summary>
|
||||
protected static bool IsNumeric(object o)
|
||||
{
|
||||
double result;
|
||||
|
||||
return (o == null) ? false : Double.TryParse(o.ToString(), out result);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
using Crawl.VideoIndexer.Ooyala;
|
||||
using Ooyala.API;
|
||||
using System.Collections;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using static Microsoft.DecisionService.Crawl.VideoIndexer;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public static class Ooyala
|
||||
{
|
||||
private static Dictionary<string, string> EmptyDict = new Dictionary<string, string>();
|
||||
|
||||
public static OoyalaVideo GetOoyalaVideo(string assetId, VideoIndexerSettings settings)
|
||||
{
|
||||
if (settings == null || string.IsNullOrEmpty(settings.OoyalaKey) || string.IsNullOrEmpty(settings.OoyalaSecret))
|
||||
return null;
|
||||
|
||||
var api = new OoyalaAPI(settings.OoyalaKey, settings.OoyalaSecret);
|
||||
|
||||
var output = new OoyalaVideo();
|
||||
|
||||
var asset = api.get("assets/" + assetId, EmptyDict) as Hashtable;
|
||||
output.Description = asset?["description"] as string;
|
||||
|
||||
var metadata = api.get($"assets/{assetId}/metadata", EmptyDict) as Hashtable;
|
||||
output.Keywords = (metadata?["keywords"] as string)?.Split(',')?.ToList();
|
||||
|
||||
if (api.get($"assets/{assetId}/streams", EmptyDict) is ArrayList streams)
|
||||
{
|
||||
var q = from stream in streams.OfType<Hashtable>()
|
||||
let streamInfo = stream["is_source"] as bool?
|
||||
where streamInfo == true
|
||||
select stream;
|
||||
|
||||
var sourceStream = q.FirstOrDefault();
|
||||
output.Url = sourceStream?["url"] as string;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,451 @@
|
|||
/*
|
||||
|
||||
Copyright 2011 © Ooyala, Inc. All rights reserved.
|
||||
|
||||
Ooyala, Inc. (“Ooyala”) hereby grants permission, free of charge, to any person or entity obtaining a copy of the software code provided in source code format via this webpage and direct links contained within this webpage and any associated documentation (collectively, the "Software"), to use, copy, modify, merge, and/or publish the Software and, subject to pass-through of all terms and conditions hereof, permission to transfer, distribute and sublicense the Software; all of the foregoing subject to the following terms and conditions:
|
||||
|
||||
1. The above copyright notice and this permission notice shall be included in all copies or portions of the Software.
|
||||
|
||||
2. For purposes of clarity, the Software does not include any APIs, but instead consists of code that may be used in conjunction with APIs that may be provided by Ooyala pursuant to a separate written agreement subject to fees.
|
||||
|
||||
3. Ooyala may in its sole discretion maintain and/or update the Software. However, the Software is provided without any promise or obligation of support, maintenance or update.
|
||||
|
||||
4. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, TITLE, AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, RELATING TO, ARISING FROM, IN CONNECTION WITH, OR INCIDENTAL TO THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
5. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, (i) IN NO EVENT SHALL OOYALA BE LIABLE FOR ANY CONSEQUENTIAL, INCIDENTAL, INDIRECT, SPECIAL, PUNITIVE, OR OTHER DAMAGES WHATSOEVER (INCLUDING, WITHOUT LIMITATION, DAMAGES FOR LOSS OF BUSINESS PROFITS, BUSINESS INTERRUPTION, LOSS OF BUSINESS INFORMATION, OR OTHER PECUNIARY LOSS) RELATING TO, ARISING FROM, IN CONNECTION WITH, OR INCIDENTAL TO THE SOFTWARE OR THE USE OF OR INABILITY TO USE THE SOFTWARE, EVEN IF OOYALA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES, AND (ii) OOYALA’S TOTAL AGGREGATE LIABILITY RELATING TO, ARISING FROM, IN CONNECTION WITH, OR INCIDENTAL TO THE SOFTWARE SHALL BE LIMITED TO THE ACTUAL DIRECT DAMAGES INCURRED UP TO MAXIMUM AMOUNT OF FIFTY DOLLARS ($50).
|
||||
|
||||
* */
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Web;
|
||||
using System.Net;
|
||||
using System.IO;
|
||||
using System.Collections;
|
||||
using Procurios.Public;
|
||||
|
||||
namespace Ooyala.API
|
||||
{
|
||||
public class OoyalaAPI
|
||||
{
|
||||
public const string BASE_URL = "https://api.ooyala.com/v2/";
|
||||
|
||||
private const int UPLOAD_TIMEOUT = 1000 * 3600;
|
||||
|
||||
public string apiKey;
|
||||
public string secretKey;
|
||||
|
||||
private HttpWebResponse response;
|
||||
|
||||
/// <summary>
|
||||
/// Default constructor. You need to provide your API key and secret key. You can find this in the developer tab in Backlot
|
||||
/// <seealso cref="http://api.ooyala.com/docs/v2/"/>
|
||||
/// </summary>
|
||||
/// <param name="apiKey">
|
||||
/// Your account's API key
|
||||
/// </param>
|
||||
/// <param name="secretKey">
|
||||
/// Your account's secret key
|
||||
/// </param>
|
||||
public OoyalaAPI(string apiKey, string secretKey)
|
||||
{
|
||||
this.apiKey = apiKey;
|
||||
this.secretKey = secretKey;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes an HTTP GET request to the Ooyala API with the specified path and using the given parameters
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path to the resource to use for the GET request, i.e "players"
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Dictionary containing the parameters to be send along with the GET request.
|
||||
/// The only required parameter is expires since api_key will be added for you with the value you passed in the constructor of this class
|
||||
/// </param>
|
||||
public Object get(string path, Dictionary<String, String> parameters)
|
||||
{
|
||||
var url = this.generateURL("GET", path, parameters, "");
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "GET";
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Makes a POST request to the Ooyala V2 API. Post request are used to create objects like Assets, Labels, Players, etc.
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path to the resource to post to.
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Hash containing a list of parameters. The expires parameter is required.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// A String containing the JSON data to use for the creation of the object
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// returns the create object data in JSON format
|
||||
/// </returns>
|
||||
public Object post(string path, Dictionary<String, String> parameters, Hashtable body)
|
||||
{
|
||||
String jsonBody = JSON.JsonEncode(body);
|
||||
var url = this.generateURL("POST", path, parameters, jsonBody);
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "POST";
|
||||
|
||||
var data = Encoding.Default.GetBytes(jsonBody);
|
||||
|
||||
request.ContentLength = data.Length;
|
||||
|
||||
var stream = request.GetRequestStream();
|
||||
|
||||
stream.Write(data, 0, data.Length);
|
||||
|
||||
stream.Close();
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Makes a POST request sending bytes encoded in UTF8 to the Ooyala V2 API. This method can be used to upload preview images or closed caption files for assets.
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path to the resource to post the bytes to.
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Hash containing parameters to be sent along with the request. Required parameter: expires.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// An array of bytes representing the image/file you want to post to the API object.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A String containing the JSON response from the Server
|
||||
/// </returns>
|
||||
public Object postBytes(string path, Dictionary<String, String> parameters, System.Byte[] body, String fileName = null)
|
||||
{
|
||||
var url = this.generateURL("POST", path, parameters, Encoding.Default.GetString(body));
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "POST";
|
||||
request.AllowWriteStreamBuffering = false;
|
||||
request.Timeout = UPLOAD_TIMEOUT;
|
||||
request.SendChunked = false;
|
||||
|
||||
request.ContentLength = body.Length;
|
||||
|
||||
var stream = request.GetRequestStream();
|
||||
stream.Write(body, 0, body.Length);
|
||||
stream.Flush();
|
||||
stream.Close();
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Puts the bytes from the specified file to the API resource.
|
||||
/// </summary>
|
||||
/// <returns>
|
||||
/// The JSON response from the server in the form of an Object.
|
||||
/// </returns>
|
||||
/// <param name='path'>
|
||||
/// The path to the resource to put the bytes to.
|
||||
/// </param>
|
||||
/// <param name='parameters'>
|
||||
/// Query string parameters in the form of a Dictionary.
|
||||
/// </param>
|
||||
/// <param name='body'>
|
||||
/// A Hashtable containing the body of the request that will later be converted into JSON.
|
||||
/// </param>
|
||||
/// <param name='fileName'>
|
||||
/// The name of the file whose bytes will be sent via the PUT request.
|
||||
/// </param>
|
||||
public Object putBytes(string path, Dictionary<String, String> parameters, System.Byte[] body, String fileName = null)
|
||||
{
|
||||
var url = this.generateURL("PUT", path, parameters, Encoding.Default.GetString(body));
|
||||
|
||||
System.Net.Cache.RequestCachePolicy requestCachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.NoCacheNoStore);
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.CachePolicy = requestCachePolicy;
|
||||
request.Method = "PUT";
|
||||
request.AllowWriteStreamBuffering = false;
|
||||
request.Timeout = UPLOAD_TIMEOUT;
|
||||
request.SendChunked = false;
|
||||
|
||||
request.ContentLength = body.Length;
|
||||
|
||||
var stream = request.GetRequestStream();
|
||||
stream.Write(body, 0, body.Length);
|
||||
stream.Flush();
|
||||
stream.Close();
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Make a PATCH request to the Ooyala V2 API. PATCH requests are used to update data on objects. <seealso cref="hhttp://api.ooyala.com/docs/v2"/>
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path on the API server that indicates the object to be patched. In a patch request the path should look something like: :object_type/:object_id i.e. assets/zxhs16
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Hash containing the list of parameters to be sent along with the request. Required parameter: expires.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// A String containing the data to be sent for the patch in JSON format.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A String containing the JSON response from the server.
|
||||
/// </returns>
|
||||
public Object patch(string path, Dictionary<String, String> parameters, Hashtable body)
|
||||
{
|
||||
var url = this.generateURL("PATCH", path, parameters, body);
|
||||
String jsonBody = JSON.JsonEncode(body);
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "PATCH";
|
||||
request.ContentType = "application/x-www-form-urlencoded";
|
||||
|
||||
var data = Encoding.UTF8.GetBytes(jsonBody);
|
||||
|
||||
request.ContentLength = data.Length;
|
||||
|
||||
var stream = request.GetRequestStream();
|
||||
|
||||
stream.Write(data, 0, data.Length);
|
||||
|
||||
stream.Close();
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Make a PUT request to the Ooyala V2 API. PATCH requests are used to update data on objects. <seealso cref="http://api.ooyala.com/docs/v2"/>
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path on the API server that indicates the object to be replaced. In a put request the path should look something like: :object_type/:object_id i.e. assets/zxhs16
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Hash containing the list of parameters to be sent along with the request. Required parameter: expires.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// A String containing the data to be sent for the put in JSON format.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A String containing the JSON response from the server.
|
||||
/// </returns>
|
||||
public Object put(string path, Dictionary<String, String> parameters, Hashtable body)
|
||||
{
|
||||
var url = this.generateURL("PUT", path, parameters, body);
|
||||
|
||||
String jsonBody = JSON.JsonEncode(body);
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "PUT";
|
||||
request.ContentType = "application/x-www-form-urlencoded";
|
||||
|
||||
var data = Encoding.UTF8.GetBytes(jsonBody);
|
||||
|
||||
request.ContentLength = data.Length;
|
||||
|
||||
var stream = request.GetRequestStream();
|
||||
|
||||
stream.Write(data, 0, data.Length);
|
||||
|
||||
stream.Close();
|
||||
|
||||
if (this.getResponse(request))
|
||||
{
|
||||
return JSON.JsonDecode(new StreamReader(response.GetResponseStream()).ReadToEnd());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Issues a DELETE request to remove an object using the Ooyala V2 API. <seealso cref="http://api.ooyala.com/docs/v2"/>
|
||||
/// </summary>
|
||||
/// <param name="path">
|
||||
/// The path of the resource to delete
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A Hash containing the list of parameters to be sent along with the request. Required parameter: expires.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// True if the asset was deleted, false if it was already deleted or does not exist
|
||||
/// </returns>
|
||||
public Boolean delete(string path, Dictionary<String, String> parameters)
|
||||
{
|
||||
var url = this.generateURL("DELETE", path, parameters, "");
|
||||
|
||||
HttpWebRequest request = HttpWebRequest.Create(url) as HttpWebRequest;
|
||||
request.Method = "DELETE";
|
||||
|
||||
return this.getResponse(request);
|
||||
}
|
||||
|
||||
private string generateURL(string HTTPMethod, string path, Dictionary<System.String, System.String> parameters, Hashtable body)
|
||||
{
|
||||
return generateURL(HTTPMethod, path, parameters, JSON.JsonEncode(body));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes in the necessary parameters to build a V2 signature for the Ooyala API
|
||||
/// </summary>
|
||||
/// <param name="HTTPMethod">
|
||||
/// The method to be used for the request. Possible values are: GET, POST, PUT, PATCH or DELETE
|
||||
/// </param>
|
||||
/// <param name="path">
|
||||
/// The path to use for the request
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A hash containing the list of parameters that will be included in the request.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// A string containing the JSON representation of the data to be sent on the request. If its a GET request, the body parameter will not be used to generate the signature.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// The URL to be used in the HTTP request.
|
||||
/// </returns>
|
||||
private string generateURL(string HTTPMethod, string path, Dictionary<System.String, System.String> parameters, String body)
|
||||
{
|
||||
var url = BASE_URL + path;
|
||||
|
||||
path = "/v2/" + path;
|
||||
|
||||
if (!parameters.ContainsKey("api_key"))
|
||||
{
|
||||
parameters.Add("api_key", this.apiKey);
|
||||
}
|
||||
|
||||
if (!parameters.ContainsKey("expires"))
|
||||
{
|
||||
DateTime now = DateTime.UtcNow;
|
||||
//Round up to the expiration to the next hour for higher caching performance
|
||||
DateTime expiresWindow = new DateTime(now.Year, now.Month, now.Day, now.Hour, 0, 0);
|
||||
expiresWindow = expiresWindow.AddHours(1);
|
||||
int expires = (int)(expiresWindow - new DateTime(1970, 1, 1)).TotalSeconds;
|
||||
parameters.Add("expires", expires.ToString());
|
||||
}
|
||||
|
||||
//Sorting the keys
|
||||
var sortedKeys = new String[parameters.Keys.Count];
|
||||
parameters.Keys.CopyTo(sortedKeys, 0);
|
||||
Array.Sort(sortedKeys);
|
||||
|
||||
for (int i = 0; i < sortedKeys.Length; i++)
|
||||
{
|
||||
url += (i == 0 && !url.Contains("?") ? "?" : "&") + sortedKeys[i] + "=" + HttpUtility.UrlEncode(parameters[sortedKeys[i]]);
|
||||
}
|
||||
|
||||
url += "&signature=" + this.generateRequestSignature(HTTPMethod, path, sortedKeys, parameters, body);
|
||||
|
||||
return url;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Generates the signature for the V2 API request based on the given parameters. <seealso cref="http://api.ooyala.com/v2/docs"/>
|
||||
/// </summary>
|
||||
/// <param name="HTTPMethod">
|
||||
/// The method to be used for the request. Possible values are: GET, POST, PUT, PATCH or DELETE
|
||||
/// </param>
|
||||
/// <param name="path">
|
||||
/// The path to use for the request
|
||||
/// </param>
|
||||
/// <param name="sortedParameterKeys">
|
||||
/// A sorted array containing the keys of the parameters hash. This is to improve efficiency and not sort them twice since generateURL already does it.
|
||||
/// </param>
|
||||
/// <param name="parameters">
|
||||
/// A hash containing the list of parameters that will be included in the request.
|
||||
/// </param>
|
||||
/// <param name="body">
|
||||
/// A string containing the JSON representation of the data to be sent on the request. If its a GET request, the body parameter will not be used to generate the signature.
|
||||
/// </param>
|
||||
/// <returns>
|
||||
/// A string containing the signature to be used in the V2 API request.
|
||||
/// </returns>
|
||||
internal string generateRequestSignature(string HTTPMethod, String path, String[] sortedParameterKeys, Dictionary<String, String> parameters, String body)
|
||||
{
|
||||
var stringToSign = this.secretKey + HTTPMethod + path;
|
||||
|
||||
for (int i = 0; i < sortedParameterKeys.Length; i++)
|
||||
{
|
||||
stringToSign += sortedParameterKeys[i] + "=" + parameters[sortedParameterKeys[i]];
|
||||
}
|
||||
|
||||
stringToSign += body;
|
||||
|
||||
var sha256 = new SHA256Managed();
|
||||
byte[] digest = sha256.ComputeHash(Encoding.Default.GetBytes(stringToSign));
|
||||
string signedInput = Convert.ToBase64String(digest);
|
||||
|
||||
//Removing the trailing = signs
|
||||
var lastEqualsSignindex = signedInput.Length - 1;
|
||||
while (signedInput[lastEqualsSignindex] == '=')
|
||||
{
|
||||
lastEqualsSignindex--;
|
||||
}
|
||||
|
||||
signedInput = signedInput.Substring(0, lastEqualsSignindex + 1);
|
||||
|
||||
return HttpUtility.UrlEncode(signedInput.Substring(0, 43));
|
||||
}
|
||||
|
||||
private Boolean getResponse(HttpWebRequest request)
|
||||
{
|
||||
try
|
||||
{
|
||||
response = request.GetResponse() as HttpWebResponse;
|
||||
return true;
|
||||
}
|
||||
catch (WebException e)
|
||||
{
|
||||
Console.WriteLine("Exception Message :" + e.Message);
|
||||
if (e.Status == WebExceptionStatus.ProtocolError)
|
||||
{
|
||||
var response = ((HttpWebResponse)e.Response);
|
||||
Console.WriteLine("Status Code : {0}", ((HttpWebResponse)e.Response).StatusCode);
|
||||
Console.WriteLine("Status Description : {0}", ((HttpWebResponse)e.Response).StatusDescription);
|
||||
|
||||
var stream = response.GetResponseStream();
|
||||
var reader = new StreamReader(stream);
|
||||
var text = reader.ReadToEnd();
|
||||
Console.WriteLine("Response Description : {0}", text);
|
||||
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using System.Collections.Generic;
|
||||
|
||||
namespace Crawl.VideoIndexer.Ooyala
|
||||
{
|
||||
public class OoyalaVideo
|
||||
{
|
||||
public string Url { get; set; }
|
||||
|
||||
public List<string> Keywords { get; set; }
|
||||
|
||||
public string Description { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoIndexerSearchResult
|
||||
{
|
||||
[JsonProperty("results")]
|
||||
public VideoIndexerSearchResultItem[] Results { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,13 @@
|
|||
using Newtonsoft.Json;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
/// <summary>
|
||||
/// There are more fields here
|
||||
/// </summary>
|
||||
public class VideoIndexerSearchResultItem
|
||||
{
|
||||
[JsonProperty("id")]
|
||||
public string Id { get; set; }
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,190 @@
|
|||
using System.Threading.Tasks;
|
||||
using Microsoft.Azure.WebJobs.Host;
|
||||
using System.Net.Http;
|
||||
using Newtonsoft.Json;
|
||||
using Microsoft.DecisionService.Crawl.Data;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System.Linq;
|
||||
using System;
|
||||
using System.Threading;
|
||||
using Microsoft.WindowsAzure.Storage;
|
||||
using Microsoft.ApplicationInsights.DataContracts;
|
||||
using Microsoft.ApplicationInsights;
|
||||
using Crawl.VideoIndexer;
|
||||
using System.Web;
|
||||
|
||||
namespace Microsoft.DecisionService.Crawl
|
||||
{
|
||||
public class VideoIndexer
|
||||
{
|
||||
private static readonly CognitiveService cogService;
|
||||
|
||||
public class VideoIndexerSettings
|
||||
{
|
||||
[JsonProperty("key")]
|
||||
public string VideoIndexerKey { get; set; }
|
||||
|
||||
[JsonProperty("ooyalaKey")]
|
||||
public string OoyalaKey { get; set; }
|
||||
|
||||
[JsonProperty("ooyalaSecret")]
|
||||
public string OoyalaSecret { get; set; }
|
||||
}
|
||||
|
||||
static VideoIndexer()
|
||||
{
|
||||
cogService = new CognitiveService("VideoIndexer");
|
||||
}
|
||||
|
||||
private static async Task<VideoIndexerSettings> GetVideoIndexerSettings(string appId)
|
||||
{
|
||||
var account = CloudStorageAccount.Parse(await cogService.GetAzureStorageConnectionStringAsync());
|
||||
var blobClient = account.CreateCloudBlobClient();
|
||||
|
||||
var keyContainer = blobClient.GetContainerReference("keys");
|
||||
if (!await keyContainer.ExistsAsync())
|
||||
return null;
|
||||
|
||||
var videoIndexerConfig = keyContainer.GetBlockBlobReference($"videoindexer.{appId}.json");
|
||||
if (!videoIndexerConfig.Exists())
|
||||
return null;
|
||||
|
||||
return JsonConvert.DeserializeObject<VideoIndexerSettings>(await videoIndexerConfig.DownloadTextAsync());
|
||||
}
|
||||
|
||||
private static CognitiveService GetCognitiveService(VideoIndexerSettings settings)
|
||||
{
|
||||
var localCogService = cogService;
|
||||
var videoIndexerKey = settings?.VideoIndexerKey;
|
||||
if (!string.IsNullOrEmpty(videoIndexerKey))
|
||||
localCogService = new CognitiveService("VideoIndexer", apiKey: videoIndexerKey);
|
||||
|
||||
return localCogService;
|
||||
}
|
||||
|
||||
private static async Task<BlobContent> GetVideoIndexerBreakdownAsync(CrawlResponse reqBody, VideoIndexerSettings settings, TraceWriter log, CancellationToken cancellationToken)
|
||||
{
|
||||
using (var operation = Services.TelemetryClient.StartOperation<DependencyTelemetry>("Crawl.VideoIndexer.GetBreakdown"))
|
||||
{
|
||||
var localCogService = GetCognitiveService(settings);
|
||||
|
||||
var searchContent = await localCogService.RequestAsync(
|
||||
log,
|
||||
reqBody.Site,
|
||||
reqBody.Id + "ext", // make sure the blob names are unique
|
||||
$"/Breakdowns/Api/Partner/Breakdowns/Search?externalId={reqBody.Id}",
|
||||
reqBody.ForceRefresh,
|
||||
isPost: false,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
var videoIndexerResponse = JsonConvert.DeserializeObject<VideoIndexerSearchResult>(searchContent.Value);
|
||||
var breakdownId = videoIndexerResponse.Results?.FirstOrDefault()?.Id;
|
||||
if (breakdownId == null)
|
||||
return null;
|
||||
|
||||
return await localCogService.RequestAsync(
|
||||
log,
|
||||
reqBody.Site,
|
||||
reqBody.Id + "br", // make sure the blob names are unique
|
||||
$"/Breakdowns/Api/Partner/Breakdowns/{breakdownId}",
|
||||
reqBody.ForceRefresh,
|
||||
isPost: false,
|
||||
cancellationToken: cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task IndexVideo(CrawlResponse reqBody, VideoIndexerSettings settings)
|
||||
{
|
||||
if (reqBody == null || string.IsNullOrEmpty(reqBody.Video))
|
||||
return;
|
||||
|
||||
using (var operation = Services.TelemetryClient.StartOperation<DependencyTelemetry>("Crawl.VideoIndexer.Enqueue"))
|
||||
{
|
||||
// https://videobreakdown.azure-api.net/Breakdowns/Api/Partner/Breakdowns[?name][&privacy][&videoUrl][&language][&externalId][&metadata][&description][&partition][&callbackUrl][&indexingPreset][&streamingPreset]
|
||||
var url = HttpUtility.UrlEncode(reqBody.Video);
|
||||
var id = HttpUtility.UrlEncode(reqBody.Id);
|
||||
var query =
|
||||
"Breakdowns/Api/Partner/Breakdowns" +
|
||||
$"?name={id}&externalId={id}" +
|
||||
$"&videoUrl={url}" +
|
||||
"&privacy=private&searchable=true";
|
||||
|
||||
if (!string.IsNullOrEmpty(reqBody.Description))
|
||||
query += "&description=" + reqBody.Description;
|
||||
|
||||
if (reqBody.Categories != null && reqBody.Categories.Count > 0)
|
||||
query += "&metadata=" + HttpUtility.UrlEncode(string.Join(" ", reqBody.Categories));
|
||||
|
||||
var localCogService = GetCognitiveService(settings);
|
||||
|
||||
var client = await localCogService.GetHttpClientAsync();
|
||||
var httpResponse = await client.PostAsync(client.BaseAddress + query, new MultipartFormDataContent());
|
||||
|
||||
operation.Telemetry.Success = httpResponse.IsSuccessStatusCode;
|
||||
operation.Telemetry.ResultCode = httpResponse.StatusCode.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task<HttpResponseMessage> Run(HttpRequestMessage req, TraceWriter log, CancellationToken cancellationToken)
|
||||
{
|
||||
string reqBodyStr = null;
|
||||
CrawlResponse reqBody = null;
|
||||
|
||||
try
|
||||
{
|
||||
using (var operation = Services.TelemetryClient.StartOperation<DependencyTelemetry>("Crawl.VideoIndexer"))
|
||||
{
|
||||
// TODO: if the id is not parsable, just ignore - make sure the others do too
|
||||
|
||||
reqBodyStr = await req.Content.ReadAsStringAsync();
|
||||
reqBody = JsonConvert.DeserializeObject<CrawlResponse>(reqBodyStr);
|
||||
|
||||
operation.Telemetry.Properties.Add("AppId", reqBody.Site);
|
||||
operation.Telemetry.Properties.Add("ActionId", reqBody.Id);
|
||||
|
||||
var settings = await GetVideoIndexerSettings(reqBody.Site);
|
||||
|
||||
// find existing breakdown
|
||||
var breakdownContent = await GetVideoIndexerBreakdownAsync(reqBody, settings, log, cancellationToken);
|
||||
|
||||
if (breakdownContent == null)
|
||||
{
|
||||
// enqueue break down indexing
|
||||
if (string.IsNullOrEmpty(reqBody.Video))
|
||||
{
|
||||
var ooyalaVideo = Ooyala.GetOoyalaVideo(reqBody.Id, settings);
|
||||
reqBody.Video = ooyalaVideo.Url;
|
||||
|
||||
if (string.IsNullOrEmpty(reqBody.Description))
|
||||
reqBody.Description = ooyalaVideo.Description;
|
||||
|
||||
if (reqBody.Categories == null || reqBody.Categories.Count == 0)
|
||||
reqBody.Categories = ooyalaVideo.Keywords;
|
||||
}
|
||||
|
||||
await IndexVideo(reqBody, settings);
|
||||
|
||||
// make sure caller comes back in 5min
|
||||
return Services.CreateResponse(new BlobContent { Expires = DateTime.UtcNow + TimeSpan.FromMinutes(5) });
|
||||
}
|
||||
|
||||
var result = JsonConvert.DeserializeObject<VideoBreakdownResult>(breakdownContent.Value);
|
||||
if (result.State != "Processed")
|
||||
// make sure caller comes back in 5min
|
||||
return Services.CreateResponse(new BlobContent { Expires = DateTime.UtcNow + TimeSpan.FromMinutes(5) });
|
||||
|
||||
|
||||
// featurize
|
||||
breakdownContent.Output = VideoIndexerFeaturizer.FeaturizeVideoIndexerBreakdown(result);
|
||||
|
||||
return Services.CreateResponse(breakdownContent);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Services.TrackException(ex, req, log, reqBodyStr, reqBody);
|
||||
throw ex;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,110 @@
|
|||
using Microsoft.DecisionService.Crawl;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Web;
|
||||
|
||||
namespace Crawl.VideoIndexer
|
||||
{
|
||||
public static class VideoIndexerFeaturizer
|
||||
{
|
||||
public static JObject FeaturizeVideoIndexerBreakdown(VideoBreakdownResult result)
|
||||
{
|
||||
var output = new JObject();
|
||||
|
||||
output.Add(new JProperty("durationInSeconds", result.DurationInSeconds));
|
||||
if (result.SummarizedInsights != null)
|
||||
{
|
||||
if (result.SummarizedInsights.Faces != null)
|
||||
{
|
||||
var lfaces = result.SummarizedInsights.Faces
|
||||
.Where(f => !f.Name.StartsWith("Unknown"))
|
||||
.GroupBy(f => f.Name)
|
||||
.Select(f =>
|
||||
{
|
||||
var x = f.First();
|
||||
return new JProperty(x.Name, x.SeenDurationRatio);
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
if (lfaces.Length > 0)
|
||||
output.Add(new JProperty("LFacesRatio", new JObject(lfaces)));
|
||||
|
||||
var faces = result.SummarizedInsights.Faces
|
||||
.Where(f => !f.Name.StartsWith("Unknown"))
|
||||
.GroupBy(f => f.Name)
|
||||
.Select(f =>
|
||||
{
|
||||
var x = f.First();
|
||||
return new[]
|
||||
{
|
||||
new JProperty(x.Name, x.SeenDuration)
|
||||
};
|
||||
})
|
||||
.ToArray();
|
||||
|
||||
if (faces.Length > 0)
|
||||
output.Add(new JProperty("HFacesDuration", new JObject(faces)));
|
||||
}
|
||||
|
||||
if (result.SummarizedInsights.Topics != null)
|
||||
{
|
||||
output.Add(new JProperty("JTopics",
|
||||
new JObject(result.SummarizedInsights.Topics
|
||||
.Select(t => t.Name)
|
||||
.Distinct()
|
||||
.Select(t => new JProperty(t, 1)))));
|
||||
}
|
||||
|
||||
if (result.SummarizedInsights.Sentiments != null)
|
||||
{
|
||||
output.Add(new JProperty("KSentiments",
|
||||
new JObject(
|
||||
result.SummarizedInsights.Sentiments
|
||||
.GroupBy(f => f.SentimentKey)
|
||||
.Select(f =>
|
||||
{
|
||||
var x = f.First();
|
||||
return new JProperty(x.SentimentKey, x.SeenDurationRatio);
|
||||
}))));
|
||||
}
|
||||
}
|
||||
|
||||
if (result.Breakdowns != null)
|
||||
{
|
||||
var bd = result.Breakdowns.FirstOrDefault();
|
||||
if (bd != null)
|
||||
{
|
||||
if (bd.Insight != null)
|
||||
{
|
||||
if (bd.Insight.ContentModeration != null)
|
||||
output.Add(new JProperty("BContentModeration",
|
||||
new JObject(
|
||||
new JProperty("AdultClassifierValue", bd.Insight.ContentModeration.AdultClassifierValue),
|
||||
new JProperty("BannedWordsCount", bd.Insight.ContentModeration.BannedWordsCount),
|
||||
new JProperty("IsAdult", bd.Insight.ContentModeration.IsAdult),
|
||||
new JProperty("IsSuspectedAsAdult", bd.Insight.ContentModeration.IsSuspectedAsAdult))));
|
||||
|
||||
if (bd.Insight.TranscriptBlocks != null)
|
||||
{
|
||||
var lines = bd.Insight.TranscriptBlocks
|
||||
.Where(t => t.Lines != null)
|
||||
.SelectMany(t => t.Lines)
|
||||
.Select(t => t.Text)
|
||||
.Where(l => !string.IsNullOrWhiteSpace(l))
|
||||
.ToArray();
|
||||
|
||||
if (lines.Length > 0)
|
||||
output.Add(new JProperty("Text",
|
||||
new JObject(
|
||||
new JProperty("_text", string.Join(" ", lines)))));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -0,0 +1,18 @@
|
|||
{
|
||||
"scriptFile": "..\\bin\\Crawl.dll",
|
||||
"entryPoint": "Microsoft.DecisionService.Crawl.VideoIndexer.Run",
|
||||
"bindings": [
|
||||
{
|
||||
"authLevel": "function",
|
||||
"name": "req",
|
||||
"type": "httpTrigger",
|
||||
"direction": "in"
|
||||
},
|
||||
{
|
||||
"name": "$return",
|
||||
"type": "http",
|
||||
"direction": "out"
|
||||
}
|
||||
],
|
||||
"disabled": false
|
||||
}
|
||||
Загрузка…
Ссылка в новой задаче