Removing Dispatcher from master (#931)
* Removing dispatcher from master * Updating rush.json * Changes to lock.yaml
This commit is contained in:
Emilio Munoz
GitHub
Родитель
5a7c9736ae
Коммит
e3d7c37c2d
|
|
@ -9,7 +9,6 @@ dependencies:
|
|||
'@rush-temp/bf-cli-config': 'file:projects/bf-cli-config.tgz'
|
||||
'@rush-temp/bf-cli-plugins': 'file:projects/bf-cli-plugins.tgz'
|
||||
'@rush-temp/bf-dialog': 'file:projects/bf-dialog.tgz'
|
||||
'@rush-temp/bf-dispatcher': 'file:projects/bf-dispatcher.tgz'
|
||||
'@rush-temp/bf-lg-cli': 'file:projects/bf-lg-cli.tgz'
|
||||
'@rush-temp/bf-lu': 'file:projects/bf-lu.tgz'
|
||||
'@rush-temp/bf-luis-cli': 'file:projects/bf-luis-cli.tgz'
|
||||
|
|
@ -17,7 +16,6 @@ dependencies:
|
|||
'@rush-temp/botframework-cli': 'file:projects/botframework-cli.tgz'
|
||||
'@snyk/nuget-semver': 1.3.0
|
||||
'@types/ansi-styles': 3.2.1
|
||||
'@types/argparse': 1.0.38
|
||||
'@types/lru-cache': 5.1.0
|
||||
'@types/proxyquire': 1.3.28
|
||||
'@types/readline-sync': 1.4.3
|
||||
|
|
@ -27,7 +25,6 @@ dependencies:
|
|||
ajv: 6.12.2
|
||||
antlr4: 4.8.0
|
||||
applicationinsights: 1.7.3
|
||||
argparse: 1.0.10
|
||||
await-delay: 1.0.0
|
||||
botbuilder-lg: 4.8.0-preview
|
||||
botframework-schema: 4.7.2
|
||||
|
|
@ -70,7 +67,6 @@ dependencies:
|
|||
sinon: 7.5.0
|
||||
source-map-support: 0.5.16
|
||||
testdouble: 3.13.0
|
||||
ts-md5: 1.2.7
|
||||
username: 4.1.0
|
||||
uuid: 3.4.0
|
||||
window-size: 1.1.1
|
||||
|
|
@ -590,10 +586,6 @@ packages:
|
|||
dev: false
|
||||
resolution:
|
||||
integrity: sha512-UFa7mfKgSutXdT+elzJo8Ulr7FHgLNAyglVIOZYXFNJVQERm8DPrcwPret5BYk66LBE7fwm1XoVGi76MJkQ6ow==
|
||||
/@types/argparse/1.0.38:
|
||||
dev: false
|
||||
resolution:
|
||||
integrity: sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==
|
||||
/@types/chai/4.2.10:
|
||||
dev: false
|
||||
resolution:
|
||||
|
|
@ -5817,10 +5809,6 @@ packages:
|
|||
node: '>=0.6'
|
||||
resolution:
|
||||
integrity: sha512-1m4RA7xVAJrSGrrXGs0L3YTwyvBs2S8PbRHaLZAkFw7JR8oIFwYtysxlBZhYIa7xSyiYJKZ3iGrrk55cGA3i9A==
|
||||
/ts-md5/1.2.7:
|
||||
dev: false
|
||||
resolution:
|
||||
integrity: sha512-emODogvKGWi1KO1l9c6YxLMBn6CEH3VrH5mVPIyOtxBG52BvV4jP3GWz6bOZCz61nLgBc3ffQYE4+EHfCD+V7w==
|
||||
/ts-node/8.6.2_typescript@3.5.3:
|
||||
dependencies:
|
||||
arg: 4.1.3
|
||||
|
|
@ -6515,7 +6503,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-chatdown'
|
||||
resolution:
|
||||
integrity: sha512-VrCjZtPd0E6O9F69w8dxYnoBFt6ejLSeinZ4Ec5CWotDhvbLqJ4Dif8Q7rcGYcsIkKkZusIxzDNUg/YPB0/1Qw==
|
||||
integrity: sha512-OhMWgCo6m7d8YRxa/ZoHuVyeBr8taeCH+u6zBHZNOXpSkL8iBRK2j5v3dp0zFHEqZkc/pcpF7Iwtu4D7FIt+2Q==
|
||||
tarball: 'file:projects/bf-chatdown.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-cli-command.tgz':
|
||||
|
|
@ -6556,7 +6544,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-cli-command'
|
||||
resolution:
|
||||
integrity: sha512-WvkxK/1q4R52y/2qKSfH4cRsul7ilKOvQdyFgph0LwEFEVp+K01ET0vvWV3Q6Ie0tMqEV51brdP6+otv7m5gFw==
|
||||
integrity: sha512-E4dGSDdlLTl2Dpeh7MSmoh19SO4XbA5wxwS5QMdDaV/uz1NdjSTVv8nz9AN7uHqUtLwZEmH/m6ixIrqyJuHL4w==
|
||||
tarball: 'file:projects/bf-cli-command.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-cli-config.tgz':
|
||||
|
|
@ -6585,7 +6573,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-cli-config'
|
||||
resolution:
|
||||
integrity: sha512-ZvUZsW96vljeLO69GRz/kfBazChVv9O6BkikOIQCHUbUP32PzIh8a0jjEmppA+/RtqajtmKQWGq0q5DEQY4x5A==
|
||||
integrity: sha512-vwZf1fXZ7BWQsJjKHFMEQ95zRfQ48Na7ae9cMH36cw/fk/a6MYUkzwSgSabel0S2D8uT/BodVucYXJXPfl9NTw==
|
||||
tarball: 'file:projects/bf-cli-config.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-cli-plugins.tgz':
|
||||
|
|
@ -6614,7 +6602,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-cli-plugins'
|
||||
resolution:
|
||||
integrity: sha512-2n8WG0y6vxBpyUrSJLVWwLcwUN+FqR8wI5htyMvDmPJOBp/PnH4WrupuQXDhl0UacI6LBhI4RWiZtunIkJ1zBw==
|
||||
integrity: sha512-fegDj5NfXQCsabktk5fStFptRwmJccyH0qzjDqasi0mhTjLi72+ucovjYrzbVmrA73LdOoIsRIAT8tuCfLkkMg==
|
||||
tarball: 'file:projects/bf-cli-plugins.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-dialog.tgz':
|
||||
|
|
@ -6669,39 +6657,9 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-dialog'
|
||||
resolution:
|
||||
integrity: sha512-GZ5UbbR1y0ayFNaRVmDaxEzeS4xmOT2ri2SbFM+nyg8BRmOD1fnKzf2GO9bT6/b4VcwbaYop08ANlHtLI80k2w==
|
||||
integrity: sha512-dgaJL7Qsjv/BIQolqX0EChhE3yL39e1JsbOlj6EL5dSzD6YYQj4aX/tLBFFv8ZQhLExPOwtjmS20/dEYmUj6yA==
|
||||
tarball: 'file:projects/bf-dialog.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-dispatcher.tgz':
|
||||
dependencies:
|
||||
'@oclif/command': 1.5.19
|
||||
'@oclif/config': 1.13.3
|
||||
'@oclif/dev-cli': 1.22.2
|
||||
'@oclif/plugin-help': 2.2.3
|
||||
'@oclif/test': 1.2.5
|
||||
'@oclif/tslint': 3.1.1_tslint@5.20.1+typescript@3.8.3
|
||||
'@types/argparse': 1.0.38
|
||||
'@types/chai': 4.2.10
|
||||
'@types/mocha': 5.2.7
|
||||
'@types/node': 10.17.17
|
||||
argparse: 1.0.10
|
||||
chai: 4.2.0
|
||||
globby: 10.0.2
|
||||
mocha: 5.2.0
|
||||
nyc: 14.1.1
|
||||
readline-sync: 1.4.10
|
||||
rimraf: 3.0.2
|
||||
ts-md5: 1.2.7
|
||||
ts-node: 8.6.2_typescript@3.8.3
|
||||
tslib: 1.11.1
|
||||
tslint: 5.20.1_typescript@3.8.3
|
||||
typescript: 3.8.3
|
||||
dev: false
|
||||
name: '@rush-temp/bf-dispatcher'
|
||||
resolution:
|
||||
integrity: sha512-hg92Iz2+Bxg8zpmNG177Ot2ZpQj0Q4/cnx1gN5Y9swG6k0ri0Ngt4s4XkByRzY3LkOs3o3GB9W8qE6KMLEq4XA==
|
||||
tarball: 'file:projects/bf-dispatcher.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-lg-cli.tgz':
|
||||
dependencies:
|
||||
'@oclif/command': 1.5.19
|
||||
|
|
@ -6738,7 +6696,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-lg-cli'
|
||||
resolution:
|
||||
integrity: sha512-no9KfuRbMshP8AveSCo5f28r38fFdQAhoNREE+hCJQue+sD2Qb8wqo8Lc2MhhqqTYUf0qFoSrWvRxCAi7UJ3Sw==
|
||||
integrity: sha512-kkytBYgClJ8lz6Tm1hgrwkNZ1CqHfHUPxvKPqB1X8E7R5UQjZjkoqmha8lwOX9cXnHH4ZwoB1/EulURSgXegmg==
|
||||
tarball: 'file:projects/bf-lg-cli.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-lu.tgz':
|
||||
|
|
@ -6780,7 +6738,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-lu'
|
||||
resolution:
|
||||
integrity: sha512-qsE6Tlrv+CymCGvmTV1duqjTLuOTc0BPFCOjjlp01dpzbnpR1nwVnudVnTn/QRpWUTTe2QqF2KY6NB4aUwkBxg==
|
||||
integrity: sha512-4LEnz+VdExYEDyZnTNu7LE7qprjEJvIO8Zyqr9HEnInbe0HcaGWBMWq2qlfadZ6Yj0defG00+Y9NFrke0ndE9w==
|
||||
tarball: 'file:projects/bf-lu.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-luis-cli.tgz':
|
||||
|
|
@ -6823,7 +6781,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-luis-cli'
|
||||
resolution:
|
||||
integrity: sha512-BqW+TgfhTF1W2QA/C/S/gzeVBJbY+qB8nSCLrddsFmjLsl2fAy/QCKmafTUVb5Hlh1KQAEQ4gFVMmHkus2VK3g==
|
||||
integrity: sha512-sD6R3xSk6SIbrBIxZaFNSvmFqIOdqySnJddQJ0349GySy27nyOchXIVdeWsjAQrdeuYZJISfM3B2mmevBx5MdQ==
|
||||
tarball: 'file:projects/bf-luis-cli.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/bf-qnamaker.tgz':
|
||||
|
|
@ -6873,7 +6831,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/bf-qnamaker'
|
||||
resolution:
|
||||
integrity: sha512-c/0EeZ4yrqB2IncE00iYze3Fe2YUFIaAddv4+s9XD/a5LuomQiq1qEGROxJzqBeXyMeod9hn5XcYhNMbApWXEQ==
|
||||
integrity: sha512-OVMhTkKSx1fq8vl5VnW3vNu8tMNBAD5Cyyf+d8qNMlQCdCflYW5n2KBCcYx2O2BXoak0bsvHuHo7X3bi1rkrpA==
|
||||
tarball: 'file:projects/bf-qnamaker.tgz'
|
||||
version: 0.0.0
|
||||
'file:projects/botframework-cli.tgz':
|
||||
|
|
@ -6909,7 +6867,7 @@ packages:
|
|||
dev: false
|
||||
name: '@rush-temp/botframework-cli'
|
||||
resolution:
|
||||
integrity: sha512-RvSJyVz/dc5glO45vQcWXP5L2c8E6Reo3TFMHz2rmA4VYTVziHwxkQ+Jr1NVfHpaYtzYsGW5H6aNe5hnMUhG1Q==
|
||||
integrity: sha512-ihuLm/1+7f7dr3e1Y5sf1XN6kYnA+b+uRdO45XOz0QelPl1nFxgW7631MshB11PSDmmU54ozVBuU+iaINReKfw==
|
||||
tarball: 'file:projects/botframework-cli.tgz'
|
||||
version: 0.0.0
|
||||
registry: ''
|
||||
|
|
@ -6924,7 +6882,6 @@ specifiers:
|
|||
'@rush-temp/bf-cli-config': 'file:./projects/bf-cli-config.tgz'
|
||||
'@rush-temp/bf-cli-plugins': 'file:./projects/bf-cli-plugins.tgz'
|
||||
'@rush-temp/bf-dialog': 'file:./projects/bf-dialog.tgz'
|
||||
'@rush-temp/bf-dispatcher': 'file:./projects/bf-dispatcher.tgz'
|
||||
'@rush-temp/bf-lg-cli': 'file:./projects/bf-lg-cli.tgz'
|
||||
'@rush-temp/bf-lu': 'file:./projects/bf-lu.tgz'
|
||||
'@rush-temp/bf-luis-cli': 'file:./projects/bf-luis-cli.tgz'
|
||||
|
|
@ -6932,7 +6889,6 @@ specifiers:
|
|||
'@rush-temp/botframework-cli': 'file:./projects/botframework-cli.tgz'
|
||||
'@snyk/nuget-semver': ~1.3.0
|
||||
'@types/ansi-styles': ^3.2.1
|
||||
'@types/argparse': ^1.0.36
|
||||
'@types/lru-cache': ^5.1.0
|
||||
'@types/proxyquire': ^1.3.28
|
||||
'@types/readline-sync': ^1.4.3
|
||||
|
|
@ -6942,7 +6898,6 @@ specifiers:
|
|||
ajv: ^6.12.2
|
||||
antlr4: ^4.7.2
|
||||
applicationinsights: ^1.0.8
|
||||
argparse: ~1.0.10
|
||||
await-delay: ^1.0.0
|
||||
botbuilder-lg: 4.8.0-preview
|
||||
botframework-schema: ^4.5.1
|
||||
|
|
@ -6985,7 +6940,6 @@ specifiers:
|
|||
sinon: ^7.5.0
|
||||
source-map-support: ~0.5.16
|
||||
testdouble: ^3.11.0
|
||||
ts-md5: ^1.2.6
|
||||
username: ^4.1.0
|
||||
uuid: ^3.3.3
|
||||
window-size: ^1.1.0
|
||||
|
|
|
|||
|
|
@ -1,11 +0,0 @@
|
|||
root = true
|
||||
|
||||
[*]
|
||||
indent_style = space
|
||||
indent_size = 4
|
||||
charset = utf-8
|
||||
trim_trailing_whitespace = true
|
||||
insert_final_newline = true
|
||||
|
||||
[*.md]
|
||||
trim_trailing_whitespace = false
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
*-debug.log
|
||||
*-error.log
|
||||
/.nyc_output
|
||||
/dist
|
||||
/lib
|
||||
/tmp
|
||||
/yarn.lock
|
||||
node_modules
|
||||
|
|
@ -1,24 +0,0 @@
|
|||
{
|
||||
"extension": [
|
||||
".ts",
|
||||
".js"
|
||||
],
|
||||
"include": [
|
||||
"src"
|
||||
],
|
||||
"exclude": [
|
||||
"**/node_modules/**",
|
||||
"**/tests/**",
|
||||
"**/coverage/**",
|
||||
"**/*.d.ts"
|
||||
],
|
||||
"reporter": [
|
||||
"html",
|
||||
"lcov",
|
||||
"text"
|
||||
],
|
||||
"all": true,
|
||||
"cache": true,
|
||||
"extends": "@istanbuljs/nyc-config-typescript",
|
||||
"check-coverage": false
|
||||
}
|
||||
|
|
@ -1,656 +0,0 @@
|
|||
@microsoft/bf-dispatcher
|
||||
========================
|
||||
|
||||
This package is intended to be consumed by other 'command' packages in the Botframework CLI suite, one example is 'bf-orchestrator'.
|
||||
|
||||
'bf-dispatcher' is a generic NLP intent classification package/library.
|
||||
It currently can do:
|
||||
|
||||
> Sampling
|
||||
> Bootstrap resampling
|
||||
> Auto active learning down sampling
|
||||
> Stratified down sampling
|
||||
|
||||
> Evaluation and Reporting
|
||||
> Cross Validation against a training set
|
||||
> Test again a test set
|
||||
|
||||
To support these features, it internally contains a Softmax Regression (MaxEnt) learner that can consume sparse text features and train models to support auto-active-learning down-sampling and cross validation. It can also produce model quality reports.
|
||||
|
||||
Currently, 'bf-dispatcher' can consume two forms of input sources: LU and TSV columnar files. It uses the bf-lu package (https://github.com/microsoft/botframework-cli/tree/master/packages/lu) to parse and load a .lu file as input.
|
||||
|
||||
To demonstrate the auto active learning process, there are some example functions implemented in
|
||||
"src/model/supervised/classifier/auto_active_learning/AppAutoActiveLearner.ts" that can do the following:
|
||||
a) consume a LU or columnar TSV file,
|
||||
b) use a bootstrap resampler to select training instances based on a prior label/instance (intent/utterance)
|
||||
distribition,
|
||||
c) iterate through batches of input utterance/label records, and train models to select most relevant utterance/intent
|
||||
pairs through an auto active learning process, and
|
||||
d) use a stratifier sampler to limit the training size.
|
||||
|
||||
Below are some examples of using the AutoActiveLearner class.
|
||||
```
|
||||
/**
|
||||
* This function can read a LU file with intent and utterance data and run through 3 steps of
|
||||
* sampling processes:
|
||||
* 0) Bootstrap Resampling
|
||||
* 1) Auto Active Learning Sampling
|
||||
* 2) Stratified Sampling
|
||||
*
|
||||
* @param luContent - a .lu file content in string form as input.
|
||||
* @param doBootstrapResampling - boolean flag to activate bootstrap resampling (BRS) logic or not.
|
||||
* @param brsDistribution - explicit distribution to control bootstrap resampling process
|
||||
* @param doAutoActiveLearning - boolean flag to activate auto active leaning (AAL) process or not.
|
||||
* @param aalLimitInitialNumberOfInstancesPerCategory - AAL initial number of instances per category/label/intent.
|
||||
* @param aalNumberOfInstancesPerIteration - AAL number of instances processed per iterations.
|
||||
* @param aalInstanceSelectionThreshold - AAL threshold to pick a tested instance for training in next iteration
|
||||
* @param learnerParameterEpochs - AAL Softmax Regression learner parameter - number of epochs.
|
||||
* @param learnerParameterMiniBatchSize - AAL Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - AAL Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - AAL Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - AAL Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - AAL Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - AAL Softmax Regression learner parameter - flag
|
||||
* @param limitingSampleSize - sample size controled by a final stratified sampling process.
|
||||
*/
|
||||
public static async mainAutoActiveLearnerWithLuContent(
|
||||
luContent: string,
|
||||
doBootstrapResampling: boolean =
|
||||
AppAutoActiveLearner.defaultDoBootstrapResampling,
|
||||
brsDistribution: TMapStringKeyGenericValue<number> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericValue<number>(),
|
||||
doAutoActiveLearning: boolean =
|
||||
AutoActiveLearner.defaultDoAutoActiveLearning,
|
||||
aalLimitInitialNumberOfInstancesPerCategory: number =
|
||||
AutoActiveLearner.defaultAalLimitInitialNumberOfInstancesPerCategory,
|
||||
aalNumberOfInstancesPerIteration: number =
|
||||
AutoActiveLearner.defaultAalNumberOfInstancesPerIteration,
|
||||
aalInstanceSelectionThreshold: number =
|
||||
AutoActiveLearner.defaultAalInstanceSelectionThreshold,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true,
|
||||
limitingSampleSize: number =
|
||||
DefaultLimitingSampleSize): Promise<{
|
||||
"newLuData": LuData,
|
||||
"learner": SoftmaxRegressionSparse,
|
||||
"seedingInstanceIndexArray": number[],
|
||||
"seedingInstanceIndexArrayInitial": number[],
|
||||
}> {
|
||||
// -----------------------------------------------------------------------
|
||||
let luData: LuData =
|
||||
await LuData.createLuData(
|
||||
luContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (doBootstrapResampling) {
|
||||
const bootstrapSamplerKeyMap: BootstrapSamplerKeyMapDistribution<number> =
|
||||
new BootstrapSamplerKeyMapDistribution<number>(
|
||||
brsDistribution,
|
||||
luData.getIntentInstanceIndexMapArray());
|
||||
// ---- NOTE-FOR-REFERENCE ---- const bootstrapSamplerKeyMap: BootstrapSamplerKeyMap<number> =
|
||||
// ---- NOTE-FOR-REFERENCE ---- new BootstrapSamplerKeyMap(data.getIntentInstanceIndexMapArray());
|
||||
Utility.debuggingLog(`luData.getIntentInstanceIndexMapArray()=` +
|
||||
`${Utility.mapToJsonSerialization(luData.getIntentInstanceIndexMapArray())}`);
|
||||
Utility.debuggingLog(`bootstrapSamplerKeyMap.computeSamplingNumberInstancesPerLabel()=` +
|
||||
`${bootstrapSamplerKeyMap.computeSamplingNumberInstancesPerLabel()}`);
|
||||
// ---- NOTE-FOR-DEBUGGING ---- const samplingIndexArrayGenerator =
|
||||
// ---- NOTE-FOR-DEBUGGING ---- bootstrapSamplerKeyMap.sampleInstances();
|
||||
// ---- NOTE-FOR-DEBUGGING ---- for (const element of samplingIndexArrayGenerator) {
|
||||
// ---- NOTE-FOR-DEBUGGING ---- Utility.debuggingLog(`element of samplingIndexArrayGenerator=` +
|
||||
// ---- NOTE-FOR-DEBUGGING ---- `${element}`);
|
||||
// ---- NOTE-FOR-DEBUGGING ---- }
|
||||
const samplingIndexArray: number[] =
|
||||
[...bootstrapSamplerKeyMap.sampleInstances()];
|
||||
Utility.debuggingLog(`samplingIndexArray.length=` +
|
||||
`${samplingIndexArray.length}`);
|
||||
const luDataBootstrapSampled: Data =
|
||||
await luData.createDataFromSamplingExistingDataUtterances(
|
||||
luData,
|
||||
-1, // ---- NOTE-NO-NEED-FOR-LuData ---- labelColumnIndex,
|
||||
-1, // ---- NOTE-NO-NEED-FOR-LuData ---- textColumnIndex,
|
||||
-1, // ---- NOTE-NO-NEED-FOR-LuData ---- linesToSkip,
|
||||
samplingIndexArray,
|
||||
false);
|
||||
luData = luDataBootstrapSampled as LuData;
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const results =
|
||||
luData.collectSmallUtteranceIndexSetCoveringAllIntentEntityLabels();
|
||||
const smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
results.smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels;
|
||||
const smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
results.smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels;
|
||||
const smallUtteranceIndexSetCoveringAllIntentEntityLabels: Set<number> =
|
||||
results.smallUtteranceIndexSetCoveringAllIntentEntityLabels;
|
||||
const remainingUtteranceIndexSet: Set<number> =
|
||||
results.remainingUtteranceIndexSet;
|
||||
Utility.debuggingLog(`smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexSetCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.setToJsonSerialization(smallUtteranceIndexSetCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`remainingUtteranceIndexSet=` +
|
||||
`${Utility.setToJsonSerialization(remainingUtteranceIndexSet)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexSetCoveringAllIntentEntityLabels.size=` +
|
||||
`${smallUtteranceIndexSetCoveringAllIntentEntityLabels.size}`);
|
||||
Utility.debuggingLog(`remainingUtteranceIndexSet.size=` +
|
||||
`${remainingUtteranceIndexSet.size}`);
|
||||
// -------------------------------------------------------------------
|
||||
if (!doAutoActiveLearning) {
|
||||
aalLimitInitialNumberOfInstancesPerCategory = -1;
|
||||
}
|
||||
const resultsInitialSampling: {
|
||||
"seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels": Map<string, Set<number>>,
|
||||
"candidateUtteranceIndexSetSampled": Set<number>,
|
||||
"candidateUtteranceIndexSetRemaining": Set<number>,
|
||||
} = luData.collectUtteranceIndexSetSeedingIntentTrainingSet(
|
||||
smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels,
|
||||
remainingUtteranceIndexSet,
|
||||
aalLimitInitialNumberOfInstancesPerCategory);
|
||||
const seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
resultsInitialSampling.seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels;
|
||||
const candidateUtteranceIndexSetSampled: Set<number> =
|
||||
resultsInitialSampling.candidateUtteranceIndexSetSampled;
|
||||
const candidateUtteranceIndexSetRemaining: Set<number> =
|
||||
resultsInitialSampling.candidateUtteranceIndexSetRemaining;
|
||||
Utility.debuggingLog(`seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetSampled=` +
|
||||
`${Utility.setToJsonSerialization(candidateUtteranceIndexSetSampled)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetRemaining=` +
|
||||
`${Utility.setToJsonSerialization(candidateUtteranceIndexSetRemaining)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetSampled.size=` +
|
||||
`${candidateUtteranceIndexSetSampled.size}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetRemaining.size=` +
|
||||
`${candidateUtteranceIndexSetRemaining.size}`);
|
||||
const countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels: number =
|
||||
[...seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels].reduce(
|
||||
(accumulation: number, entry: [string, Set<number>]) =>
|
||||
accumulation + entry[1].size, 0);
|
||||
Utility.debuggingLog(`countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels}`);
|
||||
// -------------------------------------------------------------------
|
||||
const seedingUtteranceIndexArray: number[] =
|
||||
[...seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels].reduce(
|
||||
(accumulation: number[], entry: [string, Set<number>]) =>
|
||||
accumulation.concat(Array.from(entry[1])), []);
|
||||
Utility.debuggingLog(`seedingUtteranceIndexArray.length=` +
|
||||
`${seedingUtteranceIndexArray.length}`);
|
||||
// -------------------------------------------------------------------
|
||||
const seedingInstanceIndexArray: number[] =
|
||||
Utility.cloneArray(seedingUtteranceIndexArray);
|
||||
const intentLabelIndexArray: number[] =
|
||||
luData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
luData.getUtteranceFeatureIndexArrays();
|
||||
const autoActiveLearner: AutoActiveLearner =
|
||||
new AutoActiveLearner(
|
||||
doAutoActiveLearning,
|
||||
aalLimitInitialNumberOfInstancesPerCategory,
|
||||
aalNumberOfInstancesPerIteration,
|
||||
aalInstanceSelectionThreshold,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
const learned: {
|
||||
"seedingInstanceIndexArray": number[],
|
||||
"learner": SoftmaxRegressionSparse,
|
||||
} = autoActiveLearner.learn(
|
||||
luData.getFeaturizerLabels(),
|
||||
luData.getFeaturizerLabelMap(),
|
||||
luData.getFeaturizer().getNumberLabels(),
|
||||
luData.getFeaturizer().getNumberFeatures(),
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
seedingInstanceIndexArray,
|
||||
Array.from(candidateUtteranceIndexSetRemaining));
|
||||
let aalSampledInstanceIndexArray: number[] =
|
||||
learned.seedingInstanceIndexArray;
|
||||
const learner: SoftmaxRegressionSparse =
|
||||
learned.learner;
|
||||
// -----------------------------------------------------------------------
|
||||
const numberInstancesPreSelected: number =
|
||||
seedingUtteranceIndexArray.length;
|
||||
if (limitingSampleSize > numberInstancesPreSelected) {
|
||||
limitingSampleSize -= numberInstancesPreSelected;
|
||||
const reservoirArraySampler: ReservoirArraySampler<number> = new ReservoirArraySampler(
|
||||
aalSampledInstanceIndexArray,
|
||||
numberInstancesPreSelected);
|
||||
aalSampledInstanceIndexArray =
|
||||
[...reservoirArraySampler.sampleInstances(limitingSampleSize)];
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const newLuData: LuData = await LuData.createLuDataFromFilteringExistingLuDataUtterances(
|
||||
luData,
|
||||
new Set<number>(aalSampledInstanceIndexArray),
|
||||
false);
|
||||
return {
|
||||
newLuData,
|
||||
learner,
|
||||
seedingInstanceIndexArray: aalSampledInstanceIndexArray,
|
||||
seedingInstanceIndexArrayInitial: seedingUtteranceIndexArray };
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
/**
|
||||
* This function can read a TSV columnar file with label and text data and run through 3 steps of
|
||||
* sampling processes:
|
||||
* 0) Bootstrap Resampling
|
||||
* 1) Auto Active Learning Sampling
|
||||
* 2) Stratified Sampling
|
||||
*
|
||||
* @param columnarContent - content of a TSV columnar file in string form as input.
|
||||
* @param labelColumnIndex - label/intent column index.
|
||||
* @param textColumnIndex - text/utterace column index.
|
||||
* @param linesToSkip - number of header lines skipped before processing each line as an instance record.
|
||||
* @param doBootstrapResampling - boolean flag to activate bootstrap resampling (BRS) logic or not.
|
||||
* @param brsDistribution - explicit distribution to control bootstrap resampling process
|
||||
* @param doAutoActiveLearning - boolean flag to activate auto active leaning (AAL) process or not.
|
||||
* @param aalLimitInitialNumberOfInstancesPerCategory - AAL initial number of instances per category/label/intent.
|
||||
* @param aalNumberOfInstancesPerIteration - AAL number of instances processed per iterations.
|
||||
* @param aalInstanceSelectionThreshold - AAL threshold to pick a tested instance for training in next iteration
|
||||
* @param learnerParameterEpochs - AAL Softmax Regression learner parameter - number of epochs.
|
||||
* @param learnerParameterMiniBatchSize - AAL Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - AAL Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - AAL Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - AAL Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - AAL Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - AAL Softmax Regression learner parameter - flag
|
||||
* @param limitingSampleSize - sample size controled by a final stratified sampling process.
|
||||
*/
|
||||
public static async mainAutoActiveLearnerWithColumnarContent(
|
||||
columnarContent: string,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
doBootstrapResampling: boolean =
|
||||
AppAutoActiveLearner.defaultDoBootstrapResampling,
|
||||
brsDistribution: TMapStringKeyGenericValue<number> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericValue<number>(),
|
||||
doAutoActiveLearning: boolean =
|
||||
AutoActiveLearner.defaultDoAutoActiveLearning,
|
||||
aalLimitInitialNumberOfInstancesPerCategory: number =
|
||||
AutoActiveLearner.defaultAalLimitInitialNumberOfInstancesPerCategory,
|
||||
aalNumberOfInstancesPerIteration: number =
|
||||
AutoActiveLearner.defaultAalNumberOfInstancesPerIteration,
|
||||
aalInstanceSelectionThreshold: number =
|
||||
AutoActiveLearner.defaultAalInstanceSelectionThreshold,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true,
|
||||
limitingSampleSize: number =
|
||||
DefaultLimitingSampleSize): Promise<{
|
||||
"newColumnarData": ColumnarData,
|
||||
"learner": SoftmaxRegressionSparse,
|
||||
"seedingInstanceIndexArray": number[],
|
||||
"seedingInstanceIndexArrayInitial": number[],
|
||||
}> {
|
||||
// -----------------------------------------------------------------------
|
||||
let columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
columnarContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
linesToSkip,
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (doBootstrapResampling) {
|
||||
const bootstrapSamplerKeyMap: BootstrapSamplerKeyMapDistribution<number> =
|
||||
new BootstrapSamplerKeyMapDistribution<number>(
|
||||
brsDistribution,
|
||||
columnarData.getIntentInstanceIndexMapArray());
|
||||
// ---- NOTE-FOR-REFERENCE ---- const bootstrapSamplerKeyMap: BootstrapSamplerKeyMap<number> =
|
||||
// ---- NOTE-FOR-REFERENCE ---- new BootstrapSamplerKeyMap(data.getIntentInstanceIndexMapArray());
|
||||
Utility.debuggingLog(`columnarData.getIntentInstanceIndexMapArray()=` +
|
||||
`${Utility.mapToJsonSerialization(columnarData.getIntentInstanceIndexMapArray())}`);
|
||||
Utility.debuggingLog(`bootstrapSamplerKeyMap.computeSamplingNumberInstancesPerLabel()=` +
|
||||
`${bootstrapSamplerKeyMap.computeSamplingNumberInstancesPerLabel()}`);
|
||||
// ---- NOTE-FOR-DEBUGGING ---- const samplingIndexArrayGenerator =
|
||||
// ---- NOTE-FOR-DEBUGGING ---- bootstrapSamplerKeyMap.sampleInstances();
|
||||
// ---- NOTE-FOR-DEBUGGING ---- for (const element of samplingIndexArrayGenerator) {
|
||||
// ---- NOTE-FOR-DEBUGGING ---- Utility.debuggingLog(`element of samplingIndexArrayGenerator=` +
|
||||
// ---- NOTE-FOR-DEBUGGING ---- `${element}`);
|
||||
// ---- NOTE-FOR-DEBUGGING ---- }
|
||||
const samplingIndexArray: number[] =
|
||||
[...bootstrapSamplerKeyMap.sampleInstances()];
|
||||
Utility.debuggingLog(`samplingIndexArray.length=` +
|
||||
`${samplingIndexArray.length}`);
|
||||
const columnarDataBootstrapSampled: Data =
|
||||
await columnarData.createDataFromSamplingExistingDataUtterances(
|
||||
columnarData,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
linesToSkip,
|
||||
samplingIndexArray,
|
||||
false);
|
||||
columnarData = columnarDataBootstrapSampled as ColumnarData;
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const results =
|
||||
columnarData.collectSmallUtteranceIndexSetCoveringAllIntentEntityLabels();
|
||||
const smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
results.smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels;
|
||||
const smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
results.smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels;
|
||||
const smallUtteranceIndexSetCoveringAllIntentEntityLabels: Set<number> =
|
||||
results.smallUtteranceIndexSetCoveringAllIntentEntityLabels;
|
||||
const remainingUtteranceIndexSet: Set<number> =
|
||||
results.remainingUtteranceIndexSet;
|
||||
Utility.debuggingLog(`smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexSetCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.setToJsonSerialization(smallUtteranceIndexSetCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`remainingUtteranceIndexSet=` +
|
||||
`${Utility.setToJsonSerialization(remainingUtteranceIndexSet)}`);
|
||||
Utility.debuggingLog(`smallUtteranceIndexSetCoveringAllIntentEntityLabels.size=` +
|
||||
`${smallUtteranceIndexSetCoveringAllIntentEntityLabels.size}`);
|
||||
Utility.debuggingLog(`remainingUtteranceIndexSet.size=` +
|
||||
`${remainingUtteranceIndexSet.size}`);
|
||||
// -------------------------------------------------------------------
|
||||
if (!doAutoActiveLearning) {
|
||||
aalLimitInitialNumberOfInstancesPerCategory = -1;
|
||||
}
|
||||
const resultsInitialSampling: {
|
||||
"seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels": Map<string, Set<number>>,
|
||||
"candidateUtteranceIndexSetSampled": Set<number>,
|
||||
"candidateUtteranceIndexSetRemaining": Set<number>,
|
||||
} = columnarData.collectUtteranceIndexSetSeedingIntentTrainingSet(
|
||||
smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels,
|
||||
remainingUtteranceIndexSet,
|
||||
aalLimitInitialNumberOfInstancesPerCategory);
|
||||
const seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
resultsInitialSampling.seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels;
|
||||
const candidateUtteranceIndexSetSampled: Set<number> =
|
||||
resultsInitialSampling.candidateUtteranceIndexSetSampled;
|
||||
const candidateUtteranceIndexSetRemaining: Set<number> =
|
||||
resultsInitialSampling.candidateUtteranceIndexSetRemaining;
|
||||
Utility.debuggingLog(`seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${Utility.stringMapSetToJson(seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetSampled=` +
|
||||
`${Utility.setToJsonSerialization(candidateUtteranceIndexSetSampled)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetRemaining=` +
|
||||
`${Utility.setToJsonSerialization(candidateUtteranceIndexSetRemaining)}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetSampled.size=` +
|
||||
`${candidateUtteranceIndexSetSampled.size}`);
|
||||
Utility.debuggingLog(`candidateUtteranceIndexSetRemaining.size=` +
|
||||
`${candidateUtteranceIndexSetRemaining.size}`);
|
||||
const countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels: number =
|
||||
[...seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels].reduce(
|
||||
(accumulation: number, entry: [string, Set<number>]) =>
|
||||
accumulation + entry[1].size, 0);
|
||||
Utility.debuggingLog(`countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels=` +
|
||||
`${countSeedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels}`);
|
||||
// -------------------------------------------------------------------
|
||||
const seedingUtteranceIndexArray: number[] =
|
||||
[...seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels].reduce(
|
||||
(accumulation: number[], entry: [string, Set<number>]) =>
|
||||
accumulation.concat(Array.from(entry[1])), []);
|
||||
Utility.debuggingLog(`seedingUtteranceIndexArray.length=` +
|
||||
`${seedingUtteranceIndexArray.length}`);
|
||||
// -------------------------------------------------------------------
|
||||
const seedingInstanceIndexArray: number[] =
|
||||
Utility.cloneArray(seedingUtteranceIndexArray);
|
||||
const intentLabelIndexArray: number[] =
|
||||
columnarData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
columnarData.getUtteranceFeatureIndexArrays();
|
||||
const autoActiveLearner: AutoActiveLearner =
|
||||
new AutoActiveLearner(
|
||||
doAutoActiveLearning,
|
||||
aalLimitInitialNumberOfInstancesPerCategory,
|
||||
aalNumberOfInstancesPerIteration,
|
||||
aalInstanceSelectionThreshold,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
const learned: {
|
||||
"seedingInstanceIndexArray": number[],
|
||||
"learner": SoftmaxRegressionSparse,
|
||||
} = autoActiveLearner.learn(
|
||||
columnarData.getFeaturizerLabels(),
|
||||
columnarData.getFeaturizerLabelMap(),
|
||||
columnarData.getFeaturizer().getNumberLabels(),
|
||||
columnarData.getFeaturizer().getNumberFeatures(),
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
seedingInstanceIndexArray,
|
||||
Array.from(candidateUtteranceIndexSetRemaining));
|
||||
let aalSampledInstanceIndexArray: number[] =
|
||||
learned.seedingInstanceIndexArray;
|
||||
const learner: SoftmaxRegressionSparse =
|
||||
learned.learner;
|
||||
// -----------------------------------------------------------------------
|
||||
const numberInstancesPreSelected: number =
|
||||
seedingUtteranceIndexArray.length;
|
||||
if (limitingSampleSize > numberInstancesPreSelected) {
|
||||
limitingSampleSize -= numberInstancesPreSelected;
|
||||
const reservoirArraySampler: ReservoirArraySampler<number> = new ReservoirArraySampler(
|
||||
aalSampledInstanceIndexArray,
|
||||
numberInstancesPreSelected);
|
||||
aalSampledInstanceIndexArray =
|
||||
[...reservoirArraySampler.sampleInstances(limitingSampleSize)];
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const newColumnarData: ColumnarData =
|
||||
ColumnarData.createColumnarDataFromFilteringExistingColumnarDataUtterances(
|
||||
columnarData,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
linesToSkip,
|
||||
new Set<number>(aalSampledInstanceIndexArray),
|
||||
false);
|
||||
return {
|
||||
newColumnarData,
|
||||
learner,
|
||||
seedingInstanceIndexArray: aalSampledInstanceIndexArray,
|
||||
seedingInstanceIndexArrayInitial: seedingUtteranceIndexArray };
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
```
|
||||
|
||||
In "src/model/evaluation/cross_validation/AppCrossValidator.ts", there are some example functions that
|
||||
demonstrates how to use 'bf-dispatcher' to run cross validation and evaluate model performance.
|
||||
|
||||
```
|
||||
/**
|
||||
* This function consumes a LU file content as input and run cross validation (CV) to evaluate models trained from
|
||||
* the input label/text (intent/utterance) instance set.
|
||||
*
|
||||
* @param luContent - input LU file content as input.
|
||||
* @param numberOfCrossValidationFolds - number of cross validation (CV) folds.
|
||||
* @param learnerParameterEpochs - CV Softmax Regression Learner parameter - number of epochs
|
||||
* @param learnerParameterMiniBatchSize - CV Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - CV Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - CV Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - CV Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - CV Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - CV Softmax Regression learner parameter - flag
|
||||
*/
|
||||
export async function mainCrossValidatorWithLuContent(
|
||||
luContent: string,
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true): Promise<CrossValidator> {
|
||||
// -----------------------------------------------------------------------
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
luContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (!numberOfCrossValidationFolds) {
|
||||
numberOfCrossValidationFolds = CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
const intents: string[] =
|
||||
luData.getIntents();
|
||||
const utterances: string[] =
|
||||
luData.getUtterances();
|
||||
const intentLabelIndexArray: number[] =
|
||||
luData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
luData.getUtteranceFeatureIndexArrays();
|
||||
assert(intentLabelIndexArray, "intentLabelIndexArray is undefined.");
|
||||
assert(utteranceFeatureIndexArrays, "utteranceFeatureIndexArrays is undefined.");
|
||||
const crossValidator: CrossValidator =
|
||||
new CrossValidator(
|
||||
luData.getFeaturizerLabels(),
|
||||
luData.getFeaturizerLabelMap(),
|
||||
luData.getFeaturizer().getNumberLabels(),
|
||||
luData.getFeaturizer().getNumberFeatures(),
|
||||
intents,
|
||||
utterances,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
luData.getIntentInstanceIndexMapArray(),
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
return crossValidator;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
/**
|
||||
* This function consumes a columnar TSV file content as input and run cross validation (CV) to
|
||||
* evaluate models trained from the input label/text (intent/utterance) instance set.
|
||||
*
|
||||
* @param columnarContent - content of a TSV columnar file in string form as input.
|
||||
* @param labelColumnIndex - label/intent column index.
|
||||
* @param textColumnIndex - text/utterace column index.
|
||||
* @param linesToSkip - number of header lines skipped before processing each line as an instance record.
|
||||
* @param numberOfCrossValidationFolds - number of cross validation (CV) folds.
|
||||
* @param learnerParameterEpochs - CV Softmax Regression Learner parameter - number of epochs
|
||||
* @param learnerParameterMiniBatchSize - CV Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - CV Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - CV Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - CV Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - CV Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - CV Softmax Regression learner parameter - flag
|
||||
*/
|
||||
export function mainCrossValidatorWithColumnarContent(
|
||||
columnarContent: string,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true): CrossValidator {
|
||||
// -----------------------------------------------------------------------
|
||||
const columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
columnarContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
linesToSkip,
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (!numberOfCrossValidationFolds) {
|
||||
numberOfCrossValidationFolds = CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
const intents: string[] =
|
||||
columnarData.getIntents();
|
||||
const utterances: string[] =
|
||||
columnarData.getUtterances();
|
||||
const intentLabelIndexArray: number[] =
|
||||
columnarData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
columnarData.getUtteranceFeatureIndexArrays();
|
||||
assert(intentLabelIndexArray, "intentLabelIndexArray is undefined.");
|
||||
assert(utteranceFeatureIndexArrays, "utteranceFeatureIndexArrays is undefined.");
|
||||
const crossValidator: CrossValidator =
|
||||
new CrossValidator(
|
||||
columnarData.getFeaturizerLabels(),
|
||||
columnarData.getFeaturizerLabelMap(),
|
||||
columnarData.getFeaturizer().getNumberLabels(),
|
||||
columnarData.getFeaturizer().getNumberFeatures(),
|
||||
intents,
|
||||
utterances,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
columnarData.getIntentInstanceIndexMapArray(),
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
return crossValidator;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
```
|
||||
|
||||
At last, 'bf-dispatcher' also contains several model performance reporter classes in "src/model/evaluation/report" For now, there are three report classes:
|
||||
a) DataProfileReporter.ts: consume a data file and report label distribution and feature distribution. Some example functions are implemented in AppDataProfileReporter.ts.
|
||||
b) ModelMetaDataProfileReporter.ts: load a model previously trained and generated and report its parameters. Some example functions are implemented in AppModelMetaDataProfileReporter.ts.
|
||||
c) ThresholdReporter.ts: load a model and a test file, then report the model performance. Some example functions are implemented in AppThresholdReporter.ts.
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
|
||||
require('@oclif/command').run()
|
||||
.catch(require('@oclif/errors/handle'))
|
||||
|
|
@ -1,3 +0,0 @@
|
|||
@echo off
|
||||
|
||||
node "%~dp0\run" %*
|
||||
|
|
@ -1,69 +0,0 @@
|
|||
{
|
||||
"name": "@microsoft/bf-dispatcher",
|
||||
"description": "Dispatcher contains a Softmax learner initially used for auto-active-learning down-sampling and a ML confusion-matrix evaluator on intent classification models.",
|
||||
"version": "1.0.0",
|
||||
"author": "Microsoft",
|
||||
"bugs": "https://github.com/microsoft/botframework-cli/issues",
|
||||
"main": "lib/index.js",
|
||||
"dependencies": {
|
||||
"@microsoft/bf-lu": "1.0.0",
|
||||
"@oclif/command": "~1.5.19",
|
||||
"@oclif/config": "~1.13.3",
|
||||
"tslib": "^1.10.0",
|
||||
"argparse": "~1.0.10"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@oclif/dev-cli": "^1.22.2",
|
||||
"@oclif/plugin-help": "^2.2.1",
|
||||
"@oclif/test": "^1.2.5",
|
||||
"@oclif/tslint": "^3.1.1",
|
||||
"@types/argparse": "^1.0.36",
|
||||
"@types/chai": "^4.2.4",
|
||||
"@types/mocha": "^5.2.7",
|
||||
"@types/node": "^10.17.3",
|
||||
"chai": "^4.2.0",
|
||||
"globby": "^10.0.1",
|
||||
"mocha": "^5.2.0",
|
||||
"nyc": "^14.1.1",
|
||||
"readline-sync": "^1.4.10",
|
||||
"rimraf": "^3.0.0",
|
||||
"ts-md5": "^1.2.6",
|
||||
"ts-node": "^8.4.1",
|
||||
"tslint": "^5.20.1",
|
||||
"typescript": "^3.7.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=8.0.0"
|
||||
},
|
||||
"files": [
|
||||
"/lib",
|
||||
"/npm-shrinkwrap.json",
|
||||
"/oclif.manifest.json",
|
||||
"/yarn.lock"
|
||||
],
|
||||
"homepage": "https://github.com/microsoft/botframework-cli",
|
||||
"keywords": [
|
||||
"oclif-plugin"
|
||||
],
|
||||
"license": "MIT",
|
||||
"oclif": {
|
||||
"commands": "./lib/commands",
|
||||
"bin": "oclif-example",
|
||||
"devPlugins": [
|
||||
"@oclif/plugin-help"
|
||||
]
|
||||
},
|
||||
"repository": "https://github.com/microsoft/botframework-cli/tree/master/packages/dispatcher",
|
||||
"scripts": {
|
||||
"prepack": "npm run clean && npm run build && npm run doc:readme",
|
||||
"postpack": "rimraf oclif.manifest.json",
|
||||
"posttest": "tslint -p test -t stylish",
|
||||
"build": "tsc -b",
|
||||
"clean": "rimraf ./.nyc_output ./lib ./package-lock.json ./tsconfig.tsbuildinfo",
|
||||
"test": "mocha",
|
||||
"coverage": "nyc npm run test",
|
||||
"doc": "npm run build && npm run doc:readme && rimraf oclif.manifest.json",
|
||||
"doc:readme": "oclif-dev manifest && oclif-dev readme",
|
||||
"version": "npm run doc:readme && git add README.md"
|
||||
}
|
||||
}
|
||||
|
|
@ -1,602 +0,0 @@
|
|||
Label Weight Text
|
||||
AddFlag 1 add flag to the email john just sent to me
|
||||
AddFlag 1 make it flagged
|
||||
AddFlag 1 i want to add a flag on this email
|
||||
AddFlag 1 add a flag please
|
||||
AddFlag 1 flag this email as important for me
|
||||
AddFlag 1 add a flag to the last email
|
||||
AddFlag 1 this email should be flagged
|
||||
AddFlag 1 add flag to this email
|
||||
AddFlag 1 put a flag on the new email
|
||||
AddFlag 1 add a flag
|
||||
AddFlag 1 the email to ruth needs to be flagged
|
||||
AddFlag 1 mark as flag
|
||||
AddFlag 1 add flag to it
|
||||
AddFlag 1 flag
|
||||
AddFlag 1 put a flag
|
||||
AddFlag 1 flag this email
|
||||
AddFlag 1 turn flag on
|
||||
AddFlag 1 flag it
|
||||
AddFlag 1 flag on
|
||||
AddFlag 1 flag the current email
|
||||
AddFlag 1 i want to add a flag
|
||||
AddFlag 1 flag the email from davis
|
||||
AddFlag 1 add flag
|
||||
AddFlag 1 the email from thomas should be flagged
|
||||
AddFlag 1 add a flag to this email
|
||||
AddFlag 1 flag the email
|
||||
AddFlag 1 this email needs to be flagged
|
||||
AddFlag 1 this email need to be flagged
|
||||
AddFlag 1 add flag to this message
|
||||
AddFlag 1 add flag on it
|
||||
AddFlag 1 mark the email flagged
|
||||
AddMore 1 i need to add something else to my email to cheryl
|
||||
AddMore 1 add more and change the message
|
||||
AddMore 1 i need to add more to the email
|
||||
AddMore 1 add: call me tonight after work
|
||||
AddMore 1 write more
|
||||
AddMore 1 i need to add more to the email message i am sending to vincent
|
||||
AddMore 1 put some additional lines to this message
|
||||
AddMore 1 i need to add more text
|
||||
AddMore 1 add more to message
|
||||
AddMore 1 add a picture
|
||||
AddMore 1 add a file to the email
|
||||
AddMore 1 please add it was terrible
|
||||
AddMore 1 i would like to add more to the email message
|
||||
AddMore 1 add more message
|
||||
AddMore 1 need to add information to the previous email
|
||||
AddMore 1 add some more
|
||||
AddMore 1 add a subject
|
||||
AddMore 1 i'd like to add more to the email
|
||||
AddMore 1 add photo
|
||||
AddMore 1 add more to the last email
|
||||
AddMore 1 add, by the way, what's the plan of next step
|
||||
AddMore 1 i would like to add more to the email
|
||||
AddMore 1 add to body of email
|
||||
AddMore 1 append an attachment to this email
|
||||
AddMore 1 edit email so i can type an additional message
|
||||
AddMore 1 add another line to the message
|
||||
AddMore 1 i need to add further contents
|
||||
AddMore 1 insert more lines for me please
|
||||
AddMore 1 add something
|
||||
AddMore 1 add more to the message
|
||||
AddMore 1 i forgot to add an important part to that email to james . please set it up to edit
|
||||
AddMore 1 is it ok if i add more to the email
|
||||
AddMore 1 i need to add additional lines
|
||||
AddMore 1 add more please
|
||||
AddMore 1 i need to add more message
|
||||
AddMore 1 add more details to it
|
||||
AddMore 1 add more
|
||||
AddMore 1 please add more
|
||||
AddMore 1 i need to add something else to that email to donna before it is sent
|
||||
AddMore 1 can i add more to the email
|
||||
AddMore 1 i'd like to add a bit more to the email.
|
||||
AddMore 1 i'd like to add a bit more to the message
|
||||
AddMore 1 add more to roy 's email
|
||||
AddMore 1 add more text please
|
||||
AddMore 1 i want to add more the email
|
||||
AddMore 1 can i add more to the message
|
||||
AddMore 1 add more to email body
|
||||
AddMore 1 i would like to open a new line
|
||||
AddMore 1 i wish to add more to the message
|
||||
AddMore 1 i am not done yet. i need to add some more details
|
||||
AddMore 1 add did you enjoy the entire program
|
||||
AddMore 1 ok, i need to add a few things to that
|
||||
AddMore 1 more text
|
||||
AddMore 1 add more to the email
|
||||
AddMore 1 add file to email
|
||||
AddMore 1 wait, i need to write more
|
||||
AddMore 1 add more to it
|
||||
AddMore 1 add more to text
|
||||
AddMore 1 add more text
|
||||
AddMore 1 add more don't forget to bring beer
|
||||
AddMore 1 it isn't complete, need more contents
|
||||
AddMore 1 add more to email
|
||||
AddMore 1 please add, please let me know what i can bring. i'd be happy to make a side dish or dessert
|
||||
AddMore 1 attach file
|
||||
AddMore 1 insert more text in my email
|
||||
CancelMessages 1 cancel my email to jane
|
||||
CancelMessages 1 don't show me
|
||||
CancelMessages 1 don't send that email
|
||||
CancelMessages 1 can you cancel it
|
||||
CancelMessages 1 don't email to her
|
||||
CancelMessages 1 forget about the email
|
||||
CancelMessages 1 don ' t read
|
||||
CancelMessages 1 never mind cancel the message
|
||||
CancelMessages 1 stop reading
|
||||
CancelMessages 1 never mind cancel the mail
|
||||
CancelMessages 1 okay cancel sending the mail
|
||||
CancelMessages 1 cancel this email
|
||||
CancelMessages 1 quit the sending
|
||||
CancelMessages 1 cancel the mail
|
||||
CancelMessages 1 don't send out
|
||||
CancelMessages 1 don't send this email
|
||||
CancelMessages 1 cancel email
|
||||
CancelMessages 1 don't read the message
|
||||
CancelMessages 1 neither of them
|
||||
CancelMessages 1 don't send it
|
||||
CancelMessages 1 cancel message
|
||||
CancelMessages 1 stop message
|
||||
CancelMessages 1 don't email
|
||||
CancelMessages 1 exit
|
||||
CancelMessages 1 i want you to cancel the email
|
||||
CancelMessages 1 never mind, forget about the mail
|
||||
CancelMessages 1 cancel this sending process
|
||||
CancelMessages 1 don't send
|
||||
CancelMessages 1 no, i don't want to send this message
|
||||
CancelMessages 1 no just cancel the email
|
||||
CancelMessages 1 don ' t read it
|
||||
CancelMessages 1 nevermind cancel
|
||||
CancelMessages 1 cancel email to natalie
|
||||
CancelMessages 1 cancel the message
|
||||
CancelMessages 1 don't read the email
|
||||
CancelMessages 1 no, no, cancel the reading
|
||||
CancelMessages 1 cancel searching the messages
|
||||
CancelMessages 1 abort deletion
|
||||
CancelMessages 1 cancel the email to my sister
|
||||
CancelMessages 1 cancel this message
|
||||
CancelMessages 1 don 't send the email
|
||||
CancelMessages 1 cancel the email sent to alex
|
||||
CancelMessages 1 cancel the email
|
||||
CancelMessages 1 no don't send
|
||||
CancelMessages 1 no don't send it
|
||||
CancelMessages 1 no cancel it, i don't want to send the mail
|
||||
CheckMessages 1 do i have new message
|
||||
CheckMessages 1 does anyone send message to me just then
|
||||
CheckMessages 1 do i receive new message
|
||||
CheckMessages 1 does anyone send email to me just then
|
||||
CheckMessages 1 do i have any new mail
|
||||
CheckMessages 1 does my outlook have new email
|
||||
CheckMessages 1 could you please check my emails
|
||||
CheckMessages 1 please check my emails
|
||||
CheckMessages 1 is there new email
|
||||
CheckMessages 1 any new email now
|
||||
CheckMessages 1 whether i get new message
|
||||
CheckMessages 1 check outlook please
|
||||
CheckMessages 1 whether i have new email
|
||||
CheckMessages 1 do i receive new mail in outlook
|
||||
CheckMessages 1 show the important emails in my inbox
|
||||
CheckMessages 1 any new message now
|
||||
CheckMessages 1 check my inbox
|
||||
CheckMessages 1 show my unread mails
|
||||
CheckMessages 1 i'd like to check my inbox
|
||||
CheckMessages 1 do i receive new email
|
||||
CheckMessages 1 check email
|
||||
CheckMessages 1 whether i have new message
|
||||
CheckMessages 1 do i get new email
|
||||
CheckMessages 1 check email please
|
||||
CheckMessages 1 i want to check my inbox
|
||||
CheckMessages 1 please check my inbox
|
||||
CheckMessages 1 could you please check my inbox
|
||||
CheckMessages 1 show latest emails
|
||||
CheckMessages 1 check up messages
|
||||
CheckMessages 1 check up email
|
||||
CheckMessages 1 whether i receive new email
|
||||
CheckMessages 1 show my emails
|
||||
CheckMessages 1 please check my outlook
|
||||
CheckMessages 1 check my message
|
||||
CheckMessages 1 i want to check my emails
|
||||
CheckMessages 1 check my gmail
|
||||
CheckMessages 1 check my email please
|
||||
CheckMessages 1 any new email
|
||||
CheckMessages 1 do i have new email now
|
||||
CheckMessages 1 any new email available
|
||||
CheckMessages 1 could you please check my messages
|
||||
CheckMessages 1 do i have new email
|
||||
CheckMessages 1 check my emails
|
||||
CheckMessages 1 whether i get new email
|
||||
CheckMessages 1 check my mail box
|
||||
ConfirmMessages 1 no problem, go ahead send the mail
|
||||
ConfirmMessages 1 just do it
|
||||
ConfirmMessages 1 yeah right, send to alex
|
||||
ConfirmMessages 1 ok, good, just send it
|
||||
ConfirmMessages 1 of course, just delete the mail
|
||||
ConfirmMessages 1 "yes, you can"
|
||||
ConfirmMessages 1 yes, send it
|
||||
ConfirmMessages 1 i confirm that i want to send this email
|
||||
ConfirmMessages 1 perfect thank you
|
||||
ConfirmMessages 1 alright, just send the message
|
||||
ConfirmMessages 1 okay, send it now
|
||||
ConfirmMessages 1 "sure, go ahead"
|
||||
ConfirmMessages 1 ok, good to me, send it please
|
||||
ConfirmMessages 1 correct, please send it.
|
||||
ConfirmMessages 1 yes it's right
|
||||
ConfirmMessages 1 right, send it please
|
||||
ConfirmMessages 1 ok send the mail to may
|
||||
ConfirmMessages 1 okay send it
|
||||
ConfirmMessages 1 "okay, send it"
|
||||
ConfirmMessages 1 yes that's right
|
||||
ConfirmMessages 1 okay
|
||||
Delete 1 can you help me delete it
|
||||
Delete 1 delete the previous 4 emails
|
||||
Delete 1 empty the email inbox
|
||||
Delete 1 put it in the recycle bin
|
||||
Delete 1 delete the email from my hotmail account
|
||||
Delete 1 delete all emails from tom
|
||||
Delete 1 remove the emails received yesterday
|
||||
Delete 1 delete this message permanently
|
||||
Delete 1 put the emails from this file folder to trash bin
|
||||
Delete 1 remove emails that are duplicate
|
||||
Delete 1 remove it from my inbox
|
||||
Delete 1 delete this email
|
||||
Delete 1 remove the email from mary
|
||||
Delete 1 put the email to trash bin
|
||||
Delete 1 remove emails with red flags
|
||||
Delete 1 delete all emails received tonight
|
||||
Delete 1 delete the email sent from mary jane
|
||||
Delete 1 clear my inbox
|
||||
Delete 1 delete the unread emails
|
||||
Delete 1 delete the first email for me
|
||||
Delete 1 delete what i just wrote
|
||||
Delete 1 delete the red ones
|
||||
Delete 1 put the email in the recycle bin
|
||||
Delete 1 delete the second one
|
||||
Delete 1 delete the last one
|
||||
Delete 1 delete the second mail
|
||||
Forward 1 forward message to girlfriend
|
||||
Forward 1 forward email to girlfriend
|
||||
Forward 1 could you please forward this email to my sister
|
||||
Forward 1 forward to alan tonight
|
||||
Forward 1 forward this email to patricia
|
||||
Forward 1 forward the last email to susan
|
||||
Forward 1 please forward this message
|
||||
Forward 1 forward the email from john smith to michelle by saying fyi
|
||||
Forward 1 forward to thomas please
|
||||
Forward 1 forward to partoneparttwo@gmail.com next monday
|
||||
Forward 1 forward to wife by saying i love you
|
||||
Forward 1 forward to deborah with a message saying that i don't want that
|
||||
Forward 1 forward to dorothy by typing i agree with it
|
||||
Forward 1 forward the email to dad
|
||||
Forward 1 forward to my boss and attach the schedule file
|
||||
Forward 1 forward emails to gabriel
|
||||
Forward 1 forward this email to gary brown please
|
||||
Forward 1 forward by saying if you interest to rebecca
|
||||
Forward 1 forward this email
|
||||
Forward 1 forward the email from melissa to peter
|
||||
Forward 1 forward to brian potter tonight
|
||||
Forward 1 forward this email to joseph
|
||||
Forward 1 forward email
|
||||
Forward 1 please forward this email to partoneparttwo@163.com
|
||||
Forward 1 could you forward this message to ronald and roy
|
||||
Forward 1 please forward this email to albert by typing everything goes fine
|
||||
Forward 1 please forward to benjamin
|
||||
Forward 1 forward all files from sally to austin
|
||||
Forward 1 forward this email to partone dot parttwo at gmail dot com
|
||||
Forward 1 forward to mom
|
||||
Forward 1 forward this email to eugene by typing what do you think
|
||||
Forward 1 please forward this email to partoneparttwo@outlook.com
|
||||
None 1 2
|
||||
None 1 the first one
|
||||
None 1 the second one
|
||||
None 1 the third one
|
||||
None 1 1
|
||||
None 1 3
|
||||
QueryLastText 1 please tell me who emailed me last
|
||||
QueryLastText 1 open the last email
|
||||
QueryLastText 1 come to the last
|
||||
QueryLastText 1 what was the last email
|
||||
QueryLastText 1 what is the lastest email i received from dad
|
||||
QueryLastText 1 the last email
|
||||
QueryLastText 1 whose email just then
|
||||
QueryLastText 1 show me the lastest email
|
||||
QueryLastText 1 who recently emailed me
|
||||
QueryLastText 1 who emailed me last
|
||||
QueryLastText 1 who sent me the email lastly yesterday
|
||||
QueryLastText 1 what was the last email i got from steve edwards
|
||||
QueryLastText 1 who email me just now
|
||||
QueryLastText 1 show the last email
|
||||
QueryLastText 1 who emailed me just now
|
||||
QueryLastText 1 open the lastest email i got
|
||||
QueryLastText 1 what harry last email said
|
||||
QueryLastText 1 i want to see the last email
|
||||
QueryLastText 1 what did mom just say
|
||||
QueryLastText 1 show me the newest email
|
||||
QueryLastText 1 what was the last email i got from dad
|
||||
QueryLastText 1 what henry just said
|
||||
QueryLastText 1 last email
|
||||
QueryLastText 1 who texted me
|
||||
QueryLastText 1 can you tell me the last email i received
|
||||
QueryLastText 1 who sent me the mail just now
|
||||
QueryLastText 1 whose email now
|
||||
QueryLastText 1 go to the last one
|
||||
QueryLastText 1 what is the last email i received today
|
||||
QueryLastText 1 who emailed me
|
||||
QueryLastText 1 what eric watson just said
|
||||
QueryLastText 1 who texted me just now
|
||||
ReadAloud 1 read me the email on apple
|
||||
ReadAloud 1 read email
|
||||
ReadAloud 1 read my most recent email
|
||||
ReadAloud 1 read my last email
|
||||
ReadAloud 1 read aloud my new email
|
||||
ReadAloud 1 read me my latest emails
|
||||
ReadAloud 1 read me the newest email
|
||||
ReadAloud 1 read my last email out to me
|
||||
ReadAloud 1 read out the email from liu about transfer
|
||||
ReadAloud 1 read unread message
|
||||
ReadAloud 1 read it
|
||||
ReadAloud 1 read emails
|
||||
ReadAloud 1 read last incoming emails
|
||||
ReadAloud 1 read my recent email to me
|
||||
ReadAloud 1 read my emails from patty
|
||||
ReadAloud 1 read latest email
|
||||
ReadAloud 1 read the last email
|
||||
ReadAloud 1 read the latest email from mom
|
||||
ReadAloud 1 read email to me
|
||||
ReadAloud 1 read my email messages
|
||||
ReadAloud 1 read me the last emails of the five minutes
|
||||
ReadAloud 1 read first email in the linked inbox
|
||||
ReadAloud 1 read my second email
|
||||
ReadAloud 1 read last email received
|
||||
ReadAloud 1 read emails from clay
|
||||
ReadAloud 1 read new email from david ma
|
||||
ReadAloud 1 read my email from tyler swift
|
||||
ReadAloud 1 read me the email titled happy new year
|
||||
ReadAloud 1 read my email please
|
||||
ReadAloud 1 read todays mail
|
||||
ReadAloud 1 read my email to me
|
||||
ReadAloud 1 read my email from baby
|
||||
ReadAloud 1 read most recent email
|
||||
ReadAloud 1 read first email in link box
|
||||
ReadAloud 1 read google mail
|
||||
ReadAloud 1 read email from dawn
|
||||
ReadAloud 1 read darren's mail on the movie
|
||||
ReadAloud 1 read me the email sent on thanksgiving day
|
||||
ReadAloud 1 read recent email
|
||||
ReadAloud 1 read my notification
|
||||
ReadAloud 1 read my inbox
|
||||
ReadAloud 1 read new message
|
||||
ReadAloud 1 read mary grace white email
|
||||
ReadAloud 1 please read my last email
|
||||
ReadAloud 1 read my recent email
|
||||
ReadAloud 1 read the first email in hotmail
|
||||
ReadAloud 1 read me the email
|
||||
ReadAloud 1 read me the emails from agatha
|
||||
ReadAloud 1 read my emails
|
||||
ReadAloud 1 read last mail
|
||||
ReadAloud 1 read the first email
|
||||
ReadAloud 1 read the last email message
|
||||
ReadAloud 1 read out darren's mail
|
||||
ReadAloud 1 read email from kat
|
||||
ReadAloud 1 read new email
|
||||
ReadAloud 1 read my email from hubby
|
||||
ReadAloud 1 read my new email
|
||||
ReadAloud 1 read me the last email claude sent
|
||||
ReadAloud 1 read the latest email from steve lip
|
||||
ReadAloud 1 read my recent email message please
|
||||
ReadAloud 1 read me the recent email titled abcd from jessica
|
||||
ReadAloud 1 read unread email
|
||||
ReadAloud 1 read the email
|
||||
ReadAloud 1 read the email on auto repair
|
||||
ReadAloud 1 read my outlook email
|
||||
ReadAloud 1 read today's mail
|
||||
ReadAloud 1 read me dylan's email sent on yesterday
|
||||
ReadAloud 1 read my new emails
|
||||
ReadAloud 1 read aloud the christmas party email
|
||||
ReadAloud 1 read please
|
||||
ReadAloud 1 read email from mum
|
||||
ReadAloud 1 could you read out the email on how to use the new tool
|
||||
ReadAloud 1 read my recent email messages
|
||||
ReadAloud 1 read me jessica's email on dress code for the party
|
||||
ReadAloud 1 read me the email on thanksgiving day
|
||||
ReadAloud 1 read me my last hotmail email
|
||||
ReadAloud 1 can you read my emails
|
||||
ReadAloud 1 read out xu's email about apple's news
|
||||
ReadAloud 1 read the latest email i sent
|
||||
ReadAloud 1 can you read my last email
|
||||
Reply 1 reply by saying i love you
|
||||
Reply 1 reply yee ha
|
||||
Reply 1 reply with hello
|
||||
Reply 1 email back
|
||||
Reply 1 reply to the first one
|
||||
Reply 1 email back i will call you back
|
||||
Reply 1 send email back
|
||||
Reply 1 reply by saying yes
|
||||
Reply 1 respond to lore hound
|
||||
Reply 1 create a response to the email by saying pls send me the picture again
|
||||
Reply 1 respond to the email by saying i am busy today
|
||||
Reply 1 reply yee hello
|
||||
Reply 1 reply by email thank you very much best regards jun
|
||||
Reply 1 reply required to an email
|
||||
Reply 1 respond i ' m sick i can ' t do it
|
||||
Reply 1 reply to the email
|
||||
Reply 1 send the response with i've already know
|
||||
Reply 1 reply that i am busy
|
||||
Reply 1 reply to edward
|
||||
Reply 1 reply to email i am busy now
|
||||
Reply 1 reply we'll see you later
|
||||
Reply 1 reply to susan
|
||||
Reply 1 make a response with thank you very much
|
||||
Reply 1 respond to nathan
|
||||
Reply 1 how to reply to an email
|
||||
Reply 1 reply to my last email
|
||||
Reply 1 return siberian huskies mobile
|
||||
Reply 1 return barbara on mobile
|
||||
Reply 1 reply by typing hello
|
||||
Reply 1 reply
|
||||
Reply 1 reply yes boss.
|
||||
SearchMessages 1 show me emails from clara chan
|
||||
SearchMessages 1 email sent from lisa
|
||||
SearchMessages 1 search keywordsone keywordstwo from inbox
|
||||
SearchMessages 1 did i get any email from tom
|
||||
SearchMessages 1 find mails titled recommended courses
|
||||
SearchMessages 1 detect the email containing keyword beauty
|
||||
SearchMessages 1 find emails from mom
|
||||
SearchMessages 1 find an email from abc123@outlook.com
|
||||
SearchMessages 1 search keywords keywordone keywordtwo in my emails
|
||||
SearchMessages 1 search text with words lunch together
|
||||
SearchMessages 1 show me the email about spring festival
|
||||
SearchMessages 1 search emails contain work items
|
||||
SearchMessages 1 show me the email sent from mom
|
||||
SearchMessages 1 show me the email from tom and filtering with word lunch
|
||||
SearchMessages 1 show me emails from girlfriend
|
||||
SearchMessages 1 search an email with subject background screening
|
||||
SearchMessages 1 find email titled new design
|
||||
SearchMessages 1 tell me the email from lily wong
|
||||
SearchMessages 1 list the emails contain funny picture
|
||||
SearchMessages 1 emails contains bank
|
||||
SearchMessages 1 find emails with resume
|
||||
SearchMessages 1 find an email from angela
|
||||
SearchMessages 1 query emails with bill
|
||||
SearchMessages 1 can you search my emails
|
||||
SearchMessages 1 find emails that contain malta
|
||||
SearchMessages 1 detect the email from lisa
|
||||
SearchMessages 1 find email with title production tools
|
||||
SearchMessages 1 search the emails contains money
|
||||
SearchMessages 1 search emails from mike
|
||||
SearchMessages 1 find an email on the dinner reservation
|
||||
SearchMessages 1 did i get emails from tom
|
||||
SearchMessages 1 search my emails
|
||||
SearchMessages 1 detect emails from betty
|
||||
SearchMessages 1 find an email about new year's planning
|
||||
SearchMessages 1 search emails contains coupons
|
||||
SearchMessages 1 search email with key words lunch
|
||||
SearchMessages 1 tell me the email with subject weekly report
|
||||
SearchMessages 1 show emails with "credit card"
|
||||
SearchMessages 1 search bla bla in my emails
|
||||
SearchMessages 1 show emails contain words "future plan"
|
||||
SearchMessages 1 search the email with keywords hello
|
||||
SearchMessages 1 search emails about boating
|
||||
SearchMessages 1 find an email from jay that contains halloween
|
||||
SearchMessages 1 looking for an email with hello
|
||||
SearchMessages 1 search the emails contains microsoft
|
||||
SearchMessages 1 search jensen's emails
|
||||
SearchMessages 1 search email contain outlook
|
||||
SearchMessages 1 enumerate the emails with algroithm
|
||||
SearchMessages 1 did i get the email containing keyword lunch
|
||||
SendEmail 1 email my presentation
|
||||
SendEmail 1 send and email about swim team practice
|
||||
SendEmail 1 email to cynthia and mike, that dinner last week was splendid.
|
||||
SendEmail 1 send an urgent email from my work account to christian
|
||||
SendEmail 1 send an email to jacqueline and tianyu about the test result
|
||||
SendEmail 1 send an email to larry , joseph and billy larkson
|
||||
SendEmail 1 set an email today
|
||||
SendEmail 1 send this document to an email
|
||||
SendEmail 1 send email to kai xu, mingming and my mother
|
||||
SendEmail 1 send an email about swim team practice
|
||||
SendEmail 1 start new email to friends about the club
|
||||
SendEmail 1 new email about really good talk to michelle
|
||||
SendEmail 1 send important email to evelyn and gary
|
||||
SendEmail 1 send an email to partone@gmail.com
|
||||
SendEmail 1 send the email now
|
||||
SendEmail 1 write an urgent email to bobby
|
||||
SendEmail 1 send an urgent email
|
||||
SendEmail 1 start up a new email to michelle about watching baseball
|
||||
SendEmail 1 send a new email about the problem solving to andrea, angela, and ron
|
||||
SendEmail 1 send an email to mom
|
||||
SendEmail 1 send email to partone.parttwo@outlook.com
|
||||
SendEmail 1 send an email marked with a bang to amy
|
||||
SendEmail 1 the new email is high priority that is being sent to jacob
|
||||
SendEmail 1 send an email to lily roth and abc123@microsoft.com
|
||||
SendEmail 1 send an email
|
||||
SendEmail 1 send lori a new flagged email
|
||||
SendEmail 1 send an email for me
|
||||
SendEmail 1 send the email
|
||||
SendEmail 1 send an email to jimmy klein saying this is the message about weekend plans
|
||||
SendEmail 1 send a new email about the hockey tournament to marie jane, joseph , and john
|
||||
SendEmail 1 send an email to my brother
|
||||
SendEmail 1 send thomas an email
|
||||
SendEmail 1 send an email to lu , yue and qiong about funding
|
||||
SendEmail 1 send angela an email marked as high priority
|
||||
SendEmail 1 send an important email to olivia
|
||||
SendEmail 1 new email to kimberly about wingman
|
||||
SendEmail 1 send mail to dorothy
|
||||
SendEmail 1 send an email to harry potter
|
||||
SendEmail 1 send an email to christopher carpenter about the hiking trip
|
||||
SendEmail 1 send email marked priority to yun-sim and yi
|
||||
SendEmail 1 email my brother
|
||||
SendEmail 1 send my housekeeping doc to jeffrey
|
||||
SendEmail 1 send a new high importance email to jordan
|
||||
SendEmail 1 write an email about the fundraiser
|
||||
SendEmail 1 send email to jiayi today
|
||||
SendEmail 1 email to tom white about that flower saying beautiful
|
||||
SendEmail 1 send an email to partone_parttwo@microsoft.com
|
||||
SendEmail 1 send an email about test status to mark
|
||||
SendEmail 1 send jacqueline an email with low priority
|
||||
SendEmail 1 email her the message "fine, ok"
|
||||
SendEmail 1 start a new email saying lets go to the park
|
||||
SendEmail 1 start new email about taco blog to nicole and emily
|
||||
SendEmail 1 write email
|
||||
SendEmail 1 send a mail to daniel
|
||||
SendEmail 1 email to lawrence about opening issue
|
||||
SendEmail 1 send a email to leehom wong about the piano concert saying it's wonderful
|
||||
SendEmail 1 make a new email about weather forecast
|
||||
SendEmail 1 send an email about the window that is broken
|
||||
SendEmail 1 send an email to sean about weekend plans
|
||||
SendEmail 1 send email to hannah saying test
|
||||
SendEmail 1 write email to mom subject is babysit
|
||||
SendEmail 1 send an email to a.j.ron marked as important
|
||||
SendEmail 1 send an urgent email from my work email to jack
|
||||
SendEmail 1 email the file to henry mathew
|
||||
SendEmail 1 send a new email to larry with a file attached
|
||||
SendEmail 1 send new email to christian and mark it high importance
|
||||
SendEmail 1 send an email marked for follow up to christian
|
||||
SendEmail 1 compose new email about spanish homework
|
||||
SendEmail 1 start a new email from tracy saying here is my resume
|
||||
SendEmail 1 send email to a and tian
|
||||
SendEmail 1 send a read receipt email to samuel
|
||||
SendEmail 1 write an email which title is hello and context is let's have meeting together
|
||||
SendEmail 1 send alexander a red bang email
|
||||
SendEmail 1 send an email to zachary about we can plan things let's go hiking
|
||||
SendEmail 1 i need to send an email about the words to a song
|
||||
SendEmail 1 send an email today
|
||||
SendEmail 1 send my payment visio diagram to ronald
|
||||
SendEmail 1 send email about homework plan to raymond and philip
|
||||
SendEmail 1 email to amy cooper about haha saying hello
|
||||
SendEmail 1 email to mike waters : mike, that dinner last week was splendid.
|
||||
SendEmail 1 send email to louis and mark it important
|
||||
SendEmail 1 start a new email to aaron about sleeping over tonight
|
||||
SendEmail 1 send billy an email with a red bang
|
||||
SendEmail 1 create new mail titled urgent meeting information to jonathan
|
||||
SendEmail 1 mark email for follow up and send to arthur
|
||||
SendEmail 1 start a new email about marriage counselor appointments
|
||||
SendEmail 1 send a new email to partonepartwopartthree@yahoo.com
|
||||
SendEmail 1 new email about writing documents
|
||||
SendEmail 1 send email to heather about car
|
||||
SendEmail 1 email to partoneparttwo@gmail.com
|
||||
SendEmail 1 send an email marked follow up to jerry
|
||||
SendEmail 1 will you send a marked non urgent email to james
|
||||
SendEmail 1 send an email with read receipt to peter
|
||||
SendEmail 1 send large files through email
|
||||
SendEmail 1 email to harry potter and hermione granger
|
||||
SendEmail 1 send an email to nathan with a red bang
|
||||
SendEmail 1 send a new email to nicholas and jesse about coupons
|
||||
SendEmail 1 start an email to jason about speaking up
|
||||
SendEmail 1 send a new email about facebook
|
||||
SendEmail 1 send an email to harold and bob kappus about team lunch saying same team lunch this tuesday
|
||||
ShowNext 1 move on to next mails
|
||||
ShowNext 1 move on next mail by jason
|
||||
ShowNext 1 show the next emails by wong
|
||||
ShowNext 1 show me next from mary
|
||||
ShowNext 1 next email
|
||||
ShowNext 1 go on, show me more mails
|
||||
ShowNext 1 the next important message
|
||||
ShowNext 1 move forward
|
||||
ShowNext 1 show next unread
|
||||
ShowNext 1 go to next mail
|
||||
ShowNext 1 show next email
|
||||
ShowNext 1 go forward to next mails
|
||||
ShowNext 1 next unread one
|
||||
ShowNext 1 go to the next page
|
||||
ShowNext 1 show the next messages
|
||||
ShowNext 1 show the next email from my boss
|
||||
ShowNext 1 next unread email
|
||||
ShowNext 1 show me the next five mails
|
||||
ShowNext 1 are there any unread messages? show next
|
||||
ShowNext 1 the next email
|
||||
ShowNext 1 show me the next
|
||||
ShowPrevious 1 show me the last three mails
|
||||
ShowPrevious 1 show me the previous email
|
||||
ShowPrevious 1 show previous one in inbox
|
||||
ShowPrevious 1 go to previous mails
|
||||
ShowPrevious 1 show me previous email from jack
|
||||
ShowPrevious 1 show previous in red category
|
||||
ShowPrevious 1 show me the one before
|
||||
ShowPrevious 1 previous email
|
||||
ShowPrevious 1 show the previous email from my mentor
|
||||
ShowPrevious 1 move back to last mails
|
||||
ShowPrevious 1 the previous email
|
||||
ShowPrevious 1 bring the previous one, i want to read it again
|
||||
ShowPrevious 1 back to the last one from apple
|
||||
ShowPrevious 1 previous one please
|
||||
ShowPrevious 1 show the previous one
|
||||
ShowPrevious 1 what is the previous email
|
||||
|
Не удается отобразить этот файл, потому что он содержит неожиданный символ в строке 454 и столбце 35.
|
|
|
@ -1,601 +0,0 @@
|
|||
AddFlag add flag to the email john just sent to me
|
||||
AddFlag make it flagged
|
||||
AddFlag i want to add a flag on this email
|
||||
AddFlag add a flag please
|
||||
AddFlag flag this email as important for me
|
||||
AddFlag add a flag to the last email
|
||||
AddFlag this email should be flagged
|
||||
AddFlag add flag to this email
|
||||
AddFlag put a flag on the new email
|
||||
AddFlag add a flag
|
||||
AddFlag the email to ruth needs to be flagged
|
||||
AddFlag mark as flag
|
||||
AddFlag add flag to it
|
||||
AddFlag flag
|
||||
AddFlag put a flag
|
||||
AddFlag flag this email
|
||||
AddFlag turn flag on
|
||||
AddFlag flag it
|
||||
AddFlag flag on
|
||||
AddFlag flag the current email
|
||||
AddFlag i want to add a flag
|
||||
AddFlag flag the email from davis
|
||||
AddFlag add flag
|
||||
AddFlag the email from thomas should be flagged
|
||||
AddFlag add a flag to this email
|
||||
AddFlag flag the email
|
||||
AddFlag this email needs to be flagged
|
||||
AddFlag this email need to be flagged
|
||||
AddFlag add flag to this message
|
||||
AddFlag add flag on it
|
||||
AddFlag mark the email flagged
|
||||
AddMore i need to add something else to my email to cheryl
|
||||
AddMore add more and change the message
|
||||
AddMore i need to add more to the email
|
||||
AddMore add: call me tonight after work
|
||||
AddMore write more
|
||||
AddMore i need to add more to the email message i am sending to vincent
|
||||
AddMore put some additional lines to this message
|
||||
AddMore i need to add more text
|
||||
AddMore add more to message
|
||||
AddMore add a picture
|
||||
AddMore add a file to the email
|
||||
AddMore please add it was terrible
|
||||
AddMore i would like to add more to the email message
|
||||
AddMore add more message
|
||||
AddMore need to add information to the previous email
|
||||
AddMore add some more
|
||||
AddMore add a subject
|
||||
AddMore i'd like to add more to the email
|
||||
AddMore add photo
|
||||
AddMore add more to the last email
|
||||
AddMore add, by the way, what's the plan of next step
|
||||
AddMore i would like to add more to the email
|
||||
AddMore add to body of email
|
||||
AddMore append an attachment to this email
|
||||
AddMore edit email so i can type an additional message
|
||||
AddMore add another line to the message
|
||||
AddMore i need to add further contents
|
||||
AddMore insert more lines for me please
|
||||
AddMore add something
|
||||
AddMore add more to the message
|
||||
AddMore i forgot to add an important part to that email to james . please set it up to edit
|
||||
AddMore is it ok if i add more to the email
|
||||
AddMore i need to add additional lines
|
||||
AddMore add more please
|
||||
AddMore i need to add more message
|
||||
AddMore add more details to it
|
||||
AddMore add more
|
||||
AddMore please add more
|
||||
AddMore i need to add something else to that email to donna before it is sent
|
||||
AddMore can i add more to the email
|
||||
AddMore i'd like to add a bit more to the email.
|
||||
AddMore i'd like to add a bit more to the message
|
||||
AddMore add more to roy 's email
|
||||
AddMore add more text please
|
||||
AddMore i want to add more the email
|
||||
AddMore can i add more to the message
|
||||
AddMore add more to email body
|
||||
AddMore i would like to open a new line
|
||||
AddMore i wish to add more to the message
|
||||
AddMore i am not done yet. i need to add some more details
|
||||
AddMore add did you enjoy the entire program
|
||||
AddMore ok, i need to add a few things to that
|
||||
AddMore more text
|
||||
AddMore add more to the email
|
||||
AddMore add file to email
|
||||
AddMore wait, i need to write more
|
||||
AddMore add more to it
|
||||
AddMore add more to text
|
||||
AddMore add more text
|
||||
AddMore add more don't forget to bring beer
|
||||
AddMore it isn't complete, need more contents
|
||||
AddMore add more to email
|
||||
AddMore please add, please let me know what i can bring. i'd be happy to make a side dish or dessert
|
||||
AddMore attach file
|
||||
AddMore insert more text in my email
|
||||
CancelMessages cancel my email to jane
|
||||
CancelMessages don't show me
|
||||
CancelMessages don't send that email
|
||||
CancelMessages can you cancel it
|
||||
CancelMessages don't email to her
|
||||
CancelMessages forget about the email
|
||||
CancelMessages don ' t read
|
||||
CancelMessages never mind cancel the message
|
||||
CancelMessages stop reading
|
||||
CancelMessages never mind cancel the mail
|
||||
CancelMessages okay cancel sending the mail
|
||||
CancelMessages cancel this email
|
||||
CancelMessages quit the sending
|
||||
CancelMessages cancel the mail
|
||||
CancelMessages don't send out
|
||||
CancelMessages don't send this email
|
||||
CancelMessages cancel email
|
||||
CancelMessages don't read the message
|
||||
CancelMessages neither of them
|
||||
CancelMessages don't send it
|
||||
CancelMessages cancel message
|
||||
CancelMessages stop message
|
||||
CancelMessages don't email
|
||||
CancelMessages exit
|
||||
CancelMessages i want you to cancel the email
|
||||
CancelMessages never mind, forget about the mail
|
||||
CancelMessages cancel this sending process
|
||||
CancelMessages don't send
|
||||
CancelMessages no, i don't want to send this message
|
||||
CancelMessages no just cancel the email
|
||||
CancelMessages don ' t read it
|
||||
CancelMessages nevermind cancel
|
||||
CancelMessages cancel email to natalie
|
||||
CancelMessages cancel the message
|
||||
CancelMessages don't read the email
|
||||
CancelMessages no, no, cancel the reading
|
||||
CancelMessages cancel searching the messages
|
||||
CancelMessages abort deletion
|
||||
CancelMessages cancel the email to my sister
|
||||
CancelMessages cancel this message
|
||||
CancelMessages don 't send the email
|
||||
CancelMessages cancel the email sent to alex
|
||||
CancelMessages cancel the email
|
||||
CancelMessages no don't send
|
||||
CancelMessages no don't send it
|
||||
CancelMessages no cancel it, i don't want to send the mail
|
||||
CheckMessages do i have new message
|
||||
CheckMessages does anyone send message to me just then
|
||||
CheckMessages do i receive new message
|
||||
CheckMessages does anyone send email to me just then
|
||||
CheckMessages do i have any new mail
|
||||
CheckMessages does my outlook have new email
|
||||
CheckMessages could you please check my emails
|
||||
CheckMessages please check my emails
|
||||
CheckMessages is there new email
|
||||
CheckMessages any new email now
|
||||
CheckMessages whether i get new message
|
||||
CheckMessages check outlook please
|
||||
CheckMessages whether i have new email
|
||||
CheckMessages do i receive new mail in outlook
|
||||
CheckMessages show the important emails in my inbox
|
||||
CheckMessages any new message now
|
||||
CheckMessages check my inbox
|
||||
CheckMessages show my unread mails
|
||||
CheckMessages i'd like to check my inbox
|
||||
CheckMessages do i receive new email
|
||||
CheckMessages check email
|
||||
CheckMessages whether i have new message
|
||||
CheckMessages do i get new email
|
||||
CheckMessages check email please
|
||||
CheckMessages i want to check my inbox
|
||||
CheckMessages please check my inbox
|
||||
CheckMessages could you please check my inbox
|
||||
CheckMessages show latest emails
|
||||
CheckMessages check up messages
|
||||
CheckMessages check up email
|
||||
CheckMessages whether i receive new email
|
||||
CheckMessages show my emails
|
||||
CheckMessages please check my outlook
|
||||
CheckMessages check my message
|
||||
CheckMessages i want to check my emails
|
||||
CheckMessages check my gmail
|
||||
CheckMessages check my email please
|
||||
CheckMessages any new email
|
||||
CheckMessages do i have new email now
|
||||
CheckMessages any new email available
|
||||
CheckMessages could you please check my messages
|
||||
CheckMessages do i have new email
|
||||
CheckMessages check my emails
|
||||
CheckMessages whether i get new email
|
||||
CheckMessages check my mail box
|
||||
ConfirmMessages no problem, go ahead send the mail
|
||||
ConfirmMessages just do it
|
||||
ConfirmMessages yeah right, send to alex
|
||||
ConfirmMessages ok, good, just send it
|
||||
ConfirmMessages of course, just delete the mail
|
||||
ConfirmMessages "yes, you can"
|
||||
ConfirmMessages yes, send it
|
||||
ConfirmMessages i confirm that i want to send this email
|
||||
ConfirmMessages perfect thank you
|
||||
ConfirmMessages alright, just send the message
|
||||
ConfirmMessages okay, send it now
|
||||
ConfirmMessages "sure, go ahead"
|
||||
ConfirmMessages ok, good to me, send it please
|
||||
ConfirmMessages correct, please send it.
|
||||
ConfirmMessages yes it's right
|
||||
ConfirmMessages right, send it please
|
||||
ConfirmMessages ok send the mail to may
|
||||
ConfirmMessages okay send it
|
||||
ConfirmMessages "okay, send it"
|
||||
ConfirmMessages yes that's right
|
||||
ConfirmMessages okay
|
||||
Delete can you help me delete it
|
||||
Delete delete the previous 4 emails
|
||||
Delete empty the email inbox
|
||||
Delete put it in the recycle bin
|
||||
Delete delete the email from my hotmail account
|
||||
Delete delete all emails from tom
|
||||
Delete remove the emails received yesterday
|
||||
Delete delete this message permanently
|
||||
Delete put the emails from this file folder to trash bin
|
||||
Delete remove emails that are duplicate
|
||||
Delete remove it from my inbox
|
||||
Delete delete this email
|
||||
Delete remove the email from mary
|
||||
Delete put the email to trash bin
|
||||
Delete remove emails with red flags
|
||||
Delete delete all emails received tonight
|
||||
Delete delete the email sent from mary jane
|
||||
Delete clear my inbox
|
||||
Delete delete the unread emails
|
||||
Delete delete the first email for me
|
||||
Delete delete what i just wrote
|
||||
Delete delete the red ones
|
||||
Delete put the email in the recycle bin
|
||||
Delete delete the second one
|
||||
Delete delete the last one
|
||||
Delete delete the second mail
|
||||
Forward forward message to girlfriend
|
||||
Forward forward email to girlfriend
|
||||
Forward could you please forward this email to my sister
|
||||
Forward forward to alan tonight
|
||||
Forward forward this email to patricia
|
||||
Forward forward the last email to susan
|
||||
Forward please forward this message
|
||||
Forward forward the email from john smith to michelle by saying fyi
|
||||
Forward forward to thomas please
|
||||
Forward forward to partoneparttwo@gmail.com next monday
|
||||
Forward forward to wife by saying i love you
|
||||
Forward forward to deborah with a message saying that i don't want that
|
||||
Forward forward to dorothy by typing i agree with it
|
||||
Forward forward the email to dad
|
||||
Forward forward to my boss and attach the schedule file
|
||||
Forward forward emails to gabriel
|
||||
Forward forward this email to gary brown please
|
||||
Forward forward by saying if you interest to rebecca
|
||||
Forward forward this email
|
||||
Forward forward the email from melissa to peter
|
||||
Forward forward to brian potter tonight
|
||||
Forward forward this email to joseph
|
||||
Forward forward email
|
||||
Forward please forward this email to partoneparttwo@163.com
|
||||
Forward could you forward this message to ronald and roy
|
||||
Forward please forward this email to albert by typing everything goes fine
|
||||
Forward please forward to benjamin
|
||||
Forward forward all files from sally to austin
|
||||
Forward forward this email to partone dot parttwo at gmail dot com
|
||||
Forward forward to mom
|
||||
Forward forward this email to eugene by typing what do you think
|
||||
Forward please forward this email to partoneparttwo@outlook.com
|
||||
None 2
|
||||
None the first one
|
||||
None the second one
|
||||
None the third one
|
||||
None 1
|
||||
None 3
|
||||
QueryLastText please tell me who emailed me last
|
||||
QueryLastText open the last email
|
||||
QueryLastText come to the last
|
||||
QueryLastText what was the last email
|
||||
QueryLastText what is the lastest email i received from dad
|
||||
QueryLastText the last email
|
||||
QueryLastText whose email just then
|
||||
QueryLastText show me the lastest email
|
||||
QueryLastText who recently emailed me
|
||||
QueryLastText who emailed me last
|
||||
QueryLastText who sent me the email lastly yesterday
|
||||
QueryLastText what was the last email i got from steve edwards
|
||||
QueryLastText who email me just now
|
||||
QueryLastText show the last email
|
||||
QueryLastText who emailed me just now
|
||||
QueryLastText open the lastest email i got
|
||||
QueryLastText what harry last email said
|
||||
QueryLastText i want to see the last email
|
||||
QueryLastText what did mom just say
|
||||
QueryLastText show me the newest email
|
||||
QueryLastText what was the last email i got from dad
|
||||
QueryLastText what henry just said
|
||||
QueryLastText last email
|
||||
QueryLastText who texted me
|
||||
QueryLastText can you tell me the last email i received
|
||||
QueryLastText who sent me the mail just now
|
||||
QueryLastText whose email now
|
||||
QueryLastText go to the last one
|
||||
QueryLastText what is the last email i received today
|
||||
QueryLastText who emailed me
|
||||
QueryLastText what eric watson just said
|
||||
QueryLastText who texted me just now
|
||||
ReadAloud read me the email on apple
|
||||
ReadAloud read email
|
||||
ReadAloud read my most recent email
|
||||
ReadAloud read my last email
|
||||
ReadAloud read aloud my new email
|
||||
ReadAloud read me my latest emails
|
||||
ReadAloud read me the newest email
|
||||
ReadAloud read my last email out to me
|
||||
ReadAloud read out the email from liu about transfer
|
||||
ReadAloud read unread message
|
||||
ReadAloud read it
|
||||
ReadAloud read emails
|
||||
ReadAloud read last incoming emails
|
||||
ReadAloud read my recent email to me
|
||||
ReadAloud read my emails from patty
|
||||
ReadAloud read latest email
|
||||
ReadAloud read the last email
|
||||
ReadAloud read the latest email from mom
|
||||
ReadAloud read email to me
|
||||
ReadAloud read my email messages
|
||||
ReadAloud read me the last emails of the five minutes
|
||||
ReadAloud read first email in the linked inbox
|
||||
ReadAloud read my second email
|
||||
ReadAloud read last email received
|
||||
ReadAloud read emails from clay
|
||||
ReadAloud read new email from david ma
|
||||
ReadAloud read my email from tyler swift
|
||||
ReadAloud read me the email titled happy new year
|
||||
ReadAloud read my email please
|
||||
ReadAloud read todays mail
|
||||
ReadAloud read my email to me
|
||||
ReadAloud read my email from baby
|
||||
ReadAloud read most recent email
|
||||
ReadAloud read first email in link box
|
||||
ReadAloud read google mail
|
||||
ReadAloud read email from dawn
|
||||
ReadAloud read darren's mail on the movie
|
||||
ReadAloud read me the email sent on thanksgiving day
|
||||
ReadAloud read recent email
|
||||
ReadAloud read my notification
|
||||
ReadAloud read my inbox
|
||||
ReadAloud read new message
|
||||
ReadAloud read mary grace white email
|
||||
ReadAloud please read my last email
|
||||
ReadAloud read my recent email
|
||||
ReadAloud read the first email in hotmail
|
||||
ReadAloud read me the email
|
||||
ReadAloud read me the emails from agatha
|
||||
ReadAloud read my emails
|
||||
ReadAloud read last mail
|
||||
ReadAloud read the first email
|
||||
ReadAloud read the last email message
|
||||
ReadAloud read out darren's mail
|
||||
ReadAloud read email from kat
|
||||
ReadAloud read new email
|
||||
ReadAloud read my email from hubby
|
||||
ReadAloud read my new email
|
||||
ReadAloud read me the last email claude sent
|
||||
ReadAloud read the latest email from steve lip
|
||||
ReadAloud read my recent email message please
|
||||
ReadAloud read me the recent email titled abcd from jessica
|
||||
ReadAloud read unread email
|
||||
ReadAloud read the email
|
||||
ReadAloud read the email on auto repair
|
||||
ReadAloud read my outlook email
|
||||
ReadAloud read today's mail
|
||||
ReadAloud read me dylan's email sent on yesterday
|
||||
ReadAloud read my new emails
|
||||
ReadAloud read aloud the christmas party email
|
||||
ReadAloud read please
|
||||
ReadAloud read email from mum
|
||||
ReadAloud could you read out the email on how to use the new tool
|
||||
ReadAloud read my recent email messages
|
||||
ReadAloud read me jessica's email on dress code for the party
|
||||
ReadAloud read me the email on thanksgiving day
|
||||
ReadAloud read me my last hotmail email
|
||||
ReadAloud can you read my emails
|
||||
ReadAloud read out xu's email about apple's news
|
||||
ReadAloud read the latest email i sent
|
||||
ReadAloud can you read my last email
|
||||
Reply reply by saying i love you
|
||||
Reply reply yee ha
|
||||
Reply reply with hello
|
||||
Reply email back
|
||||
Reply reply to the first one
|
||||
Reply email back i will call you back
|
||||
Reply send email back
|
||||
Reply reply by saying yes
|
||||
Reply respond to lore hound
|
||||
Reply create a response to the email by saying pls send me the picture again
|
||||
Reply respond to the email by saying i am busy today
|
||||
Reply reply yee hello
|
||||
Reply reply by email thank you very much best regards jun
|
||||
Reply reply required to an email
|
||||
Reply respond i ' m sick i can ' t do it
|
||||
Reply reply to the email
|
||||
Reply send the response with i've already know
|
||||
Reply reply that i am busy
|
||||
Reply reply to edward
|
||||
Reply reply to email i am busy now
|
||||
Reply reply we'll see you later
|
||||
Reply reply to susan
|
||||
Reply make a response with thank you very much
|
||||
Reply respond to nathan
|
||||
Reply how to reply to an email
|
||||
Reply reply to my last email
|
||||
Reply return siberian huskies mobile
|
||||
Reply return barbara on mobile
|
||||
Reply reply by typing hello
|
||||
Reply reply
|
||||
Reply reply yes boss.
|
||||
SearchMessages show me emails from clara chan
|
||||
SearchMessages email sent from lisa
|
||||
SearchMessages search keywordsone keywordstwo from inbox
|
||||
SearchMessages did i get any email from tom
|
||||
SearchMessages find mails titled recommended courses
|
||||
SearchMessages detect the email containing keyword beauty
|
||||
SearchMessages find emails from mom
|
||||
SearchMessages find an email from abc123@outlook.com
|
||||
SearchMessages search keywords keywordone keywordtwo in my emails
|
||||
SearchMessages search text with words lunch together
|
||||
SearchMessages show me the email about spring festival
|
||||
SearchMessages search emails contain work items
|
||||
SearchMessages show me the email sent from mom
|
||||
SearchMessages show me the email from tom and filtering with word lunch
|
||||
SearchMessages show me emails from girlfriend
|
||||
SearchMessages search an email with subject background screening
|
||||
SearchMessages find email titled new design
|
||||
SearchMessages tell me the email from lily wong
|
||||
SearchMessages list the emails contain funny picture
|
||||
SearchMessages emails contains bank
|
||||
SearchMessages find emails with resume
|
||||
SearchMessages find an email from angela
|
||||
SearchMessages query emails with bill
|
||||
SearchMessages can you search my emails
|
||||
SearchMessages find emails that contain malta
|
||||
SearchMessages detect the email from lisa
|
||||
SearchMessages find email with title production tools
|
||||
SearchMessages search the emails contains money
|
||||
SearchMessages search emails from mike
|
||||
SearchMessages find an email on the dinner reservation
|
||||
SearchMessages did i get emails from tom
|
||||
SearchMessages search my emails
|
||||
SearchMessages detect emails from betty
|
||||
SearchMessages find an email about new year's planning
|
||||
SearchMessages search emails contains coupons
|
||||
SearchMessages search email with key words lunch
|
||||
SearchMessages tell me the email with subject weekly report
|
||||
SearchMessages show emails with "credit card"
|
||||
SearchMessages search bla bla in my emails
|
||||
SearchMessages show emails contain words "future plan"
|
||||
SearchMessages search the email with keywords hello
|
||||
SearchMessages search emails about boating
|
||||
SearchMessages find an email from jay that contains halloween
|
||||
SearchMessages looking for an email with hello
|
||||
SearchMessages search the emails contains microsoft
|
||||
SearchMessages search jensen's emails
|
||||
SearchMessages search email contain outlook
|
||||
SearchMessages enumerate the emails with algroithm
|
||||
SearchMessages did i get the email containing keyword lunch
|
||||
SendEmail email my presentation
|
||||
SendEmail send and email about swim team practice
|
||||
SendEmail email to cynthia and mike, that dinner last week was splendid.
|
||||
SendEmail send an urgent email from my work account to christian
|
||||
SendEmail send an email to jacqueline and tianyu about the test result
|
||||
SendEmail send an email to larry , joseph and billy larkson
|
||||
SendEmail set an email today
|
||||
SendEmail send this document to an email
|
||||
SendEmail send email to kai xu, mingming and my mother
|
||||
SendEmail send an email about swim team practice
|
||||
SendEmail start new email to friends about the club
|
||||
SendEmail new email about really good talk to michelle
|
||||
SendEmail send important email to evelyn and gary
|
||||
SendEmail send an email to partone@gmail.com
|
||||
SendEmail send the email now
|
||||
SendEmail write an urgent email to bobby
|
||||
SendEmail send an urgent email
|
||||
SendEmail start up a new email to michelle about watching baseball
|
||||
SendEmail send a new email about the problem solving to andrea, angela, and ron
|
||||
SendEmail send an email to mom
|
||||
SendEmail send email to partone.parttwo@outlook.com
|
||||
SendEmail send an email marked with a bang to amy
|
||||
SendEmail the new email is high priority that is being sent to jacob
|
||||
SendEmail send an email to lily roth and abc123@microsoft.com
|
||||
SendEmail send an email
|
||||
SendEmail send lori a new flagged email
|
||||
SendEmail send an email for me
|
||||
SendEmail send the email
|
||||
SendEmail send an email to jimmy klein saying this is the message about weekend plans
|
||||
SendEmail send a new email about the hockey tournament to marie jane, joseph , and john
|
||||
SendEmail send an email to my brother
|
||||
SendEmail send thomas an email
|
||||
SendEmail send an email to lu , yue and qiong about funding
|
||||
SendEmail send angela an email marked as high priority
|
||||
SendEmail send an important email to olivia
|
||||
SendEmail new email to kimberly about wingman
|
||||
SendEmail send mail to dorothy
|
||||
SendEmail send an email to harry potter
|
||||
SendEmail send an email to christopher carpenter about the hiking trip
|
||||
SendEmail send email marked priority to yun-sim and yi
|
||||
SendEmail email my brother
|
||||
SendEmail send my housekeeping doc to jeffrey
|
||||
SendEmail send a new high importance email to jordan
|
||||
SendEmail write an email about the fundraiser
|
||||
SendEmail send email to jiayi today
|
||||
SendEmail email to tom white about that flower saying beautiful
|
||||
SendEmail send an email to partone_parttwo@microsoft.com
|
||||
SendEmail send an email about test status to mark
|
||||
SendEmail send jacqueline an email with low priority
|
||||
SendEmail email her the message "fine, ok"
|
||||
SendEmail start a new email saying lets go to the park
|
||||
SendEmail start new email about taco blog to nicole and emily
|
||||
SendEmail write email
|
||||
SendEmail send a mail to daniel
|
||||
SendEmail email to lawrence about opening issue
|
||||
SendEmail send a email to leehom wong about the piano concert saying it's wonderful
|
||||
SendEmail make a new email about weather forecast
|
||||
SendEmail send an email about the window that is broken
|
||||
SendEmail send an email to sean about weekend plans
|
||||
SendEmail send email to hannah saying test
|
||||
SendEmail write email to mom subject is babysit
|
||||
SendEmail send an email to a.j.ron marked as important
|
||||
SendEmail send an urgent email from my work email to jack
|
||||
SendEmail email the file to henry mathew
|
||||
SendEmail send a new email to larry with a file attached
|
||||
SendEmail send new email to christian and mark it high importance
|
||||
SendEmail send an email marked for follow up to christian
|
||||
SendEmail compose new email about spanish homework
|
||||
SendEmail start a new email from tracy saying here is my resume
|
||||
SendEmail send email to a and tian
|
||||
SendEmail send a read receipt email to samuel
|
||||
SendEmail write an email which title is hello and context is let's have meeting together
|
||||
SendEmail send alexander a red bang email
|
||||
SendEmail send an email to zachary about we can plan things let's go hiking
|
||||
SendEmail i need to send an email about the words to a song
|
||||
SendEmail send an email today
|
||||
SendEmail send my payment visio diagram to ronald
|
||||
SendEmail send email about homework plan to raymond and philip
|
||||
SendEmail email to amy cooper about haha saying hello
|
||||
SendEmail email to mike waters : mike, that dinner last week was splendid.
|
||||
SendEmail send email to louis and mark it important
|
||||
SendEmail start a new email to aaron about sleeping over tonight
|
||||
SendEmail send billy an email with a red bang
|
||||
SendEmail create new mail titled urgent meeting information to jonathan
|
||||
SendEmail mark email for follow up and send to arthur
|
||||
SendEmail start a new email about marriage counselor appointments
|
||||
SendEmail send a new email to partonepartwopartthree@yahoo.com
|
||||
SendEmail new email about writing documents
|
||||
SendEmail send email to heather about car
|
||||
SendEmail email to partoneparttwo@gmail.com
|
||||
SendEmail send an email marked follow up to jerry
|
||||
SendEmail will you send a marked non urgent email to james
|
||||
SendEmail send an email with read receipt to peter
|
||||
SendEmail send large files through email
|
||||
SendEmail email to harry potter and hermione granger
|
||||
SendEmail send an email to nathan with a red bang
|
||||
SendEmail send a new email to nicholas and jesse about coupons
|
||||
SendEmail start an email to jason about speaking up
|
||||
SendEmail send a new email about facebook
|
||||
SendEmail send an email to harold and bob kappus about team lunch saying same team lunch this tuesday
|
||||
ShowNext move on to next mails
|
||||
ShowNext move on next mail by jason
|
||||
ShowNext show the next emails by wong
|
||||
ShowNext show me next from mary
|
||||
ShowNext next email
|
||||
ShowNext go on, show me more mails
|
||||
ShowNext the next important message
|
||||
ShowNext move forward
|
||||
ShowNext show next unread
|
||||
ShowNext go to next mail
|
||||
ShowNext show next email
|
||||
ShowNext go forward to next mails
|
||||
ShowNext next unread one
|
||||
ShowNext go to the next page
|
||||
ShowNext show the next messages
|
||||
ShowNext show the next email from my boss
|
||||
ShowNext next unread email
|
||||
ShowNext show me the next five mails
|
||||
ShowNext are there any unread messages? show next
|
||||
ShowNext the next email
|
||||
ShowNext show me the next
|
||||
ShowPrevious show me the last three mails
|
||||
ShowPrevious show me the previous email
|
||||
ShowPrevious show previous one in inbox
|
||||
ShowPrevious go to previous mails
|
||||
ShowPrevious show me previous email from jack
|
||||
ShowPrevious show previous in red category
|
||||
ShowPrevious show me the one before
|
||||
ShowPrevious previous email
|
||||
ShowPrevious show the previous email from my mentor
|
||||
ShowPrevious move back to last mails
|
||||
ShowPrevious the previous email
|
||||
ShowPrevious bring the previous one, i want to read it again
|
||||
ShowPrevious back to the last one from apple
|
||||
ShowPrevious previous one please
|
||||
ShowPrevious show the previous one
|
||||
ShowPrevious what is the previous email
|
||||
|
|
@ -1,76 +0,0 @@
|
|||
Label Weight Text
|
||||
AddFlag 1 add flag to the email mary just sent to me
|
||||
AddFlag 1 make the email flagged
|
||||
AddFlag 1 i want to add a flag to this email
|
||||
AddFlag 1 attach a flag please
|
||||
AddFlag 1 flag this email as important
|
||||
AddMore 1 i need to add something more to my email to cheryl
|
||||
AddMore 1 add more and revise the message
|
||||
AddMore 1 i need to write more to the email
|
||||
AddMore 1 add: call me tonight after dinner
|
||||
AddMore 1 write thanks
|
||||
CancelMessages 1 cancel my draft to jane
|
||||
CancelMessages 1 don't show me the email
|
||||
CancelMessages 1 don't send the last email
|
||||
CancelMessages 1 can you cancel it, please
|
||||
CancelMessages 1 don't send to her
|
||||
CheckMessages 1 do i have new messages?
|
||||
CheckMessages 1 does anyone send message to me now
|
||||
CheckMessages 1 do i receive new message from work
|
||||
CheckMessages 1 does anyone send email to me just now
|
||||
CheckMessages 1 do i have any new mails?
|
||||
ConfirmMessages 1 no problem, go ahead and send the mail
|
||||
ConfirmMessages 1 do it
|
||||
ConfirmMessages 1 yes, send to alex
|
||||
ConfirmMessages 1 ok, good, just go ahead and send it
|
||||
ConfirmMessages 1 sure, just delete the mail
|
||||
Delete 1 please delete it
|
||||
Delete 1 delete the last 4 emails
|
||||
Delete 1 empty the email inbox and move them to recycle bin
|
||||
Delete 1 put the email in the recycle bin
|
||||
Delete 1 delete the email from my personal account
|
||||
Forward 1 forward message to mary
|
||||
Forward 1 forward email to mary
|
||||
Forward 1 could you please forward this email to my brother
|
||||
Forward 1 forward to allen tonight
|
||||
Forward 1 forward this email to patricia now
|
||||
None 1 etherb wger3b
|
||||
None 1 the first one wd
|
||||
None 1 the second one, nothing new
|
||||
None 1 the third one, nonsense
|
||||
None 1 wdfwdf wefqwefw edfwf
|
||||
QueryLastText 1 please tell me who sent me the last email
|
||||
QueryLastText 1 open the lastest email
|
||||
QueryLastText 1 move to the last
|
||||
QueryLastText 1 what was in the lastest email
|
||||
QueryLastText 1 what is the lastest email i received from mom
|
||||
ReadAloud 1 read me the email on iphone
|
||||
ReadAloud 1 read email out loud
|
||||
ReadAloud 1 read my most recent email to me
|
||||
ReadAloud 1 read my lastest email
|
||||
ReadAloud 1 read aloud my newest email
|
||||
Reply 1 reply and say i love you
|
||||
Reply 1 reply yeah
|
||||
Reply 1 reply with hi
|
||||
Reply 1 email back now
|
||||
Reply 1 reply to the first email
|
||||
SearchMessages 1 show me emails from clara chen
|
||||
SearchMessages 1 email sent from beth
|
||||
SearchMessages 1 search keyword no. one keyword no. two from inbox
|
||||
SearchMessages 1 did i get any email from chris
|
||||
SearchMessages 1 find mails with the title recommended courses
|
||||
SendEmail 1 email my presentation to manager
|
||||
SendEmail 1 send and email about swim team practice this week
|
||||
SendEmail 1 email to cynthia and mike, that the dinner last week was great.
|
||||
SendEmail 1 send an urgent email from my work account to christian now
|
||||
SendEmail 1 send an email to jacqueline about the great test result
|
||||
ShowNext 1 move on to the next mail
|
||||
ShowNext 1 move to next email by jason
|
||||
ShowNext 1 show the next email by wong
|
||||
ShowNext 1 show me next from lily
|
||||
ShowNext 1 next email, please
|
||||
ShowPrevious 1 show me the last four mails
|
||||
ShowPrevious 1 show me the previous email in inbox
|
||||
ShowPrevious 1 show the previous one in inbox
|
||||
ShowPrevious 1 go to the previous mail
|
||||
ShowPrevious 1 show me the previous email from jack
|
||||
|
|
|
@ -1,75 +0,0 @@
|
|||
AddFlag add flag to the email mary just sent to me
|
||||
AddFlag make the email flagged
|
||||
AddFlag i want to add a flag to this email
|
||||
AddFlag attach a flag please
|
||||
AddFlag flag this email as important
|
||||
AddMore i need to add something more to my email to cheryl
|
||||
AddMore add more and revise the message
|
||||
AddMore i need to write more to the email
|
||||
AddMore add: call me tonight after dinner
|
||||
AddMore write thanks
|
||||
CancelMessages cancel my draft to jane
|
||||
CancelMessages don't show me the email
|
||||
CancelMessages don't send the last email
|
||||
CancelMessages can you cancel it, please
|
||||
CancelMessages don't send to her
|
||||
CheckMessages do i have new messages?
|
||||
CheckMessages does anyone send message to me now
|
||||
CheckMessages do i receive new message from work
|
||||
CheckMessages does anyone send email to me just now
|
||||
CheckMessages do i have any new mails?
|
||||
ConfirmMessages no problem, go ahead and send the mail
|
||||
ConfirmMessages do it
|
||||
ConfirmMessages yes, send to alex
|
||||
ConfirmMessages ok, good, just go ahead and send it
|
||||
ConfirmMessages sure, just delete the mail
|
||||
Delete please delete it
|
||||
Delete delete the last 4 emails
|
||||
Delete empty the email inbox and move them to recycle bin
|
||||
Delete put the email in the recycle bin
|
||||
Delete delete the email from my personal account
|
||||
Forward forward message to mary
|
||||
Forward forward email to mary
|
||||
Forward could you please forward this email to my brother
|
||||
Forward forward to allen tonight
|
||||
Forward forward this email to patricia now
|
||||
None etherb wger3b
|
||||
None the first one wd
|
||||
None the second one, nothing new
|
||||
None the third one, nonsense
|
||||
None wdfwdf wefqwefw edfwf
|
||||
QueryLastText please tell me who sent me the last email
|
||||
QueryLastText open the lastest email
|
||||
QueryLastText move to the last
|
||||
QueryLastText what was in the lastest email
|
||||
QueryLastText what is the lastest email i received from mom
|
||||
ReadAloud read me the email on iphone
|
||||
ReadAloud read email out loud
|
||||
ReadAloud read my most recent email to me
|
||||
ReadAloud read my lastest email
|
||||
ReadAloud read aloud my newest email
|
||||
Reply reply and say i love you
|
||||
Reply reply yeah
|
||||
Reply reply with hi
|
||||
Reply email back now
|
||||
Reply reply to the first email
|
||||
SearchMessages show me emails from clara chen
|
||||
SearchMessages email sent from beth
|
||||
SearchMessages search keyword no. one keyword no. two from inbox
|
||||
SearchMessages did i get any email from chris
|
||||
SearchMessages find mails with the title recommended courses
|
||||
SendEmail email my presentation to manager
|
||||
SendEmail send and email about swim team practice this week
|
||||
SendEmail email to cynthia and mike, that the dinner last week was great.
|
||||
SendEmail send an urgent email from my work account to christian now
|
||||
SendEmail send an email to jacqueline about the great test result
|
||||
ShowNext move on to the next mail
|
||||
ShowNext move to next email by jason
|
||||
ShowNext show the next email by wong
|
||||
ShowNext show me next from lily
|
||||
ShowNext next email, please
|
||||
ShowPrevious show me the last four mails
|
||||
ShowPrevious show me the previous email in inbox
|
||||
ShowPrevious show the previous one in inbox
|
||||
ShowPrevious go to the previous mail
|
||||
ShowPrevious show me the previous email from jack
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,66 +0,0 @@
|
|||
|
||||
# ? What is a product key?
|
||||
```markdown
|
||||
<p>A product key is a 25-character code that comes with a Microsoft Office product. The product key allows you to install and activate the Office product on your PC.</p>
|
||||
```
|
||||
|
||||
# ? Where do I find my Office product key?
|
||||
```markdown
|
||||
<p>Your product key is 25 characters and is found in different locations depending on how you acquired your Office product.</p><ul><li>If you purchased Office on a traditional disc the product key should be located inside the package on a label on the card opposite the disc holder on the left side of the DVD case.</li><li>If you purchased an Office product key card the product key should be located inside the package on a label on the left side of the case.</li><li>If the product key is not inside the package, please review the instructions in the package.</li><li>If you purchased and downloaded Office online the product key should be on the confirmation page and/or email that was sent to you when you downloaded the software.</li><li>If you purchased a new computer with Office preloaded the manufacturer of your computer should have included either media or a Microsoft Product Identifier card in the box that has the product key. The product key is located on the Certificate of Authenticity.</li><li>
|
||||
If you need additional help locating your product key, please refer to these resources:
|
||||
<ul class="none-list"><li><a target="_blank" href="https://support.office.com/en-us/article/Find-your-product-key-for-Office-2010-1e8ef39c-2bd4-4581-a0ae-5cf25ebed489?ui=en-US&rs=en-US&ad=US">Office 2010</a></li><li><a target="_blank" href="https://support.office.com/en-US/article/Find-your-product-key-for-Office-for-Mac-2011-670b19d6-7743-45ae-a53c-66080cec7e33">Office for Mac 2011</a></li></ul></li></ul>
|
||||
```
|
||||
|
||||
# ? How long does it take to download?
|
||||
```markdown
|
||||
<p>Download times vary by location, internet connection speed and the size of the Office product you are downloading. It is recommended only high-speed broadband connections are used to download your file(s).</p>
|
||||
```
|
||||
|
||||
# ? Where do I download the file to?
|
||||
```markdown
|
||||
<p>We recommend that you download the file to your Desktop, so it is easy to find to initiate the installation. You can download the file anywhere you like, just make sure that you remember the location where you saved it, and the name of the file. Do not rename the file that you are downloading; it may cause installation problems.</p>
|
||||
```
|
||||
|
||||
# ? What happens after I download?
|
||||
```markdown
|
||||
<p>After the download has completed, go to the location that you saved the file at and double click on the new icon to start the installation.</p>
|
||||
```
|
||||
|
||||
# ? I am having trouble downloading, what else should I do?
|
||||
```markdown
|
||||
<p>Some pop up blocker and firewall programs, as well as proxy servers, may prevent communication with our server. If you utilize these types of applications, you may need to disable them. Please check your system for and disable any of the following and remember to turn them back on when you’re done:</p><p><strong>Firewall</strong> - These are designed to stop downloads from non-approved sites.
|
||||
<br /><strong>Anti Virus Programs </strong>- These will try to stop or scan a download. It may cause the download to be corrupt or damaged.
|
||||
<br /><strong>Download Monitors</strong> - Programs such as GoZilla! and NetZip are not compatible with our server and will sometimes block or stop downloads.
|
||||
<br /><strong>Screen Savers</strong> - In some instances, screen savers can interfere with the download process. If you have a screen saver that is starting while you are downloading, please disable it temporarily until your download is complete.
|
||||
</p><p>If you require assistance disabling any of the products noted, please contact the manufacturer’s technical support.</p>
|
||||
```
|
||||
|
||||
# ? What if the download stops or is interrupted before it is complete?
|
||||
```markdown
|
||||
<p>If you become disconnected while files are being downloaded through your web browser, reconnect to the internet and retry your download.</p>
|
||||
```
|
||||
|
||||
# ? I need more information about installing Office.
|
||||
```markdown
|
||||
<p>Click on the links below for information about installing Office products</p><ul class="none-list"><li><a target="_blank" href="https://support.office.com/en-US/article/Install-Office-2010-1b8f3c9b-bdd2-4a4f-8c88-aa756546529d">Office 2010</a></li><li><a target="_blank" href="https://support.office.com/en-us/article/Resources-to-help-you-upgrade-from-Office-2007-servers-and-clients-b2acaeca-4986-40f4-92b7-a1bdd06e549d">Office 2007</a></li><li><a target="_blank" href="https://support.office.com/article/490d054b-46e4-42d7-b5e0-7e9fba838053">Office for Mac 2011</a></li></ul>
|
||||
```
|
||||
|
||||
# ? How do I activate my Office products?
|
||||
```markdown
|
||||
<p>Click on the links below for information about activating Office products</p><ul class="none-list"><li><a target="_blank" href="https://support.office.com/en-us/article/Activate-Office-2010-1fe7340c-50e2-458f-8677-f57f5a140f46?ui=en-US&rs=en-US&ad=US">Office 2010</a></li><li><a target="_blank" href="https://support.office.com/en-us/article/Activate-Office-for-Mac-2011-7d2fe037-7a2f-4381-9a65-3f40e7915b79?ui=en-US&rs=en-US&ad=US">Office for Mac 2011</a></li></ul>
|
||||
```
|
||||
|
||||
# ? How can I get support?
|
||||
```markdown
|
||||
<p>Office 2007: If you run into problems, try searching <a target="_blank" href="https://support.office.com/en-us/Search/results?query=install+office+2007">online help</a> or get answers from the <a target="_blank" href="https://answers.microsoft.com/en-us/office/forum/office_2007-office_install?sort=lastreplydate&dir=desc&tab=threads&status=all&mod=&modAge=&advFil=&postedAfter=&postedBefore=&threadType=all&tm=1461950641627">Office Community</a>. Need to talk to a person? <a target="_blank" href="https://support.microsoft.com/ContactUs">Answer Desk</a> is ready to help you with whatever you need.</p><p>Office 2010: If you run into problems, try searching <a target="_blank" href="https://support.office.com/en-us/Search/results?query=install+office+2010">online help</a> or get answers from the <a target="_blank" href="https://answers.microsoft.com/en-us/office/forum/office_2010-office_install?sort=lastreplydate&dir=desc&tab=threads&status=all&mod=&modAge=&advFil=&postedAfter=&postedBefore=&threadType=all&tm=1461950692029">Office Community</a>. Need to talk to a person? <a target="_blank" href="https://support.microsoft.com/ContactUs">Answer Desk</a> is ready to help you with whatever you need.</p><p>Office for Mac 2011: If you run into problems, try searching <a target="_blank" href="https://support.office.com/en-us/Search/results?query=install+office+2011">online help</a> or get answers from the <a target="_blank" href="https://answers.microsoft.com/en-us/mac/forum/macoffice2011">Office Community</a>. Need to talk to a person? <a target="_blank" href="https://support.microsoft.com/ContactUs">Answer Desk</a> is ready to help you with whatever you need.</p>
|
||||
```
|
||||
|
||||
# ? Can I purchase backup media for Office 2010 Products?
|
||||
```markdown
|
||||
<p>Backup media for Office 2010 products is no longer available. You can create your own backup media by downloading to an external storage (USB) device or to a DVD.</p>
|
||||
```
|
||||
|
||||
# ? How do I find my product key that was stored on the previous Office download site?
|
||||
```markdown
|
||||
<p>If you stored your product key on a previous version of the Office download site and are unable to find it based on how you acquired Office, check your email. If you are an Office 2010 or Office 2007 download customer, you were sent an email message containing the product key when you first set up your account. The email would have come from the domain trymicrosoftoffice.com. If you downloaded Office for Mac 2011, you can contact Support through the <strong>How Can I Get Support?</strong> section in the FAQ. You’ll need to provide the email address that you use to sign in to the download site along with your product and purchase information.</p>
|
||||
```
|
||||
|
|
@ -1,70 +0,0 @@
|
|||
|
||||
# ? What's the difference between 32-bit and 64-bit versions of Windows?
|
||||
```markdown
|
||||
<p>The terms 32-bit and 64-bit refer to the way a computer's processor (also called a CPU) handles information. The 64-bit version of Windows handles large amounts of random access memory (RAM) more effectively than a 32-bit system. Not all devices can run the 64-bit versions of Windows.</p>
|
||||
```
|
||||
|
||||
# ? How do I tell if my computer can run a 64-bit version of Windows?
|
||||
```markdown
|
||||
<p>If you have a Windows operating system installed, open File Explorer or This PC.</p><ol><li>Right click on This PC or Computer in the navigation pane and select Properties.</li><li>In the System information screen, find the System type entry. This will indicate what type of processor your device has.</li></ol><p>If you do not have an operating system installed, you should refer to the documentation that came with the device. Most device and processor manufacturers also provide information regarding processor capabilities on their websites.</p><p>For other options and more information on Windows 7 visit the <a target="_blank" href="https://windows.microsoft.com/en-us/windows/32-bit-and-64-bit-windows#1TC=windows-7">Windows 7 32-bit and 64-bit FAQ</a> page or download and run the free <a target="_blank" href="https://www.microsoft.com/en-us/download/details.aspx?id=20">Windows 7 Upgrade Advisor</a>.</p>
|
||||
```
|
||||
|
||||
# ? How do I find my Windows product key?
|
||||
```markdown
|
||||
<p>The product key is located inside the product packaging, on the receipt or confirmation page for a digital purchase or in a confirmation e-mail that shows you purchased Windows. If you purchased a digital copy from <a target="_blank" href="https://www.microsoftstore.com/store/msusa/en_US/home">Microsoft Store</a>, you can locate your product key in your Account under Digital Content.</p><p>Windows 7</p><p>The product key is located inside the box that the Windows DVD came in, on the DVD, on the receipt or confirmation page for a digital purchase or in a confirmation e-mail that shows you purchased Windows. If you purchased a digital copy from <a target="_blank" href="https://www.microsoftstore.com/store/msusa/en_US/home">Microsoft Store</a>, you can locate your product key in your Account under Digital Content.</p><p>Academic Products</p><p>Your product key is located on the receipt page when you purchase or in the Order History section of the WebStore from which you ordered the software.</p><p>Devices Pre-Installed with Windows</p><p>Before using operating system copies from this site for install, re-install or recovery on devices with pre-installed operating systems, see your device manufacturer or reseller for the customized drivers and applications specific to your machine. Using operating systems copied from this site for install, re-install or recovery may void your support agreement with your manufacturer or reseller. Any drivers or programs that were installed by the device manufacturer or reseller may be removed during installation.</p><p>Windows 8.1 & 10: The product key may be embedded on the motherboard or may be on the Certificate of Authenticity sticker on the bottom of the device.</p><p>Windows 7: For devices that came with Windows 7 pre-installed, the product key may be on the Certificate of Authenticity sticker on the bottom of the device.</p><p>For more information about Windows product keys and genuine Microsoft products, see <a target="_blank" href="https://windows.microsoft.com/en-us/windows-8/what-is-product-key">What is a product key</a> and <a target="_blank" href="https://www.microsoft.com/en-us/howtotell/default.aspx">How to Tell</a> site.</p>
|
||||
```
|
||||
|
||||
# ? I purchased my copy of Windows through a university. Can I download it here?
|
||||
```markdown
|
||||
<p>Yes, but you’ll need your product key. Go to the <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/vlacademic">Academic Products</a> download page and select the version of Windows to begin.</p>
|
||||
```
|
||||
|
||||
# ? My Windows 7 product key won't verify. What's the problem?
|
||||
```markdown
|
||||
<p>The most common issue is the use of a product key for a product not currently supported by the site such as an Upgrade key, an MSDN key, product keys for pre-installed media or an Enterprise edition key. For access to MSDN products or Enterprise edition visit the <a target="_blank" href="https://msdn.microsoft.com/en-us/default.aspx">MSDN Portal</a> or the <a target="_blank" href="https://www.microsoft.com/licensing/servicecenter/default.aspx">Volume Licensing Service Center</a>. Upgrades and pre-installed media are not currently supported by the tools on the site.</p><p>If you believe you have a valid product key and are still receiving an error, please <a target="_blank" href="https://support.microsoft.com/en-us/contactus">contact Support</a>.</p>
|
||||
```
|
||||
|
||||
# ? I don't see the Windows 8.1 or Windows 10 edition I'm looking for. Where else should I check?
|
||||
```markdown
|
||||
<p>If you are looking for MSDN products or Enterprise editions visit the <a target="_blank" href="https://msdn.microsoft.com/en-us/default.aspx">MSDN Portal</a> or the <a target="_blank" href="https://www.microsoft.com/licensing/servicecenter/default.aspx">Volume Licensing Service Center</a>.</p>
|
||||
```
|
||||
|
||||
# ? I've created media using the media creation tool, now what do I do?
|
||||
```markdown
|
||||
<p>After you've created the installation media, do one of the following to install Windows:</p><ul><li>Connect the USB flash drive to the PC where you want to install Windows, browse to it in File Explorer, and then open the setup.exe file. Follow the instructions in setup.</li><li>If your PC is set up to boot from a USB drive, connect the USB flash drive to the PC where you want to install Windows, restart (reboot) it, and then follow the instructions in setup.</li><li>If your PC is set up to boot from the DVD drive, insert the DVD into your PC, restart (reboot) it, and then follow the instructions in setup.</li></ul>
|
||||
```
|
||||
|
||||
# ? I've downloaded an ISO file, now what?
|
||||
```markdown
|
||||
<p>You can use the ISO file to create bootable media for installation or recovery. You can also install Windows on your current device by opening the ISO file, selecting the Setup and following the instructions.</p><p>To create bootable media such as a bootable USB drive or DVD, you will need an ISO burning or mounting software. We recommend always using a blank USB or blank DVD because contents may be deleted when creating a bootable image.</p><p>If you are creating media from a Windows 8.1 machine, you can also right click the ISO file and select either Mount to mount the image to the current device or a USB drive or select Burn disc image to burn a DVD (this requires a DVD burner and a blank DVD).</p><p>If you are creating a DVD from a Windows 7 machine, you can right click the ISO file and select Burn disc image or Open with, then Windows Disc Image Burner to burn a DVD (this requires a DVD burner and a blank DVD). This DVD can be used to install media and is bootable.</p><p>If you are creating media from a Windows 7 machine, you may need to use a separate burning or mounting software such as the <a target="_blank" href="https://wudt.codeplex.com/">Windows 7 USB/DVD Download Tool</a>. Before using this tool, be sure to read the <a target="_blank" href="https://www.microsoftstore.com/store/msusa/html/pbPage.Help_Win7_usbdvd_dwnTool">Information and Instructions</a>. You can also right click the ISO file and select Burn disc image to burn a DVD (this requires a DVD burner and a blank DVD).</p><p>You can also visit <a target="_blank" href="https://answers.microsoft.com/en-us">Microsoft Community</a> to research other options.</p><p>To use the bootable media, make sure the device you will be installing on is set to boot from a USB or DVD. Then connect the USB or insert the DVD, restart (reboot) the device and then follow the instructions in setup.</p>
|
||||
```
|
||||
|
||||
# ? Is the media bootable?
|
||||
```markdown
|
||||
<p>The media from this site can be used to create bootable USB drives and DVDs which will allow you to access recovery tools.</p>
|
||||
```
|
||||
|
||||
# ? Windows came pre-installed on my device, can I use media from this site to download and install?
|
||||
```markdown
|
||||
<p>Before using operating system copies from this site for install, re-install or recovery on devices with pre-installed operating systems, see your device manufacturer or reseller for the customized drivers and applications specific to your machine. Using operating systems copied from this site for install, re-install or recovery may void your support agreement with your manufacturer or reseller. Any drivers or programs that were installed by the device manufacturer or reseller may be removed during installation.</p>
|
||||
```
|
||||
|
||||
# ? Will media from this site install correctly on a device that came with an operating system pre-installed?
|
||||
```markdown
|
||||
<p>Windows 8.x and Windows 10 Pre-installed: You can download and use media from this site on your device and it will install correctly. In most cases you will not be asked for a product key during installation, the product key is included on the motherboard. However, after installation, the device will NOT contain the specialized drivers and software that came pre-installed from your manufacturer. *If you are prompted for a product key during installation, refer to How do I find my Windows product key? question above.</p><p>Windows 7 Pre-installed: We strongly recommend you contact the device manufacturer for support.</p>
|
||||
```
|
||||
|
||||
# ? I have a non-Windows device (like an Apple) and I want to run Windows on it. Where do I get the media?
|
||||
```markdown
|
||||
<p>You can download an ISO file copy of Windows to use with a non-Windows device. Follow these links for <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/windows10">Windows 10</a>, <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/windows8">Windows 8.1</a> or <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/windows7">Windows 7 ISO</a> download options best for non-Windows devices.</p>
|
||||
```
|
||||
|
||||
# ? I bought Windows 7 through a website. After talking to the merchant, I was told I had a "system builder" product key. Why doesn't that work?
|
||||
```markdown
|
||||
<p>Not all “system builder” products for Windows 7 are currently available on the site. We are working to add more, but for now, we suggest you <a target="_blank" href="https://support.microsoft.com/en-us/contactus">contact Support</a>.</p>
|
||||
```
|
||||
|
||||
# ? I'm running a Mac and get an error message when I click Download Tool Now. What's wrong?
|
||||
```markdown
|
||||
<p>The media creation tool (Download Tool Now button) will not run on a Mac device. You can still download an ISO file with one of these options: <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/windows8">Windows 8.1</a> or <a target="_blank" class="mscom-link" href="https://www.microsoft.com/en-us/software-download/windows10">Windows 10</a>. This will give you a web-based download option that should run on most browsers.</p>
|
||||
```
|
||||
|
|
@ -1,127 +0,0 @@
|
|||
# Greeting
|
||||
- Hi
|
||||
- Hello
|
||||
- Good morning
|
||||
- Good evening
|
||||
|
||||
# Help
|
||||
- help
|
||||
- I need help
|
||||
- please help
|
||||
|
||||
# AskForUserName
|
||||
- {userName=vishwac}
|
||||
- I'm {userName=vishwac}
|
||||
- call me {userName=vishwac}
|
||||
- my name is {userName=vishwac}
|
||||
- {userName=vishwac} is my name
|
||||
- you can call me {userName=vishwac}
|
||||
|
||||
> # Entity definitions
|
||||
$userName:simple
|
||||
|
||||
> PREBUILT entities are global. LUIS will always provide results for these when ever a prebuilt entity is found in any utterance.
|
||||
|
||||
$PREBUILT:datetimeV2
|
||||
|
||||
# CreateAlarm
|
||||
- create an alarm
|
||||
- create an alarm for 7AM
|
||||
- set an alarm for 7AM next thursday
|
||||
|
||||
> add these as patterns
|
||||
|
||||
# DeleteAlarm
|
||||
> this utterance will be added as a pattern since there is no labelled value for the alarmTime entity
|
||||
|
||||
- delete the {alarmTime} alarm
|
||||
- remove the {alarmTime} alarm
|
||||
|
||||
> Since there is a list entity definition, any synonyms in this list will get picked up as list entity type and should not be labelled
|
||||
# CommunicationPreference
|
||||
- set phone call as my communication preference
|
||||
- I prefer to receive text message
|
||||
|
||||
> List entity definition
|
||||
|
||||
$commPreference:call=
|
||||
- phone call
|
||||
- give me a ring
|
||||
- ring
|
||||
- call
|
||||
- cell phone
|
||||
- phone
|
||||
|
||||
# Help
|
||||
- can you help
|
||||
|
||||
> you can break up list entity definitions into multiple chunks, interleaved within a .lu file or even spread across .lu files.
|
||||
|
||||
$commPreference:text=
|
||||
- message
|
||||
- text
|
||||
- sms
|
||||
- text message
|
||||
|
||||
$commPreference:fax=
|
||||
- fax
|
||||
- fascimile
|
||||
|
||||
> You can have references to external .lu files
|
||||
|
||||
[None intent definition](./none.lu)
|
||||
|
||||
[Buy chocolate definition](./buyChocolate.lu)
|
||||
|
||||
> # QnA Definitions
|
||||
> This is a QnA definition. Follows # ? Question: \<list of questions\> \```markdown \<Answer> ``` format
|
||||
|
||||
> You can add URLs for QnA maker to ingest using the #URL reference scheme
|
||||
### ? How do I change the default message
|
||||
```markdown
|
||||
You can change the default message if you use the QnAMakerDialog.
|
||||
See [this link](https://docs.botframework.com/en-us/azure-bot-service/templates/qnamaker/#navtitle) for details.
|
||||
```
|
||||
|
||||
### ? How do I programmatically update my KB?
|
||||
```markdown
|
||||
You can use our REST apis to manage your KB.
|
||||
\#1. See here for details: https://westus.dev.cognitive.microsoft.com/docs/services/58994a073d9e04097c7ba6fe/operations/58994a073d9e041ad42d9baa
|
||||
```
|
||||
|
||||
> You can add URLs for QnA maker to ingest using the #URL reference scheme
|
||||
|
||||
[QnA URL - faqs](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
|
||||
|
||||
|
||||
> You can define multilple questions for single answer as well
|
||||
### ? Who is your ceo?
|
||||
- get me your ceo info
|
||||
```markdown
|
||||
Vishwac
|
||||
```
|
||||
|
||||
> You can define filters for QnA using the \**Filters:** \<list of name=value pairs\> format
|
||||
### ? Where can I get coffee?
|
||||
- I need coffee
|
||||
|
||||
**Filters:**
|
||||
- location = seattle
|
||||
```markdown
|
||||
You can get coffee in our Seattle store at 1 pike place, Seattle, WA
|
||||
```
|
||||
|
||||
### ? Where can I get coffee?
|
||||
- I need coffee
|
||||
|
||||
**Filters:**
|
||||
- location = portland
|
||||
```markdown
|
||||
You can get coffee in our Portland store at 52 marine drive, Portland, OR
|
||||
```
|
||||
|
||||
> FAQ URLs for QnA maker to ingest.
|
||||
|
||||
[QnA maker reference](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
|
||||
|
||||
[QnA reference](./qna7.lu)
|
||||
|
|
@ -1,52 +0,0 @@
|
|||
> # QnA Definitions
|
||||
> This is a QnA definition. Follows # ? Question: \<list of questions\> \```markdown \<Answer> ``` format
|
||||
|
||||
> You can add URLs for QnA maker to ingest using the #URL reference scheme
|
||||
### ? How do I change the default message
|
||||
```markdown
|
||||
You can change the default message if you use the QnAMakerDialog.
|
||||
See [this link](https://docs.botframework.com/en-us/azure-bot-service/templates/qnamaker/#navtitle) for details.
|
||||
```
|
||||
|
||||
### ? How do I programmatically update my KB?
|
||||
```markdown
|
||||
You can use our REST apis to manage your KB.
|
||||
\#1. See here for details: https://westus.dev.cognitive.microsoft.com/docs/services/58994a073d9e04097c7ba6fe/operations/58994a073d9e041ad42d9baa
|
||||
```
|
||||
|
||||
> You can add URLs for QnA maker to ingest using the #URL reference scheme
|
||||
|
||||
[QnA URL - faqs](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
|
||||
|
||||
|
||||
> You can define multilple questions for single answer as well
|
||||
### ? Who is your ceo?
|
||||
- get me your ceo info
|
||||
```markdown
|
||||
Vishwac
|
||||
```
|
||||
|
||||
> You can define filters for QnA using the \**Filters:** \<list of name=value pairs\> format
|
||||
### ? Where can I get coffee?
|
||||
- I need coffee
|
||||
|
||||
**Filters:**
|
||||
- location = seattle
|
||||
```markdown
|
||||
You can get coffee in our Seattle store at 1 pike place, Seattle, WA
|
||||
```
|
||||
|
||||
### ? Where can I get coffee?
|
||||
- I need coffee
|
||||
|
||||
**Filters:**
|
||||
- location = portland
|
||||
```markdown
|
||||
You can get coffee in our Portland store at 52 marine drive, Portland, OR
|
||||
```
|
||||
|
||||
> FAQ URLs for QnA maker to ingest.
|
||||
|
||||
[QnA maker reference](https://docs.microsoft.com/en-in/azure/cognitive-services/qnamaker/faqs)
|
||||
|
||||
[QnA reference](./qna7.lu)
|
||||
|
|
@ -1,19 +0,0 @@
|
|||
> You can describe alternations list for QnA Maker using .lu files
|
||||
|
||||
> https://docs.microsoft.com/en-us/azure/cognitive-services/qnamaker/concepts/best-practices
|
||||
|
||||
> https://westus.dev.cognitive.microsoft.com/docs/services/5a93fcf85b4ccd136866eb37/operations/5ac266295b4ccd1554da75fd
|
||||
|
||||
> You can use qnamaker replace alterations
|
||||
|
||||
$botframework : qna-alterations=
|
||||
- bot framework
|
||||
- Azure bot service
|
||||
|
||||
$qnamaker : qna-alterations=
|
||||
- qna maker
|
||||
- QnA Maker
|
||||
- question and answer pairs
|
||||
|
||||
$webchat : qna-alterations=
|
||||
- web chat
|
||||
|
|
@ -1,711 +0,0 @@
|
|||
> # Intent definitions
|
||||
|
||||
## AddFlag
|
||||
- add a flag
|
||||
- add a flag please
|
||||
- add a flag to the {OrderReference=last} email
|
||||
- add a flag to this email
|
||||
- add flag
|
||||
- add flag on it
|
||||
- add flag to it
|
||||
- add flag to the email {SenderName=john} just sent to me
|
||||
- add {Category=flag} to this email
|
||||
- add flag to this message
|
||||
- flag
|
||||
- flag it
|
||||
- flag on
|
||||
- flag the current email
|
||||
- flag the email
|
||||
- flag the email from {SenderName=davis}
|
||||
- flag this email
|
||||
- flag this email as {Category=important} for me
|
||||
- i want to add a flag
|
||||
- i want to add a flag on this email
|
||||
- make it flagged
|
||||
- mark as flag
|
||||
- mark the email {Category=flagged}
|
||||
- put a flag
|
||||
- put a flag on the new email
|
||||
- the email from {SenderName=thomas} should be flagged
|
||||
- the email to {ContactName=ruth} needs to be flagged
|
||||
- this email need to be flagged
|
||||
- this email needs to be flagged
|
||||
- this email should be flagged
|
||||
- turn flag on
|
||||
|
||||
|
||||
## AddMore
|
||||
- add a {Attachment=file} to the email
|
||||
- add a {Attachment=picture}
|
||||
- add a subject
|
||||
- add another line to the message
|
||||
- add {Message=did you enjoy the entire program}
|
||||
- add {Attachment=file} to email
|
||||
- add more
|
||||
- add more and change the message
|
||||
- add more details to it
|
||||
- add more {Message=don't forget to bring beer}
|
||||
- add more message
|
||||
- add more please
|
||||
- add more text
|
||||
- add more text please
|
||||
- add more to email
|
||||
- add more to email body
|
||||
- add more to it
|
||||
- add more to message
|
||||
- add more to {ContactName=roy} 's email
|
||||
- add more to text
|
||||
- add more to the email
|
||||
- add more to the {OrderReference=last} email
|
||||
- add more to the message
|
||||
- add {Attachment=photo}
|
||||
- add some more
|
||||
- add something
|
||||
- add to body of email
|
||||
- add, by the way, what's the plan of next step
|
||||
- add: {Message=call me tonight after work}
|
||||
- append an {Attachment=attachment} to this email
|
||||
- attach {Attachment=file}
|
||||
- can i add more to the email
|
||||
- can i add more to the message
|
||||
- edit email so i can type an additional message
|
||||
- i am not done yet. i need to add some more details
|
||||
- i forgot to add an important part to that email to {ContactName=james} . please set it up to edit
|
||||
- i need to add additional lines
|
||||
- i need to add further contents
|
||||
- i need to add more message
|
||||
- i need to add more text
|
||||
- i need to add more to the email
|
||||
- i need to add more to the email message i am sending to {ContactName=vincent}
|
||||
- i need to add something else to my email to {ContactName=cheryl}
|
||||
- i need to add something else to that email to {ContactName=donna} before it is sent
|
||||
- i want to add more the email
|
||||
- i wish to add more to the message
|
||||
- i would like to add more to the email
|
||||
- i would like to add more to the email message
|
||||
- i would like to open a new line
|
||||
- i'd like to add a bit more to the email.
|
||||
- i'd like to add a bit more to the message
|
||||
- i'd like to add more to the email
|
||||
- insert more lines for me please
|
||||
- insert more text in my email
|
||||
- is it ok if i add more to the email?
|
||||
- it isn't complete, need more contents
|
||||
- more text
|
||||
- need to add information to the {OrderReference=previous} email
|
||||
- ok, i need to add a few things to that
|
||||
- please add {Message=it was terrible}
|
||||
- please add more
|
||||
- please add, {Message=please let me know what i can bring. i'd be happy to make a side dish or dessert}
|
||||
- put some additional lines to this message
|
||||
- wait, i need to write more
|
||||
- write more
|
||||
|
||||
|
||||
## CancelMessages
|
||||
- abort deletion
|
||||
- can you cancel it
|
||||
- cancel email
|
||||
- cancel email to {ContactName=natalie}
|
||||
- cancel message
|
||||
- cancel my email to {ContactName=jane}
|
||||
- cancel searching the messages
|
||||
- cancel the email
|
||||
- cancel the email sent to {ContactName=alex}
|
||||
- cancel the email to my {RelationshipName=sister}
|
||||
- cancel the mail
|
||||
- cancel the message
|
||||
- cancel this email
|
||||
- cancel this message
|
||||
- cancel this sending process
|
||||
- don ' t read
|
||||
- don ' t read it
|
||||
- don 't send the email
|
||||
- don't email
|
||||
- don't email to her
|
||||
- don't read the email
|
||||
- don't read the message
|
||||
- don't send
|
||||
- don't send it
|
||||
- don't send out
|
||||
- don't send that email
|
||||
- don't send this email
|
||||
- don't show me
|
||||
- exit
|
||||
- forget about the email
|
||||
- i want you to cancel the email
|
||||
- neither of them
|
||||
- never mind cancel the mail
|
||||
- never mind cancel the message
|
||||
- never mind, forget about the mail
|
||||
- nevermind cancel
|
||||
- no cancel it, i don't want to send the mail
|
||||
- no don't send
|
||||
- no don't send it
|
||||
- no just cancel the email
|
||||
- no, i don't want to send this message
|
||||
- no, no, cancel the reading
|
||||
- okay cancel sending the mail
|
||||
- quit the sending
|
||||
- stop message
|
||||
- stop reading
|
||||
- ^cancel [sending] [(my|the)] (email|mail) to {ContactName}
|
||||
|
||||
|
||||
## CheckMessages
|
||||
- any {Category=new} email
|
||||
- any {Category=new} email available
|
||||
- any {Category=new} email {Time=now}
|
||||
- any {Category=new} message {Time=now}
|
||||
- check email
|
||||
- check email please
|
||||
- check my email please
|
||||
- check my emails
|
||||
- check my {Line=gmail}
|
||||
- check my inbox
|
||||
- check my mail box
|
||||
- check my message
|
||||
- check {Line=outlook} please
|
||||
- check up email
|
||||
- check up messages
|
||||
- could you please check my emails
|
||||
- could you please check my inbox
|
||||
- could you please check my messages
|
||||
- do i get new email
|
||||
- do i have any {Category=new} mail
|
||||
- do i have {Category=new} email
|
||||
- do i have {Category=new} email {Time=now}
|
||||
- do i have {Category=new} message
|
||||
- do i receive {Category=new} email
|
||||
- do i receive {Category=new} mail in {Line=outlook}?
|
||||
- do i receive {Category=new} message
|
||||
- does anyone send email to me just then
|
||||
- does anyone send message to me {Time=just then}
|
||||
- does my {Line=outlook} have {Category=new} email
|
||||
- i want to check my emails
|
||||
- i want to check my inbox
|
||||
- i'd like to check my inbox
|
||||
- is there new email
|
||||
- please check my emails
|
||||
- please check my inbox
|
||||
- please check my {Line=outlook}
|
||||
- show {OrderReference=latest} emails
|
||||
- show my emails
|
||||
- show my {Category=unread} mails
|
||||
- show the {Category=important} emails in my inbox
|
||||
- whether i get {Category=new} email
|
||||
- whether i get {Category=new} message
|
||||
- whether i have {Category=new} email
|
||||
- whether i have {Category=new} message
|
||||
- whether i receive new email
|
||||
- show [(my|the)] [(unread|important|{Category})] (email|mail|emails)^
|
||||
|
||||
|
||||
## ConfirmMessages
|
||||
- "okay, send it"
|
||||
- "sure, go ahead"
|
||||
- "yes, you can"
|
||||
- alright, just send the message
|
||||
- correct, please send it.
|
||||
- i confirm that i want to send this email
|
||||
- just do it
|
||||
- no problem, go ahead send the mail
|
||||
- of course, just delete the mail
|
||||
- ok send the mail to {ContactName=may}
|
||||
- ok, good to me, send it please
|
||||
- ok, good, just send it
|
||||
- okay
|
||||
- okay send it
|
||||
- okay, send it now
|
||||
- perfect thank you
|
||||
- right, send it please
|
||||
- yeah right, send to {ContactName=alex}
|
||||
- yes it's right
|
||||
- yes that's right
|
||||
- yes, send it
|
||||
|
||||
|
||||
## Delete
|
||||
- can you help me delete it
|
||||
- clear my inbox
|
||||
- delete all emails from {SenderName=tom}
|
||||
- delete all emails received {Time=tonight}
|
||||
- delete the email from my {Line=hotmail} account
|
||||
- delete the email sent from {SenderName=mary jane}
|
||||
- delete the {PositionReference=first} email for me
|
||||
- delete the {OrderReference=last} one
|
||||
- delete the {OrderReference=previous} 4 emails
|
||||
- delete the {Category=red} ones
|
||||
- delete the second mail
|
||||
- delete the {PositionReference=second} one
|
||||
- delete the {Category=unread} emails
|
||||
- delete this email
|
||||
- delete this message permanently
|
||||
- delete what i just wrote
|
||||
- empty the email inbox
|
||||
- put it in the recycle bin
|
||||
- put the email in the recycle bin
|
||||
- put the email to trash bin
|
||||
- put the emails from this file folder to trash bin
|
||||
- remove emails that are duplicate
|
||||
- remove emails with {Category=red} flags
|
||||
- remove it from my inbox
|
||||
- remove the email from {SenderName=mary}
|
||||
- remove the emails received {Date=yesterday}
|
||||
- ^(delete|remove) [the] [{OrderReference}] (email|emails|mails) [from {SenderName}]
|
||||
|
||||
|
||||
## Forward
|
||||
- could you forward this message to {ContactName=ronald} and {ContactName=roy}
|
||||
- could you please forward this email to my {RelationshipName=sister}
|
||||
- forward all {Attachment=files} from {SenderName=sally} to {ContactName=austin}
|
||||
- forward by saying {Message=if you interest} to {ContactName=rebecca}
|
||||
- forward email
|
||||
- forward email to {RelationshipName=girlfriend}
|
||||
- forward emails to {ContactName=gabriel}
|
||||
- forward message to {RelationshipName=girlfriend}
|
||||
- forward the email from {SenderName=john smith} to {ContactName=michelle} by saying {Message=fyi}
|
||||
- forward the email from {SenderName=melissa} to {ContactName=peter}
|
||||
- forward the email to {RelationshipName=dad}
|
||||
- forward the {OrderReference=last} email to {ContactName=susan}
|
||||
- forward this email
|
||||
- forward this email to {ContactName=eugene} by typing {Message=what do you think}
|
||||
- forward this email to {ContactName=gary brown} please
|
||||
- forward this email to {ContactName=joseph}
|
||||
- forward this email to partone dot parttwo at gmail dot com
|
||||
- forward this email to {ContactName=patricia}
|
||||
- forward to {ContactName=alan} {Time=tonight}
|
||||
- forward to {ContactName=brian potter} {Time=tonight}
|
||||
- forward to {ContactName=deborah} with a message saying that {Message=i don't want that}
|
||||
- forward to {ContactName=dorothy} by typing {Message=i agree with it}
|
||||
- forward to {RelationshipName=mom}
|
||||
- forward to my {RelationshipName=boss} and attach the {Attachment=schedule file}
|
||||
- forward to partoneparttwo@gmail.com {Date=next monday}
|
||||
- forward to {ContactName=thomas} please
|
||||
- forward to {RelationshipName=wife} by saying {Message=i love you}
|
||||
- please forward this email to {ContactName=albert} by typing {Message=everything goes fine}
|
||||
- please forward this email to partoneparttwo@163.com
|
||||
- please forward this email to partoneparttwo@outlook.com
|
||||
- please forward this message
|
||||
- please forward to {ContactName=benjamin}
|
||||
|
||||
|
||||
## None
|
||||
- 1
|
||||
- 2
|
||||
- 3
|
||||
- the {PositionReference=first} one
|
||||
- the {PositionReference=second} one
|
||||
- the {PositionReference=third} one
|
||||
|
||||
|
||||
## QueryLastText
|
||||
- can you tell me the {OrderReference=last} email i received
|
||||
- come to the {OrderReference=last}
|
||||
- go to the {OrderReference=last} one
|
||||
- i want to see the {OrderReference=last} email
|
||||
- {OrderReference=last} email
|
||||
- open the {OrderReference=last} email
|
||||
- open the {OrderReference=lastest} email i got
|
||||
- please tell me who emailed me {OrderReference=last}
|
||||
- show me the {OrderReference=lastest} email
|
||||
- show me the {OrderReference=newest} email
|
||||
- show the {OrderReference=last} email
|
||||
- the {OrderReference=last} email
|
||||
- what did {RelationshipName=mom} just say
|
||||
- what {ContactName=eric watson} just said
|
||||
- what {SenderName=harry} {OrderReference=last} email said
|
||||
- what {ContactName=henry} just said
|
||||
- what is the {OrderReference=last} email i received {Date=today}
|
||||
- what is the {OrderReference=lastest} email i received from {FromRelationshipName=dad}
|
||||
- what was the {OrderReference=last} email
|
||||
- what was the {OrderReference=last} email i got from {FromRelationshipName=dad}
|
||||
- what was the {OrderReference=last} email i got from {SenderName=steve edwards}
|
||||
- who email me {Time=just now}
|
||||
- who emailed me
|
||||
- who emailed me just now
|
||||
- who emailed me {OrderReference=last}
|
||||
- who recently emailed me
|
||||
- who sent me the email lastly {Date=yesterday}
|
||||
- who sent me the mail just now
|
||||
- who texted me
|
||||
- who texted me {Time=just now}
|
||||
- whose email just then ?
|
||||
- whose email {Time=now} ?
|
||||
|
||||
|
||||
## ReadAloud
|
||||
- can you read my emails
|
||||
- can you read my {OrderReference=last} email
|
||||
- could you read out the email on {EmailSubject=how to use the new tool}?
|
||||
- please read my {OrderReference=last} email
|
||||
- read aloud my {Category=new} email
|
||||
- read aloud the {EmailSubject=christmas party} email
|
||||
- read {SenderName=darren}'s mail on {EmailSubject=the movie}
|
||||
- read email
|
||||
- read email from {SenderName=dawn}
|
||||
- read email from {SenderName=kat}
|
||||
- read email from {FromRelationshipName=mum}
|
||||
- read email to me
|
||||
- read emails
|
||||
- read emails from {SenderName=clay}
|
||||
- read {PositionReference=first} email in link box
|
||||
- read {PositionReference=first} email in the linked inbox
|
||||
- read {Line=google} mail
|
||||
- read it
|
||||
- read {OrderReference=last} email received
|
||||
- read {OrderReference=last} incoming emails
|
||||
- read {OrderReference=last} mail
|
||||
- read {OrderReference=latest} email
|
||||
- read {SenderName=mary grace white} email
|
||||
- read me {SenderName=dylan}'s email sent on {Date=yesterday}
|
||||
- read me {SenderName=jessica}'s email on {EmailSubject=dress code for the party}
|
||||
- read me my {OrderReference=last} {Line=hotmail} email
|
||||
- read me my {OrderReference=latest} emails
|
||||
- read me the email
|
||||
- read me the email on {EmailSubject=apple}
|
||||
- read me the email on {EmailSubject=thanksgiving day}
|
||||
- read me the email sent on {Date=thanksgiving day}
|
||||
- read me the email titled {EmailSubject=happy new year}
|
||||
- read me the emails from {SenderName=agatha}
|
||||
- read me the {OrderReference=last} email {SenderName=claude} sent
|
||||
- read me the {OrderReference=last} emails of the {Time=five minutes}
|
||||
- read me the {OrderReference=newest} email
|
||||
- read me the {OrderReference=recent} email titled {EmailSubject=abcd} from {SenderName=jessica}
|
||||
- read most {OrderReference=recent} email
|
||||
- read my email from {SenderName=baby}
|
||||
- read my email from {SenderName=hubby}
|
||||
- read my email from {SenderName=tyler swift}
|
||||
- read my email messages
|
||||
- read my email please
|
||||
- read my email to me
|
||||
- read my emails
|
||||
- read my emails from {SenderName=patty}
|
||||
- read my inbox
|
||||
- read my {OrderReference=last} email
|
||||
- read my {OrderReference=last} email out to me
|
||||
- read my {OrderReference=most recent} email
|
||||
- read my {Category=new} email
|
||||
- read my {Category=new} emails
|
||||
- read my notification
|
||||
- read my {Line=outlook} email
|
||||
- read my {OrderReference=recent} email
|
||||
- read my {OrderReference=recent} email message please
|
||||
- read my {OrderReference=recent} email messages
|
||||
- read my {OrderReference=recent} email to me
|
||||
- read my {PositionReference=second} email
|
||||
- read {Category=new} email
|
||||
- read {Category=new} email from {SenderName=david ma}
|
||||
- read {Category=new} message
|
||||
- read out {SenderName=darren}'s mail
|
||||
- read out the email from {SenderName=liu} about {EmailSubject=transfer}
|
||||
- read out {SenderName=xu}'s email about {EmailSubject=apple's news}
|
||||
- read please
|
||||
- read {OrderReference=recent} email
|
||||
- read the email
|
||||
- read the email on {EmailSubject=auto repair}
|
||||
- read the {PositionReference=first} email
|
||||
- read the {PositionReference=first} email in {Line=hotmail}
|
||||
- read the {OrderReference=last} email
|
||||
- read the {OrderReference=last} email message
|
||||
- read the {OrderReference=latest} email from {FromRelationshipName=mom}
|
||||
- read the {OrderReference=latest} email from {SenderName=steve lip}
|
||||
- read the {OrderReference=latest} email i sent
|
||||
- read {Date=todays} mail
|
||||
- read {Date=today}'s mail
|
||||
- read {Category=unread} email
|
||||
- read {Category=unread} message
|
||||
- ^read [(me|out)] [the] (email|emails) from {SenderName} [(about|on) {EmailSubject}]^
|
||||
- ^read [me] [the] (email|mail|emails) (on|about|titled) {EmailSubject}
|
||||
- ^read [me] [the] (email|mail|emails) (on|about|titled) {EmailSubject} [sent] from {SenderName}^
|
||||
- ^read [me] the (email|mail) (titled|about|on) [the] {EmailSubject}^
|
||||
|
||||
|
||||
## Reply
|
||||
- create a response to the email by saying {Message=pls send me the picture again}
|
||||
- email back
|
||||
- email back {Message=i will call you back}
|
||||
- how to reply to an email
|
||||
- make a response with {Message=thank you very much}
|
||||
- reply
|
||||
- reply by email {Message=thank you very much best regards jun}
|
||||
- reply by saying {Message=i love you}
|
||||
- reply by saying {Message=yes}
|
||||
- reply by typing {Message=hello}
|
||||
- reply {Message=required} to an email
|
||||
- reply that {Message=i am busy}
|
||||
- reply to {ContactName=edward}
|
||||
- reply to email {Message=i am busy now}
|
||||
- reply to my {OrderReference=last} email
|
||||
- reply to {ContactName=susan}
|
||||
- reply to the email
|
||||
- reply to the {PositionReference=first} one
|
||||
- reply {Message=we'll see you later}
|
||||
- reply with {Message=hello}
|
||||
- reply {Message=yee ha}
|
||||
- reply {ContactName=yee} {Message=hello}
|
||||
- reply {Message=yes boss.}
|
||||
- respond {Message=i ' m sick i can ' t do it}
|
||||
- respond to {ContactName=lore hound}
|
||||
- respond to {ContactName=nathan}
|
||||
- respond to the email by saying {Message=i am busy today}
|
||||
- return {ContactName=barbara} on {Line=mobile}
|
||||
- return {ContactName=siberian huskies} {Line=mobile}
|
||||
- send email back
|
||||
- send the response with {Message=i've already know}
|
||||
- ^reply [back] with [the] (title|subject) {EmailSubject}
|
||||
- ^reply [back] [the] message [that] {Message}^
|
||||
- ^reply [back] [with] [the] [message] [that] "{Message.Any}"
|
||||
|
||||
|
||||
## SearchMessages
|
||||
- can you search my emails
|
||||
- detect emails from {SenderName=betty}
|
||||
- detect the email containing keyword {SearchTexts=beauty}
|
||||
- detect the email from {SenderName=lisa}
|
||||
- did i get any email from {SenderName=tom}
|
||||
- did i get emails from {SenderName=tom}
|
||||
- did i get the email containing keyword {SearchTexts=lunch}
|
||||
- email sent from {SenderName=lisa}
|
||||
- emails contains {SearchTexts=bank}
|
||||
- enumerate the emails with {SearchTexts=algroithm}
|
||||
- find an email about {EmailSubject=new year's planning}
|
||||
- find an email from abc123@outlook.com
|
||||
- find an email from {SenderName=angela}
|
||||
- find an email from {SenderName=jay} that contains {SearchTexts=halloween}
|
||||
- find an email on the {EmailSubject=dinner reservation}
|
||||
- find email titled {EmailSubject=new design}
|
||||
- find email with title {EmailSubject=production tools}
|
||||
- find emails from {FromRelationshipName=mom}
|
||||
- find emails that contain {SearchTexts=malta}
|
||||
- find emails with {SearchTexts=resume}
|
||||
- find mails titled {EmailSubject=recommended courses}
|
||||
- list the emails contain {SearchTexts=funny picture}
|
||||
- looking for an email with {SearchTexts=hello}
|
||||
- query emails with {SearchTexts=bill}
|
||||
- search an email with subject {EmailSubject=background screening}
|
||||
- search {SearchTexts=bla bla} in my emails
|
||||
- search email contain {SearchTexts=outlook}
|
||||
- search email with key words {SearchTexts=lunch}
|
||||
- search emails about {EmailSubject=boating}
|
||||
- search emails contain {SearchTexts=work items}
|
||||
- search emails contains {SearchTexts=coupons}
|
||||
- search emails from {SenderName=mike}
|
||||
- search {SenderName=jensen}'s emails
|
||||
- search keywords {SearchTexts=keywordone keywordtwo} in my emails
|
||||
- search {SearchTexts=keywordsone keywordstwo} from inbox
|
||||
- search my emails
|
||||
- search text with words {SearchTexts=lunch together}
|
||||
- search the email with keywords {SearchTexts=hello}
|
||||
- search the emails contains {SearchTexts=microsoft}
|
||||
- search the emails contains {SearchTexts=money}
|
||||
- show emails contain words "{SearchTexts=future plan}"
|
||||
- show emails with "{SearchTexts=credit card}"
|
||||
- show me emails from {SenderName=clara chan}
|
||||
- show me emails from {FromRelationshipName=girlfriend}
|
||||
- show me the email about {EmailSubject=spring festival}
|
||||
- show me the email from {SenderName=tom} and filtering with word {SearchTexts=lunch}
|
||||
- show me the email sent from {FromRelationshipName=mom}
|
||||
- tell me the email from {SenderName=lily wong}
|
||||
- tell me the email with subject {EmailSubject=weekly report}
|
||||
- ^(tell|find|show) [me] [(an|the)] email (with the title|with title|titled) {EmailSubject}
|
||||
- ^(tell|find|show) [me] [(an|the)] email (with the title|with title|titled) "{EmailSubject.Any}"
|
||||
- ^(search|find|show) [me] [(an|the)] (email|emails|mail) [(containing|filtering)] [with] [the] [(texts|text|word)] "{SearchTexts.Any}"
|
||||
- ^(show|search|find) [me] [(an|the)] [{Category}] email[s] (on|about|titled) {EmailSubject} [sent] [(by|from) {SenderName}]^
|
||||
|
||||
|
||||
## SendEmail
|
||||
- compose new email about {EmailSubject=spanish homework}
|
||||
- create new mail titled {EmailSubject=urgent meeting information} to {ContactName=jonathan}
|
||||
- email her the message "fine, ok"
|
||||
- email my {RelationshipName=brother}
|
||||
- email my {Attachment=presentation}
|
||||
- email the {Attachment=file} to {ContactName=henry mathew}
|
||||
- email to {ContactName=amy cooper} about {Message=haha saying hello}
|
||||
- email to {ContactName=cynthia} and {ContactName=mike}, {Message=that dinner {OrderReference=last} week was splendid}.
|
||||
- email to {ContactName=harry potter} and {ContactName=hermione granger}
|
||||
- email to {ContactName=lawrence} about {EmailSubject=opening issue}
|
||||
- email to {ContactName=mike waters} : {Message=mike, that dinner last week was splendid.}
|
||||
- email to partoneparttwo@gmail.com
|
||||
- email to {ContactName=tom white} about {Message=that flower saying beautiful}
|
||||
- i need to send an email about the {EmailSubject=words to a song}
|
||||
- make a new email about {EmailSubject=weather forecast}
|
||||
- mark email for {Category=follow up} and send to {ContactName=arthur}
|
||||
- new email about {EmailSubject=really good talk} to {ContactName=michelle}
|
||||
- new email about {EmailSubject=writing documents}
|
||||
- new email to {ContactName=kimberly} about {EmailSubject=wingman}
|
||||
- send a email to {ContactName=leehom wong} about the {EmailSubject=piano concert} saying {Message=it's wonderful}
|
||||
- send a mail to {ContactName=daniel}
|
||||
- send a new email about {EmailSubject=facebook}
|
||||
- send a new email about the {EmailSubject=hockey tournament} to {ContactName=marie jane}, {ContactName=joseph} , and {ContactName=john}
|
||||
- send a new email about the {EmailSubject=problem solving} to {ContactName=andrea}, {ContactName=angela}, and {ContactName=ron}
|
||||
- send a new email to {ContactName=larry} with a {Attachment=file} attached
|
||||
- send a new email to {ContactName=nicholas} and {ContactName=jesse} about {EmailSubject=coupons}
|
||||
- send a new email to partonepartwopartthree@yahoo.com
|
||||
- send a new {Category=high importance} email to {ContactName=jordan}
|
||||
- send a {Category=read receipt} email to {ContactName=samuel}
|
||||
- send {ContactName=alexander} a {Category=red bang} email
|
||||
- send an email
|
||||
- send an email about {EmailSubject=swim team practice}
|
||||
- send an email about {EmailSubject=test status} to {ContactName=mark}
|
||||
- send an email about the {EmailSubject=window that is broken}
|
||||
- send an email for me
|
||||
- send an email marked {Category=follow up} to {ContactName=jerry}
|
||||
- send an email marked for {Category=follow up} to {ContactName=christian}
|
||||
- send an email marked with a {Category=bang} to {ContactName=amy}
|
||||
- send an email to {ContactName=a.j.ron} marked as {Category=important}
|
||||
- send an email to {ContactName=christopher carpenter} about the {EmailSubject=hiking trip}
|
||||
- send an email to {ContactName=harold} and {ContactName=bob kappus} about {Message=team lunch saying same team lunch this tuesday}
|
||||
- send an email to {ContactName=harry potter}
|
||||
- send an email to {ContactName=jacqueline} and {ContactName=tianyu} about the {EmailSubject=test result}
|
||||
- send an email to {ContactName=jimmy klein} saying {Message=this is the message about weekend plans}
|
||||
- send an email to {ContactName=larry} , {ContactName=joseph} and {ContactName=billy larkson}
|
||||
- send an email to {ContactName=lily roth} and abc123@microsoft.com
|
||||
- send an email to {ContactName=lu} , {ContactName=yue} and {ContactName=qiong} about {EmailSubject=funding}
|
||||
- send an email to {RelationshipName=mom}
|
||||
- send an email to my {RelationshipName=brother}
|
||||
- send an email to {ContactName=nathan} with a {Category=red bang}
|
||||
- send an email to partone@gmail.com
|
||||
- send an email to partone_parttwo@microsoft.com
|
||||
- send an email to {ContactName=sean} about {EmailSubject=weekend plans}
|
||||
- send an email to {ContactName=zachary} about {EmailSubject=we can plan things let's go hiking}
|
||||
- send an email {Date=today}
|
||||
- send an email with {Category=read receipt} to {ContactName=peter}
|
||||
- send an {Category=important} email to {ContactName=olivia}
|
||||
- send an {Category=urgent} email
|
||||
- send an {Category=urgent} email from my {Line=work account} to {ContactName=christian}
|
||||
- send an {Category=urgent} email from my {Line=work} email to {ContactName=jack}
|
||||
- send and email about {EmailSubject=swim team practice}
|
||||
- send {ContactName=angela} an email marked as {Category=high priority}
|
||||
- send {ContactName=billy} an email with a {Category=red bang}
|
||||
- send email about {EmailSubject=homework plan} to {ContactName=raymond} and {ContactName=philip}
|
||||
- send email marked {Category=priority} to {ContactName=yun-sim} and {ContactName=yi}
|
||||
- send email to {ContactName=a} and {ContactName=tian}
|
||||
- send email to {ContactName=hannah} saying {Message=test}
|
||||
- send email to {ContactName=heather} about {EmailSubject=car}
|
||||
- send email to {ContactName=jiayi} {Date=today}
|
||||
- send email to {ContactName=kai xu}, {ContactName=mingming} and my {RelationshipName=mother}
|
||||
- send email to {ContactName=louis} and mark it {Category=important}
|
||||
- send email to partone.parttwo@outlook.com
|
||||
- send {Category=important} email to {ContactName=evelyn} and {ContactName=gary}
|
||||
- send {ContactName=jacqueline} an email with {Category=low priority}
|
||||
- send {Attachment=large files} through email
|
||||
- send {ContactName=lori} a new {Category=flagged} email
|
||||
- send mail to {ContactName=dorothy}
|
||||
- send my {Attachment=housekeeping doc} to {ContactName=jeffrey}
|
||||
- send my {Attachment=payment visio diagram} to {ContactName=ronald}
|
||||
- send new email to {ContactName=christian} and mark it {Category=high importance}
|
||||
- send the email
|
||||
- send the email {Time=now}
|
||||
- send this {Attachment=document} to an email
|
||||
- send {ContactName=thomas} an email
|
||||
- set an email {Date=today}
|
||||
- start a new email about {EmailSubject=marriage counselor appointments}
|
||||
- start a new email from {SenderName=tracy} saying {Message=here is my resume}
|
||||
- start a new email saying {Message=lets go to the park}
|
||||
- start a new email to {ContactName=aaron} about {EmailSubject=sleeping over tonight}
|
||||
- start an email to {ContactName=jason} about {EmailSubject=speaking up}
|
||||
- start new email about {EmailSubject=taco blog} to {ContactName=nicole} and {ContactName=emily}
|
||||
- start new email to {RelationshipName=friends} about the {EmailSubject=club}
|
||||
- start up a new email to {ContactName=michelle} about {EmailSubject=watching baseball}
|
||||
- the new email is {Category=high priority} that is being sent to {ContactName=jacob}
|
||||
- will you send a marked {Category=non urgent} email to {ContactName=james}
|
||||
- write an email about the {EmailSubject=fundraiser}
|
||||
- write an email which title is {EmailSubject=hello} and context is {Message=let's have meeting together}
|
||||
- write an {Category=urgent} email to {ContactName=bobby}
|
||||
- write email
|
||||
- write email to {RelationshipName=mom} subject is {EmailSubject=babysit}
|
||||
- ^(write|send|start) [(a|an|the)] [new] email to {ContactName}^
|
||||
- ^[(send|write)] [(the|a)] [new] [(email|mail)] [to {ContactName}] [with message "{Message.Any}"]
|
||||
- ^(write|send|start) [(a|an|the)] [new] email to {ContactName} (about|on|with) [the subject] [that] {EmailSubject}^
|
||||
|
||||
|
||||
## ShowNext
|
||||
- are there any {Category=unread} messages? show {OrderReference=next}
|
||||
- go forward to {OrderReference=next} mails
|
||||
- go on, show me more mails
|
||||
- go to {OrderReference=next} mail
|
||||
- go to the {OrderReference=next} page
|
||||
- move forward
|
||||
- move on {OrderReference=next} mail by jason
|
||||
- move on to {OrderReference=next} mails
|
||||
- {OrderReference=next} email
|
||||
- {OrderReference=next} {Category=unread} email
|
||||
- {OrderReference=next} {Category=unread} one
|
||||
- show me {OrderReference=next} from {SenderName=mary}
|
||||
- show me the {OrderReference=next}
|
||||
- show me the {OrderReference=next} five mails
|
||||
- show {OrderReference=next} email
|
||||
- show {OrderReference=next} {Category=unread}
|
||||
- show the {OrderReference=next} email from my {FromRelationshipName=boss}
|
||||
- show the {OrderReference=next} emails by wong
|
||||
- show the {OrderReference=next} messages
|
||||
- the {OrderReference=next} email
|
||||
- the {OrderReference=next} {Category=important} message
|
||||
- ^(show|give|tell) [me] [the] next (email|message|mail)^
|
||||
|
||||
|
||||
## ShowPrevious
|
||||
- back to the {OrderReference=last} one from {SenderName=apple}
|
||||
- bring the {OrderReference=previous} one, i want to read it again
|
||||
- go to {OrderReference=previous} mails
|
||||
- move back to {OrderReference=last} mails
|
||||
- {OrderReference=previous} email
|
||||
- {OrderReference=previous} one please
|
||||
- show me {OrderReference=previous} email from {SenderName=jack}
|
||||
- show me the {OrderReference=last} three mails
|
||||
- show me the one {OrderReference=before}
|
||||
- show me the {OrderReference=previous} email
|
||||
- show {OrderReference=previous} in {Category=red} category
|
||||
- show {OrderReference=previous} one in inbox
|
||||
- show the {OrderReference=previous} email from my {FromRelationshipName=mentor}
|
||||
- show the {OrderReference=previous} one
|
||||
- the {OrderReference=previous} email
|
||||
- what is the {OrderReference=previous} email
|
||||
- ^(show|give|tell) [me] [the] previous (email|message|mail)^
|
||||
|
||||
|
||||
> # Entity definitions
|
||||
|
||||
$Attachment:simple
|
||||
|
||||
$Category:simple
|
||||
|
||||
$ContactName:simple
|
||||
|
||||
$Date:simple
|
||||
|
||||
$EmailSubject:simple
|
||||
|
||||
$FromRelationshipName:simple
|
||||
|
||||
$Line:simple
|
||||
|
||||
$Message:simple
|
||||
|
||||
$OrderReference:simple
|
||||
|
||||
$PositionReference:simple
|
||||
|
||||
$RelationshipName:simple
|
||||
|
||||
$SearchTexts:simple
|
||||
|
||||
$SenderName:simple
|
||||
|
||||
$Time:simple
|
||||
|
||||
|
||||
> # PREBUILT Entity definitions
|
||||
|
||||
$PREBUILT:email
|
||||
|
||||
$PREBUILT:ordinal
|
||||
|
||||
|
||||
> # Phrase list definitions
|
||||
|
||||
|
||||
> # List entities
|
||||
|
||||
> # RegEx entities
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { ColumnarData } from "./ColumnarData";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export function exampleFunctionData(): ColumnarData {
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(`process.argv=${process.argv}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppColumnarData",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "a LU file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-o", "--outputFilenamePrefix"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "output file name prefix",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--linesToSkip"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string = args.filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- let outputFilenamePrefix: string = args.outputFilenamePrefix;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- if (outputFilenamePrefix == null) {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix = filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- }
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const linesToSkip: number = +args.linesToSkip;
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `outputFilenamePrefix=${outputFilenamePrefix}`);
|
||||
const columnarContent: string = Utility.loadFile(filename);
|
||||
const columnarData: ColumnarData = ColumnarData.createColumnarData(
|
||||
columnarContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
true);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- columnarData.dumpLuLuisJsonStructureInLuFormat(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix + ".lu");
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- columnarData.dumpLuLuisJsonStructure(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix + ".luis", undefined, 4);
|
||||
return columnarData;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionData();
|
||||
}
|
||||
|
|
@ -1,90 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { EntityAnnotatedCorpusData } from "./EntityAnnotatedCorpusData";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export function exampleFunctionData(): EntityAnnotatedCorpusData {
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(`process.argv=${process.argv}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppEntityAnnotatedCorpusData",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "an Entity-Annotated-Corpus file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-o", "--outputFilenamePrefix"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "output file name prefix",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
parser.addArgument(
|
||||
["-ls", "--linesToSkip"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string = args.filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- let outputFilenamePrefix: string = args.outputFilenamePrefix;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- if (outputFilenamePrefix == null) {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix = filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- }
|
||||
const linesToSkip: number = +args.linesToSkip;
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `outputFilenamePrefix=${outputFilenamePrefix}`);
|
||||
const entityAnnotatedCorpusContent: string = Utility.loadFile(filename);
|
||||
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
|
||||
EntityAnnotatedCorpusData.createEntityAnnotatedCorpusData(
|
||||
entityAnnotatedCorpusContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
linesToSkip,
|
||||
true);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- entityAnnotatedCorpusData.dumpLuLuisJsonStructureInLuFormat(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix + ".lu");
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- entityAnnotatedCorpusData.dumpLuLuisJsonStructure(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- outputFilenamePrefix + ".luis", undefined, 4);
|
||||
return entityAnnotatedCorpusData;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionData();
|
||||
}
|
||||
|
|
@ -1,81 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { LuData } from "./LuData";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export async function exampleFunctionData(): Promise<string[]> {
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppLuData",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "a LU file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputFilenamePrefix"],
|
||||
{
|
||||
help: "output file name prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string = args.filename;
|
||||
let outputFilenamePrefix: string = args.outputFilenamePrefix;
|
||||
if (outputFilenamePrefix == null) {
|
||||
outputFilenamePrefix = filename;
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputFilenamePrefix=${outputFilenamePrefix}`);
|
||||
const luContent: string = Utility.loadFile(filename);
|
||||
const luData: LuData = await LuData.createLuData(
|
||||
luContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
true);
|
||||
let luLuisJsonStructureInLuFormatOutputFilename: string =
|
||||
outputFilenamePrefix + ".lu";
|
||||
luLuisJsonStructureInLuFormatOutputFilename = luData.dumpLuLuisJsonStructureInLuFormat(
|
||||
luLuisJsonStructureInLuFormatOutputFilename);
|
||||
let luLuisJsonStructureOutputFilename: string =
|
||||
outputFilenamePrefix + ".luis";
|
||||
luLuisJsonStructureOutputFilename = luData.dumpLuLuisJsonStructure(
|
||||
luLuisJsonStructureOutputFilename, undefined, 4);
|
||||
return [luLuisJsonStructureInLuFormatOutputFilename, luLuisJsonStructureOutputFilename];
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionData().then(() => { return; });
|
||||
}
|
||||
|
|
@ -1,513 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { Data } from "./Data";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export class ColumnarData extends Data {
|
||||
|
||||
public static createColumnarDataFromSamplingExistingColumnarDataUtterances(
|
||||
existingColumnarData: ColumnarData,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): ColumnarData {
|
||||
// -------------------------------------------------------------------
|
||||
const columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
existingColumnarData.getContent(),
|
||||
existingColumnarData.getFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = columnarData.luUtterances;
|
||||
const lengthUtterancesArray: number =
|
||||
luUtterances.length;
|
||||
columnarData.luUtterances = [];
|
||||
for (const index of samplingIndexArray) {
|
||||
if ((index < 0) || (index > lengthUtterancesArray)) {
|
||||
Utility.debuggingThrow(`(index|${index}|<0)||(index|${index}|>lengthUtterancesArray|${lengthUtterancesArray}|)`);
|
||||
}
|
||||
columnarData.luUtterances.push(luUtterances[index]);
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.intentInstanceIndexMapArray =
|
||||
columnarData.collectIntents(columnarData.luUtterances);
|
||||
columnarData.entityTypeInstanceIndexMapArray =
|
||||
columnarData.collectEntityTypes(columnarData.luUtterances);
|
||||
columnarData.intentsUtterancesWeights.intents = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
columnarData.intentsUtterancesWeights.utterances = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
columnarData.intentsUtterancesWeights.weights = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
columnarData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return columnarData;
|
||||
}
|
||||
|
||||
public static createColumnarDataFromFilteringExistingColumnarDataUtterances(
|
||||
existingColumnarData: ColumnarData,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): ColumnarData {
|
||||
// -------------------------------------------------------------------
|
||||
const columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
existingColumnarData.getContent(),
|
||||
existingColumnarData.getFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> =
|
||||
columnarData.luUtterances;
|
||||
columnarData.luUtterances = luUtterances.filter(
|
||||
(value: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number },
|
||||
index: number,
|
||||
array: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }>) => {
|
||||
return (filteringIndexSet.has(index));
|
||||
});
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.intentInstanceIndexMapArray =
|
||||
columnarData.collectIntents(columnarData.luUtterances);
|
||||
columnarData.entityTypeInstanceIndexMapArray =
|
||||
columnarData.collectEntityTypes(columnarData.luUtterances);
|
||||
columnarData.intentsUtterancesWeights.intents = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
columnarData.intentsUtterancesWeights.utterances = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
columnarData.intentsUtterancesWeights.weights = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
columnarData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return columnarData;
|
||||
}
|
||||
|
||||
public static createColumnarData(
|
||||
content: string,
|
||||
featurizer: NgramSubwordFeaturizer,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): ColumnarData {
|
||||
// -------------------------------------------------------------------
|
||||
const columnarData: ColumnarData =
|
||||
new ColumnarData(
|
||||
featurizer,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip);
|
||||
columnarData.content =
|
||||
content;
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.luUtterances =
|
||||
columnarData.retrieveColumnarUtterances(content);
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.intentInstanceIndexMapArray =
|
||||
columnarData.collectIntents(columnarData.luUtterances);
|
||||
columnarData.entityTypeInstanceIndexMapArray =
|
||||
columnarData.collectEntityTypes(columnarData.luUtterances);
|
||||
columnarData.intentsUtterancesWeights.intents = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
columnarData.intentsUtterancesWeights.utterances = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
columnarData.intentsUtterancesWeights.weights = columnarData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
columnarData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
columnarData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return columnarData;
|
||||
}
|
||||
|
||||
protected labelColumnIndex: number = 0;
|
||||
protected textColumnIndex: number = 1;
|
||||
protected weightColumnIndex: number = -1;
|
||||
protected linesToSkip: number = 0;
|
||||
|
||||
protected constructor(
|
||||
featurizer: NgramSubwordFeaturizer,
|
||||
labelColumnIndex: number = 0,
|
||||
textColumnIndex: number = 1,
|
||||
weightColumnIndex: number = -1,
|
||||
linesToSkip: number = 0) {
|
||||
super(featurizer);
|
||||
this.labelColumnIndex = labelColumnIndex;
|
||||
this.textColumnIndex = textColumnIndex;
|
||||
this.weightColumnIndex = weightColumnIndex;
|
||||
this.linesToSkip = linesToSkip;
|
||||
}
|
||||
|
||||
public async createDataFromSamplingExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
return ColumnarData.createColumnarDataFromSamplingExistingColumnarDataUtterances(
|
||||
existingData as ColumnarData,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
samplingIndexArray,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public async createDataFromFilteringExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
return ColumnarData.createColumnarDataFromFilteringExistingColumnarDataUtterances(
|
||||
existingData as ColumnarData,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
filteringIndexSet,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public retrieveColumnarUtterances(content: string): Array<{ // ---- NOTE the return is newly allocated, unlike the one of LuData
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> {
|
||||
const intentsUtterancesWeights: { "intents": string[], "utterances": string[], "weights": number[] } =
|
||||
Utility.loadLabelUtteranceColumnarContent(
|
||||
content, // ---- filename: string,
|
||||
this.getLabelColumnIndex(), // ---- labelColumnIndex: number = 0,
|
||||
this.getTextColumnIndex(), // ---- textColumnIndex: number = 1,
|
||||
this.getWeightColumnIndex(), // ---- weightColumnIndex: number = -1,
|
||||
this.getLinesToSkip(), // ---- lineIndexToStart: number = 0,
|
||||
"\t", // ---- columnDelimiter: string = "\t",
|
||||
"\n", // ---- rowDelimiter: string = "\n",
|
||||
"utf8", // ---- encoding: string = "utf8",
|
||||
-1, // ---- lineIndexToEnd: number = -1
|
||||
);
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = [];
|
||||
const intents: string[] = intentsUtterancesWeights.intents;
|
||||
const utterances: string[] = intentsUtterancesWeights.utterances;
|
||||
const weights: number[] = intentsUtterancesWeights.weights;
|
||||
for (let i = 0; i < intents.length; i++) {
|
||||
const intent: string = intents[i];
|
||||
const text: string = utterances[i];
|
||||
const weight: number = weights[i];
|
||||
const luUtterance: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number } = {
|
||||
entities: [],
|
||||
partOfSpeechTags: [],
|
||||
intent,
|
||||
text,
|
||||
weight,
|
||||
};
|
||||
luUtterances.push(luUtterance);
|
||||
}
|
||||
return luUtterances;
|
||||
}
|
||||
|
||||
public getLuObject(): any {
|
||||
return null; // ---- NOTE: not constructued from a ColumnarData file.
|
||||
}
|
||||
public getLuLuisJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from a ColumnarData file.
|
||||
}
|
||||
public getLuQnaJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from a ColumnarData file.
|
||||
}
|
||||
public getLuQnaAlterationsJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from a ColumnarData file.
|
||||
}
|
||||
|
||||
public getLabelColumnIndex(): number {
|
||||
return this.labelColumnIndex;
|
||||
}
|
||||
public getTextColumnIndex(): number {
|
||||
return this.textColumnIndex;
|
||||
}
|
||||
public getWeightColumnIndex(): number {
|
||||
return this.weightColumnIndex;
|
||||
}
|
||||
public getLinesToSkip(): number {
|
||||
return this.linesToSkip;
|
||||
}
|
||||
|
||||
public dumpLuObject(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ==== NOTE-TODO ==== a ColumnarData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure out of columnar content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- JSON.stringify(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuObject(),
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- replacer,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- space));
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructure(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ==== NOTE-TODO ==== a ColumnarData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure out of columnar content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- JSON.stringify(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuLuisJsonStructure(),
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- replacer,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- space));
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructureInLuFormat(
|
||||
filename: string): string {
|
||||
// ==== NOTE-TODO ==== a ColumnarData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure out of columnar content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- constructMdFromLUIS(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuLuisJsonStructure()));
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,103 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { isNull } from "util";
|
||||
import { isUndefined } from "util";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { EntityAnnotatedCorpusData } from "./EntityAnnotatedCorpusData";
|
||||
import { ColumnarData } from "./ColumnarData";
|
||||
import { LuData } from "./LuData";
|
||||
import { Data } from "./Data";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export class DataUtility {
|
||||
|
||||
public static readonly DATA_FORMAT_TYPE_LU: string = "lu";
|
||||
public static readonly DATA_FORMAT_TYPE_QUESTION_AND_ANSWER: string = "qna";
|
||||
public static readonly DATA_FORMAT_TYPE_ENTITY_ANNOTATED_CORPUS: string = "eac";
|
||||
public static readonly DATA_FORMAT_TYPE_TAB_DELIMITED: string = "tsv";
|
||||
|
||||
public static readonly DATA_FORMAT_FILE_ENTENSION_LU: string =
|
||||
"." + DataUtility.DATA_FORMAT_TYPE_LU;
|
||||
public static readonly DATA_FORMAT_FILE_ENTENSION_QUESTION_AND_ANSWER: string =
|
||||
"." + DataUtility.DATA_FORMAT_TYPE_QUESTION_AND_ANSWER;
|
||||
public static readonly DATA_FORMAT_FILE_ENTENSION_ENTITY_ANNOTATED_CORPUS: string =
|
||||
"." + DataUtility.DATA_FORMAT_TYPE_ENTITY_ANNOTATED_CORPUS;
|
||||
public static readonly DATA_FORMAT_FILE_ENTENSION_TAB_DELIMITED: string =
|
||||
"." + DataUtility.DATA_FORMAT_TYPE_TAB_DELIMITED;
|
||||
|
||||
public static getDataFileTypeFromFilenameExtension(filename: string): string {
|
||||
let filetype: string = DataUtility.DATA_FORMAT_TYPE_TAB_DELIMITED;
|
||||
if (filename.endsWith(DataUtility.DATA_FORMAT_FILE_ENTENSION_LU)) {
|
||||
filetype = DataUtility.DATA_FORMAT_TYPE_LU;
|
||||
} else if (filename.endsWith(DataUtility.DATA_FORMAT_FILE_ENTENSION_QUESTION_AND_ANSWER)) {
|
||||
filetype = DataUtility.DATA_FORMAT_TYPE_QUESTION_AND_ANSWER;
|
||||
} else if (filename.endsWith(DataUtility.DATA_FORMAT_FILE_ENTENSION_ENTITY_ANNOTATED_CORPUS)) {
|
||||
filetype = DataUtility.DATA_FORMAT_TYPE_ENTITY_ANNOTATED_CORPUS;
|
||||
} else if (filename.endsWith(DataUtility.DATA_FORMAT_FILE_ENTENSION_TAB_DELIMITED)) {
|
||||
filetype = DataUtility.DATA_FORMAT_TYPE_TAB_DELIMITED;
|
||||
}
|
||||
return filetype;
|
||||
}
|
||||
public static async LoadData(
|
||||
filename: string,
|
||||
featurizerNullable: NgramSubwordFeaturizer|null = null,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean = true,
|
||||
filetype: string = "",
|
||||
labelColumnIndex: number = 0,
|
||||
textColumnIndex: number = 1,
|
||||
weightColumnIndex: number = -1,
|
||||
linesToSkip: number = 0): Promise<Data> {
|
||||
const content: string =
|
||||
Utility.loadFile(filename);
|
||||
if (Utility.isEmptyString(filetype)) {
|
||||
filetype = DataUtility.getDataFileTypeFromFilenameExtension(filename);
|
||||
}
|
||||
if (isNull(featurizerNullable) || (isUndefined(featurizerNullable))) {
|
||||
featurizerNullable = new NgramSubwordFeaturizer();
|
||||
toResetFeaturizerLabelFeatureMaps = true;
|
||||
}
|
||||
if (filetype === DataUtility.DATA_FORMAT_TYPE_LU) {
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
content,
|
||||
featurizerNullable,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
return luData;
|
||||
}
|
||||
if (filetype === DataUtility.DATA_FORMAT_TYPE_QUESTION_AND_ANSWER) {
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
content,
|
||||
featurizerNullable,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
return luData;
|
||||
}
|
||||
if (filetype === DataUtility.DATA_FORMAT_TYPE_ENTITY_ANNOTATED_CORPUS) {
|
||||
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
|
||||
EntityAnnotatedCorpusData.createEntityAnnotatedCorpusData(
|
||||
content,
|
||||
featurizerNullable,
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
return entityAnnotatedCorpusData;
|
||||
}
|
||||
{
|
||||
const columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
content,
|
||||
featurizerNullable,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
return columnarData;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,463 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { Data } from "./Data";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export class EntityAnnotatedCorpusData extends Data {
|
||||
|
||||
public static createEntityAnnotatedCorpusDataFromSamplingExistingEntityAnnotatedCorpusDataUtterances(
|
||||
existingEntityAnnotatedCorpusData: EntityAnnotatedCorpusData,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): EntityAnnotatedCorpusData {
|
||||
// -------------------------------------------------------------------
|
||||
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
|
||||
EntityAnnotatedCorpusData.createEntityAnnotatedCorpusData(
|
||||
existingEntityAnnotatedCorpusData.getContent(),
|
||||
existingEntityAnnotatedCorpusData.getFeaturizer(),
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = entityAnnotatedCorpusData.luUtterances;
|
||||
const lengthUtterancesArray: number =
|
||||
luUtterances.length;
|
||||
entityAnnotatedCorpusData.luUtterances = [];
|
||||
for (const index of samplingIndexArray) {
|
||||
if ((index < 0) || (index > lengthUtterancesArray)) {
|
||||
Utility.debuggingThrow(`(index|${index}|<0)||(index|${index}|>lengthUtterancesArray|${lengthUtterancesArray}|)`);
|
||||
}
|
||||
entityAnnotatedCorpusData.luUtterances.push(luUtterances[index]);
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.intentInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectIntents(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.entityTypeInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectEntityTypes(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.intents = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.utterances = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.weights = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
entityAnnotatedCorpusData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return entityAnnotatedCorpusData;
|
||||
}
|
||||
|
||||
public static createEntityAnnotatedCorpusDataFromFilteringExistingEntityAnnotatedCorpusDataUtterances(
|
||||
existingEntityAnnotatedCorpusData: EntityAnnotatedCorpusData,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): EntityAnnotatedCorpusData {
|
||||
// -------------------------------------------------------------------
|
||||
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
|
||||
EntityAnnotatedCorpusData.createEntityAnnotatedCorpusData(
|
||||
existingEntityAnnotatedCorpusData.getContent(),
|
||||
existingEntityAnnotatedCorpusData.getFeaturizer(),
|
||||
linesToSkip,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = entityAnnotatedCorpusData.luUtterances;
|
||||
entityAnnotatedCorpusData.luUtterances = luUtterances.filter(
|
||||
(value: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number },
|
||||
index: number,
|
||||
array: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }>) => {
|
||||
return (filteringIndexSet.has(index));
|
||||
});
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.intentInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectIntents(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.entityTypeInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectEntityTypes(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.intents = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.utterances = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.weights = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
entityAnnotatedCorpusData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return entityAnnotatedCorpusData;
|
||||
}
|
||||
|
||||
public static createEntityAnnotatedCorpusData(
|
||||
content: string,
|
||||
featurizer: NgramSubwordFeaturizer,
|
||||
linesToSkip: number,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): EntityAnnotatedCorpusData {
|
||||
// -------------------------------------------------------------------
|
||||
const entityAnnotatedCorpusData: EntityAnnotatedCorpusData =
|
||||
new EntityAnnotatedCorpusData(
|
||||
featurizer,
|
||||
linesToSkip);
|
||||
entityAnnotatedCorpusData.content =
|
||||
content;
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.luUtterances = entityAnnotatedCorpusData.retrieveEntityAnnotatedCorpusUtterances(
|
||||
content);
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.intentInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectIntents(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.entityTypeInstanceIndexMapArray =
|
||||
entityAnnotatedCorpusData.collectEntityTypes(entityAnnotatedCorpusData.luUtterances);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.intents = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.intent as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.utterances = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.text as string);
|
||||
entityAnnotatedCorpusData.intentsUtterancesWeights.weights = entityAnnotatedCorpusData.luUtterances.map(
|
||||
(entry: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
entityAnnotatedCorpusData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
entityAnnotatedCorpusData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return entityAnnotatedCorpusData;
|
||||
}
|
||||
|
||||
protected linesToSkip: number = 0;
|
||||
|
||||
protected constructor(
|
||||
featurizer: NgramSubwordFeaturizer,
|
||||
linesToSkip: number = 0) {
|
||||
super(featurizer);
|
||||
this.linesToSkip = linesToSkip;
|
||||
}
|
||||
|
||||
public async createDataFromSamplingExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
// tslint:disable-next-line: max-line-length
|
||||
return EntityAnnotatedCorpusData.createEntityAnnotatedCorpusDataFromSamplingExistingEntityAnnotatedCorpusDataUtterances(
|
||||
existingData as EntityAnnotatedCorpusData,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- labelColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- textColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- weightColumnIndex,
|
||||
linesToSkip,
|
||||
samplingIndexArray,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public async createDataFromFilteringExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
// tslint:disable-next-line: max-line-length
|
||||
return EntityAnnotatedCorpusData.createEntityAnnotatedCorpusDataFromFilteringExistingEntityAnnotatedCorpusDataUtterances(
|
||||
existingData as EntityAnnotatedCorpusData,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- labelColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- textColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-EntityAnnotatedCorpusData ---- weightColumnIndex,
|
||||
linesToSkip,
|
||||
filteringIndexSet,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public retrieveEntityAnnotatedCorpusUtterances( // ---- NOTE the return is newly allocated, unlike the one of LuData
|
||||
content: string,
|
||||
includePartOfSpeechTagTagAsEntities: boolean = true,
|
||||
utteranceReconstructionDelimiter: string = " ",
|
||||
defaultEntityTag: string = "O",
|
||||
useIdForIntent: boolean = true): Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> {
|
||||
const entityAnnotatedCorpusTypes: {
|
||||
"ids": string[],
|
||||
"wordArrays": string[][],
|
||||
"partOfSpeechTagArrays": string[][],
|
||||
"entityTagArrays": string[][] } =
|
||||
Utility.loadEntityAnnotatedCorpusContent(
|
||||
content, // ---- filename: string,
|
||||
this.getLinesToSkip(), // ---- lineIndexToStart: number = 0,
|
||||
",", // ---- columnDelimiter: string = ",",
|
||||
"\n", // ---- rowDelimiter: string = "\n",
|
||||
-1, // ---- lineIndexToEnd: number = -1
|
||||
);
|
||||
const entityAnnotatedCorpusUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> =
|
||||
Utility.entityAnnotatedCorpusTypesToEntityAnnotatedCorpusUtterances(
|
||||
entityAnnotatedCorpusTypes,
|
||||
includePartOfSpeechTagTagAsEntities,
|
||||
utteranceReconstructionDelimiter,
|
||||
defaultEntityTag,
|
||||
useIdForIntent);
|
||||
return entityAnnotatedCorpusUtterances;
|
||||
}
|
||||
|
||||
public getLuObject(): any {
|
||||
return null; // ---- NOTE: not constructued from an EntityAnnotatedCorpusData file.
|
||||
}
|
||||
public getLuLuisJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from an EntityAnnotatedCorpusData file.
|
||||
}
|
||||
public getLuQnaJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from an EntityAnnotatedCorpusData file.
|
||||
}
|
||||
public getLuQnaAlterationsJsonStructure(): any {
|
||||
return null; // ---- NOTE: not constructued from an EntityAnnotatedCorpusData file.
|
||||
}
|
||||
|
||||
public getLinesToSkip(): number {
|
||||
return this.linesToSkip;
|
||||
}
|
||||
|
||||
public dumpLuObject(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ==== NOTE-TODO ==== a EntityAnnotatedCorpusData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure
|
||||
// ==== NOTE-TODO ==== out of EntityAnnotatedCorpusData content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- JSON.stringify(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuObject(),
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- replacer,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- space));
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructure(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ==== NOTE-TODO ==== a EntityAnnotatedCorpusData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure
|
||||
// ==== NOTE-TODO ==== out of EntityAnnotatedCorpusData content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- JSON.stringify(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuLuisJsonStructure(),
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- replacer,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- space));
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructureInLuFormat(
|
||||
filename: string): string {
|
||||
// ==== NOTE-TODO ==== a EntityAnnotatedCorpusData source does not have a LU LUIS structure,
|
||||
// ==== NOTE-TODO ==== need to develop logic for creating a LU LUIS structure
|
||||
// ==== NOTE-TODO ==== out of EntityAnnotatedCorpusData content!
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.dumpFile(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- filename,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- constructMdFromLUIS(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- this.getLuLuisJsonStructure()));
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
|
@ -1,309 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// tslint:disable-next-line: no-var-requires
|
||||
const parseFile = require("@microsoft/bf-lu").parser.parseFile;
|
||||
// tslint:disable-next-line: no-var-requires
|
||||
const constructMdFromLUIS = require("@microsoft/bf-lu").refresh.constructMdFromLUIS;
|
||||
|
||||
import { Data } from "./Data";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export class LuData extends Data {
|
||||
|
||||
public static async createLuDataFromSamplingExistingLuDataUtterances(
|
||||
existingLuData: LuData,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<LuData> {
|
||||
// -------------------------------------------------------------------
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
existingLuData.getContent(),
|
||||
existingLuData.getFeaturizer(),
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luLuisJsonStructure: any =
|
||||
luData.getLuLuisJsonStructure();
|
||||
const utterancesArray: any[] =
|
||||
luData.retrieveLuisLuUtterances(luLuisJsonStructure);
|
||||
const lengthUtterancesArray: number =
|
||||
utterancesArray.length;
|
||||
luLuisJsonStructure.utterances = [];
|
||||
for (const index of samplingIndexArray) {
|
||||
if ((index < 0) || (index > lengthUtterancesArray)) {
|
||||
Utility.debuggingThrow(`(index|${index}|<0)||(index|${index}|>lengthUtterancesArray|${lengthUtterancesArray}|)`);
|
||||
}
|
||||
luLuisJsonStructure.utterances.push(utterancesArray[index]);
|
||||
}
|
||||
// ---- NOTE-FOR-REFERENCE ---- luLuisJsonStructure.utterances = utterancesArray.filter(
|
||||
// ---- NOTE-FOR-REFERENCE ---- (value: any, index: number, array: any[]) => {
|
||||
// ---- NOTE-FOR-REFERENCE ---- return (samplingIndexArray.has(index));
|
||||
// ---- NOTE-FOR-REFERENCE ---- });
|
||||
// -------------------------------------------------------------------
|
||||
luData.luUtterances =
|
||||
luData.retrieveLuUtterances(luLuisJsonStructure);
|
||||
// -------------------------------------------------------------------
|
||||
luData.intentInstanceIndexMapArray =
|
||||
luData.collectIntents(luData.luUtterances);
|
||||
luData.entityTypeInstanceIndexMapArray =
|
||||
luData.collectEntityTypes(luData.luUtterances);
|
||||
luData.intentsUtterancesWeights.intents = luData.luUtterances.map(
|
||||
(entry: any) => entry.intent as string);
|
||||
luData.intentsUtterancesWeights.utterances = luData.luUtterances.map(
|
||||
(entry: any) => entry.text as string);
|
||||
luData.intentsUtterancesWeights.weights = luData.luUtterances.map(
|
||||
(entry: any) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
luData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
luData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return luData;
|
||||
}
|
||||
|
||||
public static async createLuDataFromFilteringExistingLuDataUtterances(
|
||||
existingLuData: LuData,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<LuData> {
|
||||
// -------------------------------------------------------------------
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
existingLuData.getContent(),
|
||||
existingLuData.getFeaturizer(),
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
// -------------------------------------------------------------------
|
||||
const luLuisJsonStructure: any =
|
||||
luData.getLuLuisJsonStructure();
|
||||
const utterancesArray: any[] =
|
||||
luData.retrieveLuisLuUtterances(luLuisJsonStructure);
|
||||
const lengthUtterancesArray: number =
|
||||
utterancesArray.length;
|
||||
luLuisJsonStructure.utterances = [];
|
||||
for (const index of filteringIndexSet) {
|
||||
if ((index < 0) || (index > lengthUtterancesArray)) {
|
||||
Utility.debuggingThrow(`(index|${index}|<0)||(index|${index}|>lengthUtterancesArray|${lengthUtterancesArray}|)`);
|
||||
}
|
||||
luLuisJsonStructure.utterances.push(utterancesArray[index]);
|
||||
}
|
||||
// ---- NOTE-FOR-REFERENCE ---- luLuisJsonStructure.utterances = utterancesArray.filter(
|
||||
// ---- NOTE-FOR-REFERENCE ---- (value: any, index: number, array: any[]) => {
|
||||
// ---- NOTE-FOR-REFERENCE ---- return (filteringIndexSet.has(index));
|
||||
// ---- NOTE-FOR-REFERENCE ---- });
|
||||
// -------------------------------------------------------------------
|
||||
luData.luUtterances =
|
||||
luData.retrieveLuUtterances(luLuisJsonStructure);
|
||||
// -------------------------------------------------------------------
|
||||
luData.intentInstanceIndexMapArray =
|
||||
luData.collectIntents(luData.luUtterances);
|
||||
luData.entityTypeInstanceIndexMapArray =
|
||||
luData.collectEntityTypes(luData.luUtterances);
|
||||
luData.intentsUtterancesWeights.intents = luData.luUtterances.map(
|
||||
(entry: any) => entry.intent as string);
|
||||
luData.intentsUtterancesWeights.utterances = luData.luUtterances.map(
|
||||
(entry: any) => entry.text as string);
|
||||
luData.intentsUtterancesWeights.weights = luData.luUtterances.map(
|
||||
(entry: any) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
luData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
luData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return luData;
|
||||
}
|
||||
|
||||
public static async createLuData(
|
||||
content: string,
|
||||
featurizer: NgramSubwordFeaturizer,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<LuData> {
|
||||
// -------------------------------------------------------------------
|
||||
const luData: LuData =
|
||||
new LuData(featurizer);
|
||||
luData.content =
|
||||
content;
|
||||
// -------------------------------------------------------------------
|
||||
luData.luObject =
|
||||
await parseFile(content);
|
||||
// -------------------------------------------------------------------
|
||||
const luLuisJsonStructure: any =
|
||||
luData.getLuLuisJsonStructure();
|
||||
// -------------------------------------------------------------------
|
||||
luData.luUtterances =
|
||||
luData.retrieveLuUtterances(luLuisJsonStructure);
|
||||
// -------------------------------------------------------------------
|
||||
luData.intentInstanceIndexMapArray =
|
||||
luData.collectIntents(luData.luUtterances);
|
||||
luData.entityTypeInstanceIndexMapArray =
|
||||
luData.collectEntityTypes(luData.luUtterances);
|
||||
luData.intentsUtterancesWeights.intents = luData.luUtterances.map(
|
||||
(entry: any) => entry.intent as string);
|
||||
luData.intentsUtterancesWeights.utterances = luData.luUtterances.map(
|
||||
(entry: any) => entry.text as string);
|
||||
luData.intentsUtterancesWeights.weights = luData.luUtterances.map(
|
||||
(entry: any) => entry.weight as number);
|
||||
// -------------------------------------------------------------------
|
||||
if (toResetFeaturizerLabelFeatureMaps) {
|
||||
luData.resetFeaturizerLabelFeatureMaps();
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
luData.featurizeIntentsUtterances();
|
||||
// -------------------------------------------------------------------
|
||||
return luData;
|
||||
}
|
||||
|
||||
protected luObject: any = null;
|
||||
|
||||
protected constructor(
|
||||
featurizer: NgramSubwordFeaturizer) {
|
||||
super(featurizer);
|
||||
}
|
||||
|
||||
public async createDataFromSamplingExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
return await LuData.createLuDataFromSamplingExistingLuDataUtterances(
|
||||
existingData as LuData,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- labelColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- textColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- weightColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- linesToSkip,
|
||||
samplingIndexArray,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public async createDataFromFilteringExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data> {
|
||||
return LuData.createLuDataFromFilteringExistingLuDataUtterances(
|
||||
existingData as LuData,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- labelColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- textColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- weightColumnIndex,
|
||||
// ---- NOTE-NO-NEED-FOR-LuData ---- linesToSkip,
|
||||
filteringIndexSet,
|
||||
toResetFeaturizerLabelFeatureMaps);
|
||||
}
|
||||
|
||||
public retrieveLuisLuUtterances(luLuisJsonStructure: any): any[] { // ---- NOTE: a shallow copy
|
||||
return (luLuisJsonStructure.utterances as any[]);
|
||||
}
|
||||
public retrieveLuUtterances(luLuisJsonStructure: any): Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> {
|
||||
const weight: number = 1;
|
||||
const utterancesArray: any[] =
|
||||
this.retrieveLuisLuUtterances(luLuisJsonStructure);
|
||||
const luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = [];
|
||||
utterancesArray.forEach(
|
||||
(entry: any) => {
|
||||
const entities: Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}> = entry.entities;
|
||||
const partOfSpeechTags: Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}> = [];
|
||||
const intent: string =
|
||||
entry.intent;
|
||||
const text: string =
|
||||
entry.text;
|
||||
luUtterances.push({
|
||||
entities,
|
||||
partOfSpeechTags,
|
||||
intent,
|
||||
text,
|
||||
weight,
|
||||
});
|
||||
});
|
||||
return luUtterances;
|
||||
// ---- NOTE-FOR-REFERENCE ---- return (luLuisJsonStructure.utterances as any[]);
|
||||
}
|
||||
|
||||
public getLuObject(): any {
|
||||
return this.luObject;
|
||||
}
|
||||
public getLuLuisJsonStructure(): any {
|
||||
return this.luObject.LUISJsonStructure;
|
||||
}
|
||||
public getLuQnaJsonStructure(): any {
|
||||
return this.luObject.qnaJsonStructure;
|
||||
}
|
||||
public getLuQnaAlterationsJsonStructure(): any {
|
||||
return this.luObject.qnaAlterations;
|
||||
}
|
||||
|
||||
public dumpLuObject(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
return Utility.dumpFile(
|
||||
filename,
|
||||
JSON.stringify(
|
||||
this.getLuObject(),
|
||||
replacer,
|
||||
space));
|
||||
}
|
||||
public dumpLuLuisJsonStructure(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
return Utility.dumpFile(
|
||||
filename,
|
||||
JSON.stringify(
|
||||
this.getLuLuisJsonStructure(),
|
||||
replacer,
|
||||
space));
|
||||
}
|
||||
public dumpLuLuisJsonStructureInLuFormat(
|
||||
filename: string): string {
|
||||
return Utility.dumpFile(
|
||||
filename,
|
||||
constructMdFromLUIS(
|
||||
this.getLuLuisJsonStructure()));
|
||||
}
|
||||
}
|
||||
|
|
@ -1,502 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { isNull } from "util";
|
||||
import { isUndefined } from "util";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../utility/Utility";
|
||||
|
||||
export abstract class Data {
|
||||
|
||||
protected content: string = "";
|
||||
protected luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> = [];
|
||||
protected intentInstanceIndexMapArray: Map<string, number[]> = new Map<string, number[]>();
|
||||
protected entityTypeInstanceIndexMapArray: Map<string, number[]> = new Map<string, number[]>();
|
||||
|
||||
protected intentsUtterancesWeights: {
|
||||
"intents": string[],
|
||||
"utterances": string[],
|
||||
"weights": number[] } =
|
||||
{ intents: [], utterances: [], weights: [] };
|
||||
protected intentUtteranceSparseIndexArrays: {
|
||||
"intentLabelIndexArray": number[],
|
||||
"utteranceFeatureIndexArrays": number[][] } =
|
||||
{ intentLabelIndexArray: [], utteranceFeatureIndexArrays: [] };
|
||||
|
||||
protected featurizer: NgramSubwordFeaturizer;
|
||||
|
||||
protected constructor(featurizer: NgramSubwordFeaturizer) {
|
||||
if (isNull(featurizer) || (isUndefined(featurizer))) {
|
||||
Utility.debuggingThrow(
|
||||
"input featurizer is null");
|
||||
}
|
||||
this.featurizer = featurizer;
|
||||
}
|
||||
|
||||
public abstract async createDataFromSamplingExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
samplingIndexArray: number[],
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data>;
|
||||
|
||||
public abstract async createDataFromFilteringExistingDataUtterances(
|
||||
existingData: Data,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
filteringIndexSet: Set<number>,
|
||||
toResetFeaturizerLabelFeatureMaps: boolean): Promise<Data>;
|
||||
|
||||
public collectEntityTypes(luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }>): Map<string, number[]> {
|
||||
const entityTypeInstanceIndexMapArray: Map<string, number[]> = new Map<string, number[]>();
|
||||
luUtterances.forEach(
|
||||
(element: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number },
|
||||
index: number) => {
|
||||
const entities: Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}> = element.entities;
|
||||
entities.forEach((entityElement: {
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}) => {
|
||||
const entityType: string = entityElement.entity as string;
|
||||
if (entityType) {
|
||||
Utility.addKeyValueToNumberMapArray(
|
||||
entityTypeInstanceIndexMapArray,
|
||||
entityType,
|
||||
index);
|
||||
}
|
||||
});
|
||||
});
|
||||
return entityTypeInstanceIndexMapArray;
|
||||
}
|
||||
public collectIntents(luUtterances: Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }>): Map<string, number[]> {
|
||||
const intentInstanceIndexMapArray: Map<string, number[]> = new Map<string, number[]>();
|
||||
luUtterances.forEach(
|
||||
(element: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number },
|
||||
index: number) => {
|
||||
const intent: string = element.intent as string;
|
||||
if (intent) {
|
||||
Utility.addKeyValueToNumberMapArray(
|
||||
intentInstanceIndexMapArray,
|
||||
intent,
|
||||
index);
|
||||
}
|
||||
});
|
||||
return intentInstanceIndexMapArray;
|
||||
}
|
||||
|
||||
public getContent(): string {
|
||||
return this.content;
|
||||
}
|
||||
|
||||
public getLuObject(): any {
|
||||
return null;
|
||||
}
|
||||
public getLuLuisJsonStructure(): any {
|
||||
return null;
|
||||
}
|
||||
public getLuQnaJsonStructure(): any {
|
||||
return null;
|
||||
}
|
||||
public getLuQnaAlterationsJsonStructure(): any {
|
||||
return null;
|
||||
}
|
||||
|
||||
public getLuUtterances(): Array<{
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number }> {
|
||||
return this.luUtterances;
|
||||
}
|
||||
public getIntentInstanceIndexMapArray(): Map<string, number[]> {
|
||||
return this.intentInstanceIndexMapArray;
|
||||
}
|
||||
public getEntityTypeInstanceIndexMapArray(): Map<string, number[]> {
|
||||
return this.entityTypeInstanceIndexMapArray;
|
||||
}
|
||||
|
||||
public dumpLuObject(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ---- NOTE: a child class can override this function.
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructure(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
// ---- NOTE: a child class can override this function.
|
||||
return "";
|
||||
}
|
||||
public dumpLuLuisJsonStructureInLuFormat(
|
||||
filename: string): string {
|
||||
// ---- NOTE: a child class can override this function.
|
||||
return "";
|
||||
}
|
||||
|
||||
public dumpLuUtterances(
|
||||
filename: string,
|
||||
replacer?: (this: any, key: string, value: any) => any,
|
||||
space?: string | number): string {
|
||||
return Utility.dumpFile(
|
||||
filename,
|
||||
JSON.stringify(
|
||||
this.getLuUtterances(),
|
||||
replacer,
|
||||
space));
|
||||
}
|
||||
|
||||
public getNumberLuUtterances(): number {
|
||||
return this.getLuUtterances().length;
|
||||
}
|
||||
public getNumberIntents(): number {
|
||||
return this.getIntentInstanceIndexMapArray().size;
|
||||
}
|
||||
public getNumberEntityTypes(): number {
|
||||
return this.getEntityTypeInstanceIndexMapArray().size;
|
||||
}
|
||||
|
||||
public getIntentsUtterancesWeights(): {
|
||||
"intents": string[],
|
||||
"utterances": string[],
|
||||
"weights": number[] } {
|
||||
return this.intentsUtterancesWeights;
|
||||
}
|
||||
public getIntents(): string[] {
|
||||
return this.intentsUtterancesWeights.intents;
|
||||
}
|
||||
public getUtterances(): string[] {
|
||||
return this.intentsUtterancesWeights.utterances;
|
||||
}
|
||||
public getWeights(): number[] {
|
||||
return this.intentsUtterancesWeights.weights;
|
||||
}
|
||||
|
||||
public getIntentUtteranceSparseIndexArrays(): {
|
||||
"intentLabelIndexArray": number[],
|
||||
"utteranceFeatureIndexArrays": number[][] } {
|
||||
return this.intentUtteranceSparseIndexArrays;
|
||||
}
|
||||
public getIntentLabelIndexArray(): number[] {
|
||||
return this.intentUtteranceSparseIndexArrays.intentLabelIndexArray;
|
||||
}
|
||||
public getUtteranceFeatureIndexArrays(): number[][] {
|
||||
return this.intentUtteranceSparseIndexArrays.utteranceFeatureIndexArrays;
|
||||
}
|
||||
|
||||
public resetFeaturizerLabelFeatureMaps(): void {
|
||||
this.getFeaturizer().resetLabelFeatureMaps(
|
||||
this.getIntentsUtterancesWeights());
|
||||
}
|
||||
public featurizeIntentsUtterances(): void {
|
||||
this.intentUtteranceSparseIndexArrays =
|
||||
this.getFeaturizer().createIntentUtteranceSparseIndexArrays(
|
||||
this.getIntentsUtterancesWeights());
|
||||
}
|
||||
public featurize(inputUtterance: string): string[] {
|
||||
return this.getFeaturizer().featurize(inputUtterance);
|
||||
}
|
||||
public getFeaturizer(): NgramSubwordFeaturizer {
|
||||
return this.featurizer;
|
||||
}
|
||||
public getFeaturizerLabels(): string[] {
|
||||
return this.getFeaturizer().getLabels();
|
||||
}
|
||||
public getFeaturizerLabelMap(): { [id: string]: number } {
|
||||
return this.getFeaturizer().getLabelMap();
|
||||
}
|
||||
public getFeaturizerFeatures(): string[] {
|
||||
return this.getFeaturizer().getFeatures();
|
||||
}
|
||||
public getFeaturizerFeatureMap(): { [id: string]: number } {
|
||||
return this.getFeaturizer().getFeatureMap();
|
||||
}
|
||||
|
||||
public collectUtteranceIndexSetSeedingIntentTrainingSet(
|
||||
seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>>,
|
||||
candidateUtteranceIndexSet: Set<number>,
|
||||
limitInitialNumberOfInstancesPerCategory: number = 10): {
|
||||
"seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels": Map<string, Set<number>>,
|
||||
"candidateUtteranceIndexSetSampled": Set<number>,
|
||||
"candidateUtteranceIndexSetRemaining": Set<number>,
|
||||
} {
|
||||
const candidateUtteranceIndexSetSampled: Set<number> = new Set<number>();
|
||||
for (const luUtteranceIndex of candidateUtteranceIndexSet) {
|
||||
const luUtterance: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number } =
|
||||
this.luUtterances[luUtteranceIndex];
|
||||
const intent: string =
|
||||
luUtterance.intent as string;
|
||||
const utteranceIndexIntentSet: Set<number> =
|
||||
seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels.get(intent) as Set<number>;
|
||||
const existingNumberUtterances: number =
|
||||
utteranceIndexIntentSet.size;
|
||||
if ((limitInitialNumberOfInstancesPerCategory < 0) ||
|
||||
(existingNumberUtterances < limitInitialNumberOfInstancesPerCategory)) {
|
||||
utteranceIndexIntentSet.add(luUtteranceIndex);
|
||||
candidateUtteranceIndexSetSampled.add(luUtteranceIndex);
|
||||
}
|
||||
}
|
||||
const candidateUtteranceIndexSetRemaining: Set<number> =
|
||||
new Set<number>([...candidateUtteranceIndexSet].filter((entry: number) => {
|
||||
return !candidateUtteranceIndexSetSampled.has(entry);
|
||||
}));
|
||||
return {
|
||||
candidateUtteranceIndexSetRemaining,
|
||||
candidateUtteranceIndexSetSampled,
|
||||
seedingUtteranceIndexIntentMapCoveringAllIntentEntityLabels,
|
||||
};
|
||||
}
|
||||
|
||||
public collectSmallUtteranceIndexSetCoveringAllIntentEntityLabels(
|
||||
toShuffle: boolean = true,
|
||||
toEnsureEachIntentHasOneUtteranceLabel: boolean = true,
|
||||
toEnsureEachEntityTypeHasOneUtteranceLabel: boolean = true): {
|
||||
"smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels": Map<string, Set<number>>,
|
||||
"smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels": Map<string, Set<number>>,
|
||||
"smallUtteranceIndexSetCoveringAllIntentEntityLabels": Set<number>,
|
||||
"remainingUtteranceIndexSet": Set<number>,
|
||||
} {
|
||||
const luUtteranceIndexes: number[] =
|
||||
Array.from(Array(this.getNumberLuUtterances()).keys());
|
||||
if (toShuffle) {
|
||||
Utility.shuffle(luUtteranceIndexes);
|
||||
}
|
||||
const smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
new Map<string, Set<number>>();
|
||||
const smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels: Map<string, Set<number>> =
|
||||
new Map<string, Set<number>>();
|
||||
const smallUtteranceIndexSetCoveringAllIntentEntityLabels: Set<number> =
|
||||
new Set<number>();
|
||||
if (toEnsureEachIntentHasOneUtteranceLabel || toEnsureEachEntityTypeHasOneUtteranceLabel) {
|
||||
const numberIntents: number =
|
||||
this.getNumberIntents();
|
||||
const numberEntityTypes: number =
|
||||
this.getNumberEntityTypes();
|
||||
const intentSet: Set<string> =
|
||||
new Set<string>();
|
||||
const entityTypeSet: Set<string> =
|
||||
new Set<string>();
|
||||
for (const luUtteranceIndex of luUtteranceIndexes) {
|
||||
// for (let i: number = 0; i < luUtteranceIndexes.length; i++) {
|
||||
// const luUtteranceIndex: number = luUtteranceIndexes[i];
|
||||
const luUtterance: {
|
||||
"entities": Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"partOfSpeechTags": Array<{
|
||||
"partOfSpeechTag": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}>,
|
||||
"intent": string,
|
||||
"text": string,
|
||||
"weight": number } = this.luUtterances[luUtteranceIndex];
|
||||
let hasNewUtteranceFoundForCoveringAllIntentEntityLabels: boolean = false;
|
||||
if (toEnsureEachIntentHasOneUtteranceLabel) {
|
||||
if (intentSet.size < numberIntents) {
|
||||
const intent: string = luUtterance.intent as string;
|
||||
if (!(intentSet.has(intent))) {
|
||||
intentSet.add(intent);
|
||||
hasNewUtteranceFoundForCoveringAllIntentEntityLabels = true;
|
||||
// Utility.debuggingLog(
|
||||
// `i=${i}, ` +
|
||||
// `luUtteranceIndex=${luUtteranceIndex}, ` +
|
||||
// `intent=${intent}, ` +
|
||||
// `intentSet.size=${intentSet.size}, ` +
|
||||
// `smallUtteranceIndexSetCoveringAllIntentEntityLabels.size=` +
|
||||
// `${smallUtteranceIndexSetCoveringAllIntentEntityLabels.size}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (toEnsureEachEntityTypeHasOneUtteranceLabel) {
|
||||
if (entityTypeSet.size < numberEntityTypes) {
|
||||
const entities: Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}> = luUtterance.entities;
|
||||
entities.forEach((entityElement: {
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}) => {
|
||||
const entityType: string = entityElement.entity as string;
|
||||
if (entityType) {
|
||||
if (!(entityTypeSet.has(entityType))) {
|
||||
entityTypeSet.add(entityType);
|
||||
hasNewUtteranceFoundForCoveringAllIntentEntityLabels = true;
|
||||
// Utility.debuggingLog(
|
||||
// `i=${i}, ` +
|
||||
// `luUtteranceIndex=${luUtteranceIndex}, ` +
|
||||
// `entityType=${entityType}, ` +
|
||||
// `entityTypeSet.size=${entityTypeSet.size}, ` +
|
||||
// `smallUtteranceIndexSetCoveringAllIntentEntityLabels.size=` +
|
||||
// `${smallUtteranceIndexSetCoveringAllIntentEntityLabels.size}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
if (hasNewUtteranceFoundForCoveringAllIntentEntityLabels) {
|
||||
{
|
||||
smallUtteranceIndexSetCoveringAllIntentEntityLabels.add(luUtteranceIndex);
|
||||
}
|
||||
{
|
||||
const intent: string = luUtterance.intent as string;
|
||||
Utility.addKeyValueToNumberMapSet(
|
||||
smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels,
|
||||
intent,
|
||||
luUtteranceIndex);
|
||||
}
|
||||
{
|
||||
const entities: Array<{
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}> = luUtterance.entities;
|
||||
entities.forEach((entityElement: {
|
||||
"entity": string,
|
||||
"startPos": number,
|
||||
"endPos": number,
|
||||
}) => {
|
||||
const entityType: string = entityElement.entity as string;
|
||||
if (entityType) {
|
||||
Utility.addKeyValueToNumberMapSet(
|
||||
smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels,
|
||||
entityType,
|
||||
luUtteranceIndex);
|
||||
}
|
||||
});
|
||||
}
|
||||
// Utility.debuggingLog(
|
||||
// `i=${i}, ` +
|
||||
// `luUtteranceIndex=${luUtteranceIndex}, ` +
|
||||
// `smallUtteranceIndexSetCoveringAllIntentEntityLabels.size=` +
|
||||
// `${smallUtteranceIndexSetCoveringAllIntentEntityLabels.size}`);
|
||||
}
|
||||
if (toEnsureEachIntentHasOneUtteranceLabel && (intentSet.size < numberIntents)) {
|
||||
continue;
|
||||
}
|
||||
if (toEnsureEachEntityTypeHasOneUtteranceLabel && (entityTypeSet.size < numberEntityTypes)) {
|
||||
continue;
|
||||
}
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
const remainingUtteranceIndexSet: Set<number> =
|
||||
new Set<number>(luUtteranceIndexes.filter((entry: number) => {
|
||||
return !smallUtteranceIndexSetCoveringAllIntentEntityLabels.has(entry);
|
||||
}));
|
||||
return {
|
||||
remainingUtteranceIndexSet,
|
||||
smallUtteranceIndexEntityTypeMapCoveringAllIntentEntityLabels,
|
||||
smallUtteranceIndexIntentMapCoveringAllIntentEntityLabels,
|
||||
smallUtteranceIndexSetCoveringAllIntentEntityLabels,
|
||||
};
|
||||
}
|
||||
}
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryNumberIdGenericArray<T> {
|
||||
[id: number]: T[];
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryNumberIdGenericArrays<T> {
|
||||
[id: number]: T[][];
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryNumberIdGenericSet<T> {
|
||||
[id: number]: Set<T>;
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryNumberIdGenericValue<T> {
|
||||
[id: number]: T;
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryStringIdGenericArray<T> {
|
||||
[id: string]: T[];
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryStringIdGenericArrays<T> {
|
||||
[id: string]: T[][];
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryStringIdGenericSet<T> {
|
||||
[id: string]: Set<T>;
|
||||
}
|
||||
|
|
@ -1,8 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export interface IDictionaryStringIdGenericValue<T> {
|
||||
[id: string]: T;
|
||||
}
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArray } from "./TMapGenericKeyGenericArray";
|
||||
export type TMapAnyKeyGenericArray<T> = TMapGenericKeyGenericArray<any, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArrays } from "./TMapGenericKeyGenericArrays";
|
||||
export type TMapAnyKeyGenericArrays<T> = TMapGenericKeyGenericArrays<any, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericSet } from "./TMapGenericKeyGenericSet";
|
||||
export type TMapAnyKeyGenericSet<T> = TMapGenericKeyGenericSet<any, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericValue } from "./TMapGenericKeyGenericValue";
|
||||
export type TMapAnyKeyGenericValue<T> = TMapGenericKeyGenericValue<any, T>;
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export type TMapGenericKeyGenericArray<I, T> = Map<I, T[]>;
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export type TMapGenericKeyGenericArrays<I, T> = Map<I, T[][]>;
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export type TMapGenericKeyGenericSet<I, T> = Map<I, Set<T>>;
|
||||
|
|
@ -1,6 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export type TMapGenericKeyGenericValue<I, T> = Map<I, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArray } from "./TMapGenericKeyGenericArray";
|
||||
export type TMapNumberKeyGenericArray<T> = TMapGenericKeyGenericArray<number, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArrays } from "./TMapGenericKeyGenericArrays";
|
||||
export type TMapNumberKeyGenericArrays<T> = TMapGenericKeyGenericArrays<number, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericSet } from "./TMapGenericKeyGenericSet";
|
||||
export type TMapNumberKeyGenericSet<T> = TMapGenericKeyGenericSet<number, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericValue } from "./TMapGenericKeyGenericValue";
|
||||
export type TMapNumberKeyGenericValue<T> = TMapGenericKeyGenericValue<number, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArray } from "./TMapGenericKeyGenericArray";
|
||||
export type TMapStringKeyGenericArray<T> = TMapGenericKeyGenericArray<string, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericArrays } from "./TMapGenericKeyGenericArrays";
|
||||
export type TMapStringKeyGenericArrays<T> = TMapGenericKeyGenericArrays<string, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericSet } from "./TMapGenericKeyGenericSet";
|
||||
export type TMapStringKeyGenericSet<T> = TMapGenericKeyGenericSet<string, T>;
|
||||
|
|
@ -1,7 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapGenericKeyGenericValue } from "./TMapGenericKeyGenericValue";
|
||||
export type TMapStringKeyGenericValue<T> = TMapGenericKeyGenericValue<string, T>;
|
||||
|
|
@ -1,97 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
export { ColumnarData } from "./data/ColumnarData";
|
||||
export { Data } from "./data/Data";
|
||||
export { DataUtility } from "./data/DataUtility";
|
||||
export { EntityAnnotatedCorpusData } from "./data/EntityAnnotatedCorpusData";
|
||||
export { LuData } from "./data/LuData";
|
||||
|
||||
export { DictionaryMapUtility } from "./data_structure/DictionaryMapUtility";
|
||||
export { IDictionaryNumberIdGenericArray } from "./data_structure/IDictionaryNumberIdGenericArray";
|
||||
export { IDictionaryNumberIdGenericArrays } from "./data_structure/IDictionaryNumberIdGenericArrays";
|
||||
export { IDictionaryNumberIdGenericSet } from "./data_structure/IDictionaryNumberIdGenericSet";
|
||||
export { IDictionaryNumberIdGenericValue } from "./data_structure/IDictionaryNumberIdGenericValue";
|
||||
export { IDictionaryStringIdGenericArray } from "./data_structure/IDictionaryStringIdGenericArray";
|
||||
export { IDictionaryStringIdGenericArrays } from "./data_structure/IDictionaryStringIdGenericArrays";
|
||||
export { IDictionaryStringIdGenericSet } from "./data_structure/IDictionaryStringIdGenericSet";
|
||||
export { IDictionaryStringIdGenericValue } from "./data_structure/IDictionaryStringIdGenericValue";
|
||||
export { TMapAnyKeyGenericArray } from "./data_structure/TMapAnyKeyGenericArray";
|
||||
export { TMapAnyKeyGenericArrays } from "./data_structure/TMapAnyKeyGenericArrays";
|
||||
export { TMapAnyKeyGenericSet } from "./data_structure/TMapAnyKeyGenericSet";
|
||||
export { TMapAnyKeyGenericValue } from "./data_structure/TMapAnyKeyGenericValue";
|
||||
export { TMapGenericKeyGenericArray } from "./data_structure/TMapGenericKeyGenericArray";
|
||||
export { TMapGenericKeyGenericArrays } from "./data_structure/TMapGenericKeyGenericArrays";
|
||||
export { TMapGenericKeyGenericSet } from "./data_structure/TMapGenericKeyGenericSet";
|
||||
export { TMapGenericKeyGenericValue } from "./data_structure/TMapGenericKeyGenericValue";
|
||||
export { TMapNumberKeyGenericArray } from "./data_structure/TMapNumberKeyGenericArray";
|
||||
export { TMapNumberKeyGenericArrays } from "./data_structure/TMapNumberKeyGenericArrays";
|
||||
export { TMapNumberKeyGenericSet } from "./data_structure/TMapNumberKeyGenericSet";
|
||||
export { TMapNumberKeyGenericValue } from "./data_structure/TMapNumberKeyGenericValue";
|
||||
export { TMapStringKeyGenericArray } from "./data_structure/TMapStringKeyGenericArray";
|
||||
export { TMapStringKeyGenericArrays } from "./data_structure/TMapStringKeyGenericArrays";
|
||||
export { TMapStringKeyGenericSet } from "./data_structure/TMapStringKeyGenericSet";
|
||||
export { TMapStringKeyGenericValue } from "./data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
export { mainConfusionMatrix } from "./mathematics/confusion_matrix/AppConfusionMatrix";
|
||||
export { mainConfusionMatrixFunction } from "./mathematics/confusion_matrix/AppConfusionMatrix";
|
||||
export { BinaryConfusionMatrix } from "./mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
export { ConfusionMatrix } from "./mathematics/confusion_matrix/ConfusionMatrix";
|
||||
export { ConfusionMatrixBase } from "./mathematics/confusion_matrix/ConfusionMatrixBase";
|
||||
export { IConfusionMatrix } from "./mathematics/confusion_matrix/IConfusionMatrix";
|
||||
export { IMultiLabelConfusionMatrix } from "./mathematics/confusion_matrix/IMultiLabelConfusionMatrix";
|
||||
export { ISingleLabelConfusionMatrix } from "./mathematics/confusion_matrix/ISingleLabelConfusionMatrix";
|
||||
export { MultiLabelConfusionMatrix } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrix";
|
||||
export { MultiLabelConfusionMatrixBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixBase";
|
||||
export { MultiLabelConfusionMatrixSubset } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixSubset";
|
||||
export { MultiLabelConfusionMatrixWithBinaryArrayBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixWithBinaryArrayBase";
|
||||
export { MultiLabelConfusionMatrixWithBinaryBase } from "./mathematics/confusion_matrix/MultiLabelConfusionMatrixWithBinaryBase";
|
||||
|
||||
export { IMathematicsHelper } from "./mathematics/mathematics_helper/IMathematicsHelper";
|
||||
export { MathematicsHelper } from "./mathematics/mathematics_helper/MathematicsHelper";
|
||||
|
||||
export { AbstractBaseBootstrapSampler } from "./mathematics/sampler/AbstractBaseBootstrapSampler";
|
||||
export { AbstractBaseReservoirSampler } from "./mathematics/sampler/AbstractBaseReservoirSampler";
|
||||
export { AbstractBaseSampler } from "./mathematics/sampler/AbstractBaseSampler";
|
||||
export { BootstrapSampler } from "./mathematics/sampler/BootstrapSampler";
|
||||
export { BootstrapSamplerDistribution } from "./mathematics/sampler/BootstrapSamplerDistribution";
|
||||
export { BootstrapSamplerKeyMap } from "./mathematics/sampler/BootstrapSamplerKeyMap";
|
||||
export { BootstrapSamplerKeyMapDistribution } from "./mathematics/sampler/BootstrapSamplerKeyMapDistribution";
|
||||
export { ReservoirArraySampler } from "./mathematics/sampler/ReservoirArraySampler";
|
||||
export { ReservoirSampler } from "./mathematics/sampler/ReservoirSampler";
|
||||
export { ReservoirSamplerKeyMap } from "./mathematics/sampler/ReservoirSamplerKeyMap";
|
||||
|
||||
export { mainCrossValidatorWithColumnarContent } from "./model/evaluation/cross_validation/AppCrossValidator";
|
||||
export { mainCrossValidatorWithLuContent } from "./model/evaluation/cross_validation/AppCrossValidator";
|
||||
export { mainCrossValidator } from "./model/evaluation/cross_validation/AppCrossValidator";
|
||||
export { CrossValidator } from "./model/evaluation/cross_validation/CrossValidator";
|
||||
|
||||
export { mainPredictor } from "./model/evaluation/predict/AppPredictor";
|
||||
export { Predictor } from "./model/evaluation/predict/Predictor";
|
||||
|
||||
export { mainDataProfileReporter } from "./model/evaluation/report/AppDataProfileReporter";
|
||||
export { DataProfileReporter } from "./model/evaluation/report/DataProfileReporter";
|
||||
export { mainModelMetaDataProfileReporter } from "./model/evaluation/report/AppModelMetaDataProfileReporter";
|
||||
export { ModelMetaDataProfileReporter } from "./model/evaluation/report/ModelMetaDataProfileReporter";
|
||||
export { mainThresholdReporter } from "./model/evaluation/report/AppThresholdReporter";
|
||||
export { ThresholdReporter } from "./model/evaluation/report/ThresholdReporter";
|
||||
|
||||
export { mainTester } from "./model/evaluation/test/AppTester";
|
||||
export { Tester } from "./model/evaluation/test/Tester";
|
||||
|
||||
export { ISparseTextFeaturizer } from "./model/language_understanding/featurizer/ISparseTextFeaturizer";
|
||||
export { ITextFeaturizer } from "./model/language_understanding/featurizer/ITextFeaturizer";
|
||||
export { NgramSubwordFeaturizer } from "./model/language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
export { AppAutoActiveLearner } from "./model/supervised/classifier/auto_active_learning/AppAutoActiveLearner";
|
||||
export { AutoActiveLearner } from "./model/supervised/classifier/auto_active_learning/AutoActiveLearner";
|
||||
|
||||
export { AppSoftmaxRegressionSparse } from "./model/supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
|
||||
export { SoftmaxRegressionSparse } from "./model/supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
export { Utility } from "./utility/Utility";
|
||||
export { ListArrayUtility } from "./utility/ListArrayUtility";
|
||||
|
||||
export default {};
|
||||
|
|
@ -1,32 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionBinaryConfusionMatrix(): void {
|
||||
const cell11: number = 1;
|
||||
const row1: number = 2;
|
||||
const column1: number = 2;
|
||||
const total: number = 4;
|
||||
const binaryConfusionMatrix = new BinaryConfusionMatrix(
|
||||
total,
|
||||
cell11,
|
||||
row1,
|
||||
column1);
|
||||
Utility.debuggingLog(
|
||||
"getTotal() = " + binaryConfusionMatrix.getTotal());
|
||||
Utility.debuggingLog(
|
||||
"getPrecision() = " + binaryConfusionMatrix.getPrecision());
|
||||
Utility.debuggingLog(
|
||||
"getRecall() = " + binaryConfusionMatrix.getRecall());
|
||||
Utility.debuggingLog(
|
||||
"getF1Score() = " + binaryConfusionMatrix.getF1Score());
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionBinaryConfusionMatrix();
|
||||
}
|
||||
|
|
@ -1,380 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
import { ConfusionMatrix } from "./ConfusionMatrix";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function mainConfusionMatrixFunction(
|
||||
scoreFilename: string,
|
||||
labelFilename: string,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
identifierColumnIndex: number,
|
||||
scoreColumnBeginIndex: number,
|
||||
predictedLabelColumnIndex: number,
|
||||
revisedTextColumnIndex: number,
|
||||
lineIndexToStart: number): {
|
||||
"labels": string[],
|
||||
"labelMap": { [id: string]: number },
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"confusionMatrix": ConfusionMatrix } {
|
||||
// -----------------------------------------------------------------------
|
||||
if (Utility.isEmptyString(scoreFilename)) {
|
||||
throw new Error("scoreFilename is empty");
|
||||
}
|
||||
if (Utility.isEmptyString(labelFilename)) {
|
||||
throw new Error("labelFilename is empty");
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
let labels: string[] = [];
|
||||
let labelMap: { [id: string]: number } = {};
|
||||
if (!Utility.isEmptyString(labelFilename)) {
|
||||
const labelsAndLabelMap: { "stringArray": string[], "stringMap": { [id: string]: number } } =
|
||||
DictionaryMapUtility.buildStringIdNumberValueDictionaryFromUniqueStringArrayFile(labelFilename);
|
||||
labels = labelsAndLabelMap.stringArray;
|
||||
labelMap = labelsAndLabelMap.stringMap;
|
||||
}
|
||||
if (Utility.isEmptyStringArray(labels)) {
|
||||
throw new Error("labels is empty");
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const confusionMatrix = new ConfusionMatrix(labels, labelMap);
|
||||
// -----------------------------------------------------------------------
|
||||
const scoreDataStructure: {
|
||||
"labels": string[],
|
||||
"texts": string[],
|
||||
"weights": number[],
|
||||
"identifiers": string[],
|
||||
"scoreArrays": number[][],
|
||||
"predictedLabels": string[],
|
||||
"revisedTexts": string[] } =
|
||||
Utility.loadLabelTextScoreFile(
|
||||
scoreFilename,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
scoreColumnBeginIndex,
|
||||
labels.length,
|
||||
identifierColumnIndex,
|
||||
predictedLabelColumnIndex,
|
||||
revisedTextColumnIndex,
|
||||
lineIndexToStart);
|
||||
const numberInstances: number = scoreDataStructure.labels.length;
|
||||
let numberMatches: number = 0;
|
||||
for (let i = 0; i < numberInstances; i++) {
|
||||
const label: string = scoreDataStructure.labels[i];
|
||||
// const text: string = scoreDataStructure.texts[i];
|
||||
// const weight: number = scoreDataStructure.weights[i];
|
||||
// const identifier: string = scoreDataStructure.identifiers[i];
|
||||
// const scoreArray: number[] = scoreDataStructure.scoreArrays[i];
|
||||
// const labelId: number = labelMap[label];
|
||||
// const weight: number = scoreDataStructure.weights[i];
|
||||
const predictedLabel: string = scoreDataStructure.predictedLabels[i];
|
||||
// const revisedText: string = scoreDataStructure.revisedTexts[i];
|
||||
// Utility.debuggingLog(
|
||||
// "label=" + label);
|
||||
// Utility.debuggingLog(
|
||||
// "predictedLabel=" + predictedLabel);
|
||||
if (label === predictedLabel) {
|
||||
numberMatches++;
|
||||
}
|
||||
confusionMatrix.addInstanceByLabel(label, predictedLabel);
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
"numberMatches=" + numberMatches);
|
||||
Utility.debuggingLog(
|
||||
"numberInstances=" + numberInstances);
|
||||
Utility.debuggingLog(
|
||||
"accuracy=" + numberMatches / numberInstances);
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
"labels=" + confusionMatrix.getLabels());
|
||||
Utility.debuggingLog(
|
||||
Utility.jsonStringify(confusionMatrix.getLabelMap()));
|
||||
Utility.debuggingLog(
|
||||
"rows=" + confusionMatrix.getConfusionMatrixRows());
|
||||
Utility.debuggingLog(
|
||||
"columns=" + confusionMatrix.getConfusionMatrixColumns());
|
||||
Utility.debuggingLog(
|
||||
"total=" + confusionMatrix.getConfusionMatrixTotal());
|
||||
const binaryConfusionMatrices: BinaryConfusionMatrix[] = confusionMatrix.getBinaryConfusionMatrices();
|
||||
const confusionMatrixLabels: string[] = confusionMatrix.getLabels();
|
||||
for (let i = 0; i < binaryConfusionMatrices.length; i++) {
|
||||
const binaryConfusionMatrix = binaryConfusionMatrices[i];
|
||||
const label: string = confusionMatrixLabels[i];
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", precision = " + binaryConfusionMatrix.getPrecision());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", recall = " + binaryConfusionMatrix.getRecall());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", F1 = " + binaryConfusionMatrix.getF1Score());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", support = " + binaryConfusionMatrix.getSupport());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", total = " + binaryConfusionMatrix.getTotal());
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
"micro-average metrics = " + confusionMatrix.getMicroAverageMetrics());
|
||||
Utility.debuggingLog(
|
||||
"macro-average metrics = " + confusionMatrix.getMacroAverageMetrics());
|
||||
Utility.debuggingLog(
|
||||
"weighted-macro-average metrics = " + confusionMatrix.getWeightedMacroAverageMetrics());
|
||||
Utility.debuggingLog(
|
||||
"labels=" + confusionMatrix.getLabels());
|
||||
// -----------------------------------------------------------------------
|
||||
return {
|
||||
labels,
|
||||
labelMap,
|
||||
binaryConfusionMatrices,
|
||||
confusionMatrix };
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
export function mainConfusionMatrix(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppConfusionMatrix",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--scoreFilename"],
|
||||
{
|
||||
help: "an input score file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-si", "--scoreColumnBeginIndex"],
|
||||
{
|
||||
help: "score column begin index",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-l", "--labelFilename"],
|
||||
{
|
||||
help: "an input label file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ii", "--identifierColumnIndex"],
|
||||
{
|
||||
defaultValue: 2,
|
||||
help: "identifier column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-pli", "--predictedLabelColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "predicted label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-rti", "--revisedTextColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "revised text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--lineIndexToStart"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const scoreFilename: string =
|
||||
args.scoreFilename;
|
||||
if (!Utility.exists(scoreFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input score file ${scoreFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(scoreFilename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`scoreFilename=${scoreFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const labelFilename: string =
|
||||
args.labelFilename;
|
||||
Utility.debuggingLog(
|
||||
`labelFilename=${labelFilename}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const identifierColumnIndex: number = +args.identifierColumnIndex;
|
||||
const scoreColumnBeginIndex: number = +args.scoreColumnBeginIndex;
|
||||
const predictedLabelColumnIndex: number = +args.predictedLabelColumnIndex;
|
||||
const revisedTextColumnIndex: number = +args.revisedTextColumnIndex;
|
||||
const lineIndexToStart: number = +args.lineIndexToStart;
|
||||
Utility.debuggingLog(
|
||||
`labelColumnIndex=${labelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`textColumnIndex=${textColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`weightColumnIndex=${weightColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`identifierColumnIndex=${identifierColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`scoreColumnBeginIndex=${scoreColumnBeginIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`predictedLabelColumnIndex=${predictedLabelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`revisedTextColumnIndex=${revisedTextColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`lineIndexToStart=${lineIndexToStart}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const mainConfusionMatrixFunctionResult: {
|
||||
"labels": string[],
|
||||
"labelMap": { [id: string]: number },
|
||||
"confusionMatrix": ConfusionMatrix } = mainConfusionMatrixFunction(
|
||||
scoreFilename,
|
||||
labelFilename,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
identifierColumnIndex,
|
||||
scoreColumnBeginIndex,
|
||||
predictedLabelColumnIndex,
|
||||
revisedTextColumnIndex,
|
||||
lineIndexToStart);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
export function exampleFunctionConfusionMatrix(): void {
|
||||
const labels: string[] = [ "label0", "label1", "label2" ];
|
||||
const labelMap: { [id: string]: number } = { };
|
||||
labelMap.label0 = 0;
|
||||
labelMap.label1 = 1;
|
||||
labelMap.label2 = 2;
|
||||
const confusionMatrix = new ConfusionMatrix(labels, labelMap);
|
||||
confusionMatrix.addInstanceByLabel("label0", "label0");
|
||||
confusionMatrix.addInstanceByLabel("label0", "label1");
|
||||
confusionMatrix.addInstanceByLabel("label0", "label2");
|
||||
confusionMatrix.addInstanceByLabel("label1", "label0");
|
||||
confusionMatrix.addInstanceByLabel("label1", "label1");
|
||||
confusionMatrix.addInstanceByLabel("label1", "label2");
|
||||
confusionMatrix.addInstanceByLabel("label2", "label0");
|
||||
confusionMatrix.addInstanceByLabel("label2", "label1");
|
||||
confusionMatrix.addInstanceByLabel("label2", "label2");
|
||||
Utility.debuggingLog(
|
||||
"labels=" + confusionMatrix.getLabels());
|
||||
Utility.debuggingLog(
|
||||
Utility.jsonStringify(confusionMatrix.getLabelMap()));
|
||||
Utility.debuggingLog(
|
||||
"rows=" + confusionMatrix.getConfusionMatrixRows());
|
||||
Utility.debuggingLog(
|
||||
"columns=" + confusionMatrix.getConfusionMatrixColumns());
|
||||
Utility.debuggingLog(
|
||||
"total=" + confusionMatrix.getConfusionMatrixTotal());
|
||||
const binaryConfusionMatrices = confusionMatrix.getBinaryConfusionMatrices();
|
||||
const confusionMatrixLabels: string[] = confusionMatrix.getLabels();
|
||||
for (let i = 0; i < binaryConfusionMatrices.length; i++) {
|
||||
const binaryConfusionMatrix = binaryConfusionMatrices[i];
|
||||
const label: string = confusionMatrixLabels[i];
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", precision = " + binaryConfusionMatrix.getPrecision());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", recall = " + binaryConfusionMatrix.getRecall());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", F1 = " + binaryConfusionMatrix.getF1Score());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", support = " + binaryConfusionMatrix.getSupport());
|
||||
Utility.debuggingLog(
|
||||
label + ":" + i + ", total = " + binaryConfusionMatrix.getTotal());
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
"micro-average metrics = " + confusionMatrix.getMicroAverageMetrics());
|
||||
Utility.debuggingLog(
|
||||
"macro-average metrics = " + confusionMatrix.getMacroAverageMetrics());
|
||||
Utility.debuggingLog(
|
||||
"weighted-macro-average metrics = " + confusionMatrix.getWeightedMacroAverageMetrics());
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainConfusionMatrix();
|
||||
// ---- exampleFunctionConfusionMatrix();
|
||||
}
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,110 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ISingleLabelConfusionMatrix } from "./ISingleLabelConfusionMatrix";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class ConfusionMatrix
|
||||
extends ConfusionMatrixBase
|
||||
implements ISingleLabelConfusionMatrix {
|
||||
|
||||
protected confusionMatrix: number[][] = [];
|
||||
protected confusionMatrixRows: number[] = [];
|
||||
protected confusionMatrixColumns: number[] = [];
|
||||
protected confusionMatrixTotal: number = 0;
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
this.reset();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
this.confusionMatrix = [];
|
||||
this.confusionMatrixRows = [];
|
||||
this.confusionMatrixColumns = [];
|
||||
this.confusionMatrixTotal = 0;
|
||||
const numberLabels: number = this.getNumberLabels();
|
||||
for (let row: number = 0; row < numberLabels; row++) {
|
||||
this.confusionMatrixRows[row] = 0;
|
||||
this.confusionMatrixColumns[row] = 0;
|
||||
this.confusionMatrix[row] = [];
|
||||
for (let column: number = 0; column < numberLabels; column++) {
|
||||
this.confusionMatrix[row][column] = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public addFrom(other: ConfusionMatrix): void {
|
||||
Utility.validateStringArrayPairEquality(this.labels, other.labels);
|
||||
DictionaryMapUtility.validateStringIdNumberValueDictionaryPair(this.labelMap, other.labelMap);
|
||||
this.confusionMatrixTotal += other.confusionMatrixTotal;
|
||||
const numberLabels: number = this.getNumberLabels();
|
||||
for (let row: number = 0; row < numberLabels; row++) {
|
||||
this.confusionMatrixRows[row] += other.confusionMatrixRows[row];
|
||||
this.confusionMatrixColumns[row] += other.confusionMatrixColumns[row];
|
||||
for (let column: number = 0; column < numberLabels; column++) {
|
||||
this.confusionMatrix[row][column] += other.confusionMatrix[row][column];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public getConfusionMatrixRows(): number[] {
|
||||
return this.confusionMatrixRows;
|
||||
}
|
||||
public getConfusionMatrixColumns(): number[] {
|
||||
return this.confusionMatrixColumns;
|
||||
}
|
||||
public getConfusionMatrixTotal(): number {
|
||||
return this.confusionMatrixTotal;
|
||||
}
|
||||
public getConfusionMatrix(): number[][] {
|
||||
return this.confusionMatrix;
|
||||
}
|
||||
|
||||
public addInstanceByLabelIndex(
|
||||
groundTrueLabelId: number,
|
||||
predictedLabelId: number,
|
||||
value: number = 1): void {
|
||||
this.validateLabelId(groundTrueLabelId);
|
||||
this.validateLabelId(predictedLabelId);
|
||||
this.confusionMatrixRows[groundTrueLabelId] += value;
|
||||
this.confusionMatrixColumns[predictedLabelId] += value;
|
||||
this.confusionMatrixTotal += value;
|
||||
this.confusionMatrix[groundTrueLabelId][predictedLabelId] += value;
|
||||
}
|
||||
|
||||
public addInstanceByLabel(
|
||||
groundTrueLabel: string,
|
||||
predictedLabel: string,
|
||||
value: number = 1): void {
|
||||
this.validateLabel(groundTrueLabel);
|
||||
this.validateLabel(predictedLabel);
|
||||
const groundTrueLabelId: number = this.labelMap[groundTrueLabel];
|
||||
const predictedLabelId: number = this.labelMap[predictedLabel];
|
||||
this.addInstanceByLabelIndex(groundTrueLabelId, predictedLabelId, value);
|
||||
}
|
||||
|
||||
public getBinaryConfusionMatrices(): BinaryConfusionMatrix[] {
|
||||
const numberLabels: number = this.getNumberLabels();
|
||||
const binaryConfusionMatrices: BinaryConfusionMatrix[] =
|
||||
new Array<BinaryConfusionMatrix>(numberLabels);
|
||||
for (let labelId: number = 0; labelId < numberLabels; labelId++) {
|
||||
binaryConfusionMatrices[labelId] =
|
||||
new BinaryConfusionMatrix(
|
||||
this.confusionMatrixTotal,
|
||||
this.confusionMatrix[labelId][labelId],
|
||||
this.confusionMatrixRows[labelId],
|
||||
this.confusionMatrixColumns[labelId]);
|
||||
}
|
||||
return binaryConfusionMatrices;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,464 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IConfusionMatrix } from "./IConfusionMatrix";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class ConfusionMatrixBase implements IConfusionMatrix {
|
||||
|
||||
protected labels: string[] = [];
|
||||
protected labelMap: { [id: string]: number } = {};
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
this.resetLabelsAndMap(labels, labelMap);
|
||||
}
|
||||
|
||||
public abstract reset(): void;
|
||||
|
||||
public generateConfusionMatrixMetricStructure(): {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } } {
|
||||
const confusionMatrix: IConfusionMatrix = this;
|
||||
const crossValidationBinaryConfusionMatrix: BinaryConfusionMatrix[] =
|
||||
confusionMatrix.getBinaryConfusionMatrices();
|
||||
const labelMap: { [id: string]: number } =
|
||||
confusionMatrix.getLabelMap();
|
||||
const labelBinaryConfusionMatrixBasicMetricMap: { [id: string]: { [id: string]: number } } =
|
||||
Object.entries(labelMap).reduce(
|
||||
(accumulant: { [id: string]: { [id: string]: number } }, [id, value]) =>
|
||||
({...accumulant, [id]: crossValidationBinaryConfusionMatrix[value].getBasicMetrics()}), {});
|
||||
const labelBinaryConfusionMatrixMap: { [id: string]: BinaryConfusionMatrix } =
|
||||
Object.entries(labelMap).reduce(
|
||||
(accumulant: { [id: string]: BinaryConfusionMatrix }, [id, value]) =>
|
||||
({...accumulant, [id]: crossValidationBinaryConfusionMatrix[value]}), {});
|
||||
const binaryConfusionMatrices: BinaryConfusionMatrix[] =
|
||||
confusionMatrix.getBinaryConfusionMatrices();
|
||||
const microAverageMetricArray: {
|
||||
"averagePrecisionRecallF1Accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"total": number } =
|
||||
confusionMatrix.getMicroAverageMetrics(binaryConfusionMatrices);
|
||||
const accuracy: number =
|
||||
microAverageMetricArray.averagePrecisionRecallF1Accuracy;
|
||||
const truePositives: number =
|
||||
microAverageMetricArray.truePositives;
|
||||
const falsePositives: number =
|
||||
microAverageMetricArray.falsePositives;
|
||||
const falseNegatives: number =
|
||||
microAverageMetricArray.falseNegatives;
|
||||
const supportMicroAverage: number =
|
||||
microAverageMetricArray.total;
|
||||
const microAverageMetrics: {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number } = {
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"accuracy": accuracy,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"truePositives": truePositives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"falsePositives": falsePositives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"falseNegatives": falseNegatives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"support": supportMicroAverage };
|
||||
const macroAverageMetricArray: {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"total": number } =
|
||||
confusionMatrix.getMacroAverageMetrics(binaryConfusionMatrices);
|
||||
const averagePrecision: number =
|
||||
macroAverageMetricArray.averagePrecision;
|
||||
const averageRecall: number =
|
||||
macroAverageMetricArray.averageRecall;
|
||||
const averageF1Score: number =
|
||||
macroAverageMetricArray.averageF1Score;
|
||||
const averageAccuracy: number =
|
||||
macroAverageMetricArray.averageAccuracy;
|
||||
const averageTruePositives: number =
|
||||
macroAverageMetricArray.averageTruePositives;
|
||||
const averageFalsePositives: number =
|
||||
macroAverageMetricArray.averageFalsePositives;
|
||||
const averageTrueNegatives: number =
|
||||
macroAverageMetricArray.averageTrueNegatives;
|
||||
const averageFalseNegatives: number =
|
||||
macroAverageMetricArray.averageFalseNegatives;
|
||||
const averageSupport: number =
|
||||
macroAverageMetricArray.averageSupport;
|
||||
const supportMacroAverage: number =
|
||||
macroAverageMetricArray.total;
|
||||
const macroAverageMetrics: {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number } = {
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averagePrecision": averagePrecision,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageRecall": averageRecall,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageF1Score": averageF1Score,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageAccuracy": averageAccuracy,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageTruePositives": averageTruePositives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageFalsePositives": averageFalsePositives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageTrueNegatives": averageTrueNegatives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageFalseNegatives": averageFalseNegatives,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"averageSupport": averageSupport,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"support": supportMacroAverage };
|
||||
const weightedMacroAverageMetricArray: {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number } =
|
||||
confusionMatrix.getWeightedMacroAverageMetrics(binaryConfusionMatrices);
|
||||
const weightedAveragePrecision: number =
|
||||
weightedMacroAverageMetricArray.averagePrecision;
|
||||
const weightedAverageRecall: number =
|
||||
weightedMacroAverageMetricArray.averageRecall;
|
||||
const weightedAverageF1Score: number =
|
||||
weightedMacroAverageMetricArray.averageF1Score;
|
||||
const weightedAverageAccuracy: number =
|
||||
weightedMacroAverageMetricArray.averageAccuracy;
|
||||
const weightedAverageSupport: number =
|
||||
weightedMacroAverageMetricArray.averageSupport;
|
||||
const supportWeightedMacroAverage: number =
|
||||
weightedMacroAverageMetricArray.total;
|
||||
const weightedMacroAverageMetrics: {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } = {
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"weightedAveragePrecision": weightedAveragePrecision,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"weightedAverageRecall": weightedAverageRecall,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"weightedAverageF1Score": weightedAverageF1Score,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"weightedAverageAccuracy": weightedAverageAccuracy,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"weightedAverageSupport": weightedAverageSupport,
|
||||
// tslint:disable-next-line: object-literal-key-quotes
|
||||
"support": supportWeightedMacroAverage };
|
||||
const confusionMatrixMetricStructure: {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } } = {
|
||||
confusionMatrix,
|
||||
labelBinaryConfusionMatrixBasicMetricMap,
|
||||
labelBinaryConfusionMatrixMap,
|
||||
microAverageMetrics,
|
||||
macroAverageMetrics,
|
||||
weightedMacroAverageMetrics,
|
||||
};
|
||||
return confusionMatrixMetricStructure;
|
||||
}
|
||||
|
||||
public getNumberLabels(): number {
|
||||
return this.labels.length;
|
||||
}
|
||||
public getLabels(): string[] {
|
||||
return this.labels;
|
||||
}
|
||||
public getLabelMap(): { [id: string]: number } {
|
||||
return this.labelMap;
|
||||
}
|
||||
|
||||
public abstract getBinaryConfusionMatrices(): BinaryConfusionMatrix[];
|
||||
|
||||
public getTotal(binaryConfusionMatrices: BinaryConfusionMatrix[] = []): number {
|
||||
if (Utility.isEmptyArray(binaryConfusionMatrices)) {
|
||||
binaryConfusionMatrices =
|
||||
this.getBinaryConfusionMatrices();
|
||||
}
|
||||
const total: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getPositives(), 0);
|
||||
return total;
|
||||
}
|
||||
|
||||
public getMicroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[] = []): {
|
||||
"averagePrecisionRecallF1Accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"total": number } {
|
||||
if (Utility.isEmptyArray(binaryConfusionMatrices)) {
|
||||
binaryConfusionMatrices =
|
||||
this.getBinaryConfusionMatrices();
|
||||
}
|
||||
// ---- NOTE-use-getTotal() ---- const total: number =
|
||||
// ---- NOTE-use-getTotal() ---- binaryConfusionMatrices.reduce(
|
||||
// ---- NOTE-use-getTotal() ---- (accumulation, entry) => accumulation + entry.getPositives(), 0);
|
||||
const total: number =
|
||||
this.getTotal(binaryConfusionMatrices);
|
||||
const truePositives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getTruePositives(), 0);
|
||||
const falsePositives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getFalsePositives(), 0);
|
||||
const falseNegatives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getFalseNegatives(), 0);
|
||||
const averagePrecisionRecallF1Accuracy: number =
|
||||
truePositives / total;
|
||||
const microAverageMetrics: {
|
||||
"averagePrecisionRecallF1Accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"total": number } = {
|
||||
averagePrecisionRecallF1Accuracy,
|
||||
truePositives,
|
||||
falsePositives,
|
||||
falseNegatives,
|
||||
total };
|
||||
return microAverageMetrics;
|
||||
}
|
||||
|
||||
public getMacroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[] = []): {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number } {
|
||||
if (Utility.isEmptyArray(binaryConfusionMatrices)) {
|
||||
binaryConfusionMatrices =
|
||||
this.getBinaryConfusionMatrices();
|
||||
}
|
||||
const numberLabels: number =
|
||||
binaryConfusionMatrices.length;
|
||||
const averagePrecision: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getPrecision(), 0) / numberLabels;
|
||||
const averageRecall: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getRecall(), 0) / numberLabels;
|
||||
const averageF1Score: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getF1Score(), 0) / numberLabels;
|
||||
const averageTruePositives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getTruePositives(), 0) / numberLabels;
|
||||
const averageFalsePositives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getFalsePositives(), 0) / numberLabels;
|
||||
const averageTrueNegatives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getTrueNegatives(), 0) / numberLabels;
|
||||
const averageFalseNegatives: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getFalseNegatives(), 0) / numberLabels;
|
||||
const averageAccuracy: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getAccuracy(), 0) / numberLabels;
|
||||
const averageSupport: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getSupport(), 0) / numberLabels;
|
||||
// ---- NOTE-use-getTotal() ---- const total: number =
|
||||
// ---- NOTE-use-getTotal() ---- binaryConfusionMatrices.reduce(
|
||||
// ---- NOTE-use-getTotal() ---- (accumulation, entry) => accumulation + entry.getPositives(), 0);
|
||||
const total: number =
|
||||
this.getTotal(binaryConfusionMatrices);
|
||||
const macroAverageMetrics: {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number } = {
|
||||
averagePrecision,
|
||||
averageRecall,
|
||||
averageF1Score,
|
||||
averageTruePositives,
|
||||
averageFalsePositives,
|
||||
averageTrueNegatives,
|
||||
averageFalseNegatives,
|
||||
averageAccuracy,
|
||||
averageSupport,
|
||||
total };
|
||||
return macroAverageMetrics;
|
||||
}
|
||||
|
||||
public getWeightedMacroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[] = []): {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number } {
|
||||
if (Utility.isEmptyArray(binaryConfusionMatrices)) {
|
||||
binaryConfusionMatrices =
|
||||
this.getBinaryConfusionMatrices();
|
||||
}
|
||||
// ---- NOTE-use-getTotal() ---- const total: number =
|
||||
// ---- NOTE-use-getTotal() ---- binaryConfusionMatrices.reduce(
|
||||
// ---- NOTE-use-getTotal() ---- (accumulation, entry) => accumulation + entry.getPositives(), 0);
|
||||
const total: number =
|
||||
this.getTotal(binaryConfusionMatrices);
|
||||
const averagePrecision: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getPrecision() * entry.getPositives(), 0) / total;
|
||||
const averageRecall: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getRecall() * entry.getPositives(), 0) / total;
|
||||
const averageF1Score: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getF1Score() * entry.getPositives(), 0) / total;
|
||||
const averageAccuracy: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getAccuracy() * entry.getPositives(), 0) / total;
|
||||
const averageSupport: number =
|
||||
binaryConfusionMatrices.reduce(
|
||||
(accumulation, entry) => accumulation + entry.getSupport() * entry.getPositives(), 0) / total;
|
||||
const macroAverageMetrics: {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number } = {
|
||||
averagePrecision,
|
||||
averageRecall,
|
||||
averageF1Score,
|
||||
averageAccuracy,
|
||||
averageSupport,
|
||||
total };
|
||||
return macroAverageMetrics;
|
||||
}
|
||||
|
||||
public validateLabelId(
|
||||
labelId: number,
|
||||
throwIfNotLegal: boolean = true): boolean {
|
||||
if (labelId < 0) {
|
||||
if (throwIfNotLegal) {
|
||||
Utility.debuggingThrow(
|
||||
`labelId=${labelId}, small than 0`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
if (labelId >= this.getNumberLabels()) {
|
||||
if (throwIfNotLegal) {
|
||||
Utility.debuggingThrow(
|
||||
`labelId=${labelId}, greater or equal to number of labels, ${this.getNumberLabels()}`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public validateLabel(
|
||||
label: string,
|
||||
throwIfNotLegal: boolean = true): boolean {
|
||||
if (!(label in this.getLabelMap())) {
|
||||
if (throwIfNotLegal) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}, not int the label map=${Utility.jsonStringify(this.getLabelMap())}`);
|
||||
}
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
protected resetLabelsAndMap(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }): void {
|
||||
DictionaryMapUtility.validateStringArrayAndStringIdNumberValueDictionary(labels, labelMap);
|
||||
this.labels = labels;
|
||||
this.labelMap = labelMap;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
export interface IConfusionMatrix {
|
||||
|
||||
reset(): void;
|
||||
|
||||
generateConfusionMatrixMetricStructure(): {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } };
|
||||
|
||||
getNumberLabels(): number;
|
||||
getLabels(): string[];
|
||||
getLabelMap(): { [id: string]: number };
|
||||
|
||||
getBinaryConfusionMatrices(): BinaryConfusionMatrix[];
|
||||
|
||||
getTotal(binaryConfusionMatrices: BinaryConfusionMatrix[]): number;
|
||||
|
||||
getMicroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[]): {
|
||||
"averagePrecisionRecallF1Accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"total": number };
|
||||
|
||||
getMacroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[]): {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number };
|
||||
|
||||
getWeightedMacroAverageMetrics(binaryConfusionMatrices: BinaryConfusionMatrix[]): {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageSupport": number,
|
||||
"total": number };
|
||||
|
||||
validateLabelId(
|
||||
labelId: number,
|
||||
throwIfNotLegal: boolean): boolean;
|
||||
validateLabel(
|
||||
label: string,
|
||||
throwIfNotLegal: boolean): boolean;
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IConfusionMatrix } from "./IConfusionMatrix";
|
||||
|
||||
export interface IMultiLabelConfusionMatrix extends IConfusionMatrix {
|
||||
|
||||
addInstanceByLabelIndexes(
|
||||
groundTrueLabelIds: number[],
|
||||
predictedLabelIds: number[],
|
||||
value: number): void;
|
||||
addInstanceByLabels(
|
||||
groundTrueLabels: string[],
|
||||
predictedLabels: string[],
|
||||
value: number): void;
|
||||
|
||||
validateLabelIds(
|
||||
labelIds: number[],
|
||||
throwIfNotLegal: boolean): boolean;
|
||||
validateLabels(
|
||||
labels: string[],
|
||||
throwIfNotLegal: boolean): boolean;
|
||||
}
|
||||
|
|
@ -1,18 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IConfusionMatrix } from "./IConfusionMatrix";
|
||||
|
||||
export interface ISingleLabelConfusionMatrix extends IConfusionMatrix {
|
||||
|
||||
addInstanceByLabelIndex(
|
||||
groundTrueLabelId: number,
|
||||
predictedLabelId: number,
|
||||
value: number): void;
|
||||
addInstanceByLabel(
|
||||
groundTrueLabel: string,
|
||||
predictedLabel: string,
|
||||
value: number): void;
|
||||
}
|
||||
|
|
@ -1,66 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// import { IMultiLabelConfusionMatrix } from "./IMultiLabelConfusionMatrix";
|
||||
import { MultiLabelConfusionMatrixWithBinaryArrayBase } from "./MultiLabelConfusionMatrixWithBinaryArrayBase";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class MultiLabelConfusionMatrix
|
||||
extends MultiLabelConfusionMatrixWithBinaryArrayBase {
|
||||
|
||||
public static evaluateMultiLabelPrediction(groundTruths: any[], predictions: any[]): number {
|
||||
if (predictions.length <= 0) {
|
||||
if (groundTruths.length <= 0) {
|
||||
return 3; // ---- NOTE ---- 3 for true negative as there is no prediction on an empty ground-truth set.
|
||||
}
|
||||
return 1; // ---- NOTE ---- 1 for false negative as there is no prediction on a non-empty ground-truth set.
|
||||
}
|
||||
for (const prediction of predictions) {
|
||||
if (!groundTruths.includes(prediction)) {
|
||||
return 2; // ---- NOTE ---- 2 for false positive as there is a prediction not in the ground-truth set.
|
||||
}
|
||||
}
|
||||
return 0; // ---- NOTE ---- 0 for true positive as every prediction is in the ground-trueh set.
|
||||
}
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
}
|
||||
|
||||
public addInstanceByLabelIndexes(
|
||||
groundTrueLabelIds: number[],
|
||||
predictedLabelIds: number[],
|
||||
value: number = 1): void {
|
||||
this.validateLabelIds(groundTrueLabelIds);
|
||||
this.validateLabelIds(predictedLabelIds);
|
||||
for (let labelId: number = 0; labelId < this.getNumberLabels(); labelId++) {
|
||||
const isInGroundTruthLabelIds: boolean = this.isLabelIdInArray(groundTrueLabelIds, labelId);
|
||||
const isInPredictedLabelIds: boolean = this.isLabelIdInArray(predictedLabelIds, labelId);
|
||||
if (isInGroundTruthLabelIds) {
|
||||
if (isInPredictedLabelIds) {
|
||||
this.getBinaryConfusionMatrices()[labelId].addToTruePositives(value, false);
|
||||
} else {
|
||||
this.getBinaryConfusionMatrices()[labelId].addToFalseNegatives(value, false);
|
||||
}
|
||||
} else {
|
||||
if (isInPredictedLabelIds) {
|
||||
this.getBinaryConfusionMatrices()[labelId].addToFalsePositives(value, false);
|
||||
} else {
|
||||
this.getBinaryConfusionMatrices()[labelId].addToTrueNegatives(value, false);
|
||||
}
|
||||
}
|
||||
}
|
||||
for (let labelId: number = 0; labelId < this.getNumberLabels(); labelId++) {
|
||||
this.getBinaryConfusionMatrices()[labelId].calculateDerivedCells();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,80 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IMultiLabelConfusionMatrix } from "./IMultiLabelConfusionMatrix";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class MultiLabelConfusionMatrixBase
|
||||
extends ConfusionMatrixBase
|
||||
implements IMultiLabelConfusionMatrix {
|
||||
|
||||
protected binaryConfusionMatrices: BinaryConfusionMatrix[] = [];
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
}
|
||||
|
||||
public addFrom(other: MultiLabelConfusionMatrixBase): void {
|
||||
Utility.validateStringArrayPairEquality(this.labels, other.labels);
|
||||
DictionaryMapUtility.validateStringIdNumberValueDictionaryPair(this.labelMap, other.labelMap);
|
||||
const numbergetBinaryConfusionMatrices: number = this.getBinaryConfusionMatrices().length;
|
||||
for (let l: number = 0; l < numbergetBinaryConfusionMatrices; l++) {
|
||||
this.binaryConfusionMatrices[l].addFrom(other.binaryConfusionMatrices[l]);
|
||||
}
|
||||
}
|
||||
|
||||
public abstract addInstanceByLabelIndexes(
|
||||
groundTrueLabelIds: number[],
|
||||
predictedLabelIds: number[],
|
||||
value: number): void;
|
||||
|
||||
public addInstanceByLabels(
|
||||
groundTrueLabels: string[],
|
||||
predictedLabels: string[],
|
||||
value: number = 1): void {
|
||||
this.validateLabels(groundTrueLabels);
|
||||
this.validateLabels(predictedLabels);
|
||||
const groundTrueLabelIds: number[] = groundTrueLabels.map((x: string) => this.labelMap[x]);
|
||||
const predictedLabelIds: number[] = predictedLabels.map((x: string) => this.labelMap[x]);
|
||||
this.addInstanceByLabelIndexes(groundTrueLabelIds, predictedLabelIds, value);
|
||||
}
|
||||
|
||||
public getBinaryConfusionMatrices(): BinaryConfusionMatrix[] {
|
||||
return this.binaryConfusionMatrices;
|
||||
}
|
||||
|
||||
public validateLabelIds(
|
||||
labelIds: number[],
|
||||
throwIfNotLegal: boolean = true): boolean {
|
||||
for (const labelId of labelIds) {
|
||||
if (!this.validateLabelId(labelId, throwIfNotLegal)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
public validateLabels(
|
||||
labels: string[],
|
||||
throwIfNotLegal: boolean = true): boolean {
|
||||
for (const label of labels) {
|
||||
if (!this.validateLabel(label, throwIfNotLegal)) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// ---- NOTE ---- label id set is usually very small, mostly 1, so a linear search is sufficiently fast.
|
||||
public isLabelIdInArray(labelIds: number[], labelId: number): boolean {
|
||||
return labelIds.includes(labelId);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// import { IMultiLabelConfusionMatrix } from "./IMultiLabelConfusionMatrix";
|
||||
import { MultiLabelConfusionMatrixWithBinaryBase } from "./MultiLabelConfusionMatrixWithBinaryBase";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class MultiLabelConfusionMatrixSubset
|
||||
extends MultiLabelConfusionMatrixWithBinaryBase {
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
}
|
||||
|
||||
public addInstanceByLabelIndexes(
|
||||
groundTrueLabelIds: number[],
|
||||
predictedLabelIds: number[],
|
||||
value: number = 1): void {
|
||||
this.validateLabelIds(groundTrueLabelIds);
|
||||
this.validateLabelIds(predictedLabelIds);
|
||||
if (Utility.isEmptyNumberArray(predictedLabelIds)) {
|
||||
if (Utility.isEmptyNumberArray(groundTrueLabelIds)) {
|
||||
this.getBinaryConfusionMatrix().addToTrueNegatives(value, true);
|
||||
}
|
||||
this.getBinaryConfusionMatrix().addToFalseNegatives(value, true);
|
||||
return;
|
||||
}
|
||||
let isPredictionSubsetOfGroundTruthLabelIds: boolean = true;
|
||||
for (const predictedLabelId of predictedLabelIds) {
|
||||
if (!this.isLabelIdInArray(groundTrueLabelIds, predictedLabelId)) {
|
||||
isPredictionSubsetOfGroundTruthLabelIds = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (isPredictionSubsetOfGroundTruthLabelIds) {
|
||||
this.getBinaryConfusionMatrix().addToTruePositives(value, true);
|
||||
} else {
|
||||
this.getBinaryConfusionMatrix().addToFalsePositives(value, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,33 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// import { IMultiLabelConfusionMatrix } from "./IMultiLabelConfusionMatrix";
|
||||
import { MultiLabelConfusionMatrixBase } from "./MultiLabelConfusionMatrixBase";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class MultiLabelConfusionMatrixWithBinaryArrayBase
|
||||
extends MultiLabelConfusionMatrixBase {
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
this.reset();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
const numberLabels: number = this.getNumberLabels();
|
||||
this.binaryConfusionMatrices =
|
||||
new Array<BinaryConfusionMatrix>(numberLabels);
|
||||
for (let l: number = 0; l < numberLabels; l++) {
|
||||
this.binaryConfusionMatrices[l] = new BinaryConfusionMatrix();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,55 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// import { IMultiLabelConfusionMatrix } from "./IMultiLabelConfusionMatrix";
|
||||
import { MultiLabelConfusionMatrixBase } from "./MultiLabelConfusionMatrixBase";
|
||||
import { ConfusionMatrixBase } from "./ConfusionMatrixBase";
|
||||
import { BinaryConfusionMatrix } from "./BinaryConfusionMatrix";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class MultiLabelConfusionMatrixWithBinaryBase
|
||||
extends MultiLabelConfusionMatrixBase {
|
||||
|
||||
protected binaryConfusionMatrix: BinaryConfusionMatrix = new BinaryConfusionMatrix();
|
||||
|
||||
constructor(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(labels, labelMap);
|
||||
this.reset();
|
||||
}
|
||||
|
||||
public reset(): void {
|
||||
const numberLabels: number = this.getNumberLabels();
|
||||
this.binaryConfusionMatrix =
|
||||
new BinaryConfusionMatrix();
|
||||
this.binaryConfusionMatrices =
|
||||
new Array<BinaryConfusionMatrix>(1);
|
||||
this.binaryConfusionMatrices[0] =
|
||||
this.binaryConfusionMatrix;
|
||||
}
|
||||
|
||||
public getBinaryConfusionMatrix(): BinaryConfusionMatrix {
|
||||
return this.binaryConfusionMatrix;
|
||||
}
|
||||
|
||||
public getTotal(binaryConfusionMatrices: BinaryConfusionMatrix[] = []): number {
|
||||
if (Utility.isEmptyArray(binaryConfusionMatrices)) {
|
||||
binaryConfusionMatrices =
|
||||
this.getBinaryConfusionMatrices();
|
||||
}
|
||||
let binaryConfusionMatrix: BinaryConfusionMatrix =
|
||||
this.getBinaryConfusionMatrix();
|
||||
if (!Utility.isEmptyArray(binaryConfusionMatrices) && (binaryConfusionMatrices.length > 0)) {
|
||||
binaryConfusionMatrix = binaryConfusionMatrices[0];
|
||||
}
|
||||
const total: number =
|
||||
binaryConfusionMatrix.getTotal();
|
||||
return total;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IMathematicsHelper } from "./IMathematicsHelper";
|
||||
import { MathematicsHelper } from "./MathematicsHelper";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
const MathematicsHelperObject: IMathematicsHelper =
|
||||
MathematicsHelper.GetMathematicsHelperObject();
|
||||
|
||||
export function exampleFunctionMathematicsHelper(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const numerator = 1;
|
||||
const denominator = 2;
|
||||
Utility.debuggingLog(
|
||||
MathematicsHelperObject.safeDivide(numerator, denominator));
|
||||
Utility.debuggingLog(
|
||||
MathematicsHelperObject.safeLog(
|
||||
MathematicsHelperObject.safeDivide(numerator, denominator)));
|
||||
// -----------------------------------------------------------------------
|
||||
const inputValue: number = 0.5;
|
||||
const inputValueSigmoid: number = MathematicsHelperObject.sigmoidLogisticFunction(inputValue);
|
||||
Utility.debuggingLog(inputValueSigmoid);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
export function exampleFunctionMathematicsHelperSoftmax(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const numerator = 1;
|
||||
const denominator = 2;
|
||||
Utility.debuggingLog(
|
||||
MathematicsHelperObject.safeDivide(numerator, denominator));
|
||||
Utility.debuggingLog(
|
||||
MathematicsHelperObject.safeLog(
|
||||
MathematicsHelperObject.safeDivide(numerator, denominator)));
|
||||
// -----------------------------------------------------------------------
|
||||
const inputValue: number = 0.5;
|
||||
const inputValueSigmoid: number = MathematicsHelperObject.sigmoidLogisticFunction(inputValue);
|
||||
Utility.debuggingLog(inputValueSigmoid);
|
||||
// -----------------------------------------------------------------------
|
||||
/*
|
||||
import numpy as mp
|
||||
|
||||
Xraw = [ [ 1, 0.1, 0.5 ],
|
||||
[ 1, 1.1, 2.3 ],
|
||||
[ 1, -1.1, -2.3 ],
|
||||
[ 1, -1.5, -2.5 ] ]
|
||||
|
||||
biasraw = [ 0.01, 0.1, 0.1 ]
|
||||
|
||||
Wraw = [ [ 0.01, 0.1, 0.1 ],
|
||||
[ 0.1, 0.2, 0.3 ],
|
||||
[ 0.1, 0.2, 0.3 ] ]
|
||||
|
||||
bias = np.array(biasraw)
|
||||
X = np.array(Xraw)
|
||||
W = np.array(Wraw)
|
||||
|
||||
Z = W * np.transpose(X)
|
||||
|
||||
*/
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionMathematicsHelper();
|
||||
exampleFunctionMathematicsHelperSoftmax();
|
||||
}
|
||||
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
Разница между файлами не показана из-за своего большого размера
Загрузить разницу
|
|
@ -1,27 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { AbstractBaseSampler } from "./AbstractBaseSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseBootstrapSampler<T, TLabelMap, TInstances> extends
|
||||
AbstractBaseSampler<T, TLabelMap, TInstances> {
|
||||
|
||||
protected sampleSizeConfiguration: number = 1;
|
||||
|
||||
public constructor(
|
||||
instances: TInstances,
|
||||
toResetLabelsAndMap: boolean = true,
|
||||
sampleSizeConfiguration: number = 1) {
|
||||
super(instances, toResetLabelsAndMap);
|
||||
}
|
||||
|
||||
public getSampleSizeConfiguration(): number {
|
||||
return this.sampleSizeConfiguration;
|
||||
}
|
||||
|
||||
public abstract createSampleInstancesGenerator(): Generator<T, void, unknown>;
|
||||
}
|
||||
|
|
@ -1,23 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { AbstractBaseSampler } from "./AbstractBaseSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseReservoirSampler<T, TLabelMap, TInstances> extends
|
||||
AbstractBaseSampler<T, TLabelMap, TInstances> {
|
||||
|
||||
public constructor(
|
||||
instances: TInstances,
|
||||
toResetLabelsAndMap: boolean = true) {
|
||||
super(instances, toResetLabelsAndMap);
|
||||
}
|
||||
|
||||
public abstract createSampleInstancesGenerator(
|
||||
sampleSize: number,
|
||||
minNumberPerLabel: number,
|
||||
toShuffle: boolean): Generator<T, void, unknown>;
|
||||
}
|
||||
|
|
@ -1,42 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseSampler<T, TLabelMap, TInstances> {
|
||||
|
||||
protected labels: string[] = [];
|
||||
protected labelMap: TLabelMap | undefined;
|
||||
protected instances: TInstances;
|
||||
|
||||
public constructor(
|
||||
instances: TInstances,
|
||||
toResetLabelsAndMap: boolean = true) {
|
||||
this.labels = [];
|
||||
this.instances = instances;
|
||||
if (toResetLabelsAndMap) {
|
||||
this.resetLabelsAndMap();
|
||||
}
|
||||
}
|
||||
|
||||
public abstract addInstance(label: string, instance: T): void;
|
||||
|
||||
public abstract resetLabelsAndMap(): void;
|
||||
|
||||
public getNumberLabels(): number {
|
||||
return this.labels.length;
|
||||
}
|
||||
public getLabels(): string[] {
|
||||
return this.labels;
|
||||
}
|
||||
public getLabelMap(): TLabelMap {
|
||||
return (this.labelMap as TLabelMap);
|
||||
}
|
||||
public getInstances(): TInstances {
|
||||
return this.instances;
|
||||
}
|
||||
|
||||
public abstract getNumberInstances(): number;
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArray } from "../../data_structure/IDictionaryStringIdGenericArray";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { BootstrapSampler } from "./BootstrapSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionBootstrapSampler(): void {
|
||||
const sampler: BootstrapSampler<string> = new BootstrapSampler<string>();
|
||||
sampler.addInstance("label0", "label0 - utterance 0");
|
||||
sampler.addInstance("label0", "label0 - utterance 1");
|
||||
sampler.addInstance("label0", "label0 - utterance 2");
|
||||
sampler.addInstance("label0", "label0 - utterance 3");
|
||||
sampler.addInstance("label1", "label1 - utterance 0");
|
||||
sampler.addInstance("label1", "label1 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 0");
|
||||
sampler.addInstance("label2", "label2 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 2");
|
||||
sampler.addInstance("label2", "label2 - utterance 3");
|
||||
sampler.addInstance("label2", "label2 - utterance 4");
|
||||
sampler.addInstance("label2", "label2 - utterance 5");
|
||||
sampler.resetLabelsAndMap();
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances();
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
export function exampleFunctionBootstrapSamplerWithArrayInput(): void {
|
||||
const instances: IDictionaryStringIdGenericArray<number> = {};
|
||||
instances.a = [0, 1, 2, 3];
|
||||
instances.b = [4, 5, 6];
|
||||
instances.c = [7, 8, 9];
|
||||
const sampler: BootstrapSampler<number> = new BootstrapSampler<number>(instances);
|
||||
const typeInstances = typeof instances;
|
||||
Utility.debuggingLog(
|
||||
`typeInstances=${typeInstances}`);
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances();
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionBootstrapSampler();
|
||||
exampleFunctionBootstrapSamplerWithArrayInput();
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { TMapStringKeyGenericArray } from "../../data_structure/TMapStringKeyGenericArray";
|
||||
import { TMapStringKeyGenericValue } from "../../data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
import { BootstrapSamplerKeyMap } from "./BootstrapSamplerKeyMap";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionBootstrapSamplerKeyMap(): void {
|
||||
const sampler: BootstrapSamplerKeyMap<string> =
|
||||
new BootstrapSamplerKeyMap<string>(DictionaryMapUtility.newTMapStringKeyGenericArray<string>());
|
||||
sampler.addInstance("label0", "label0 - utterance 0");
|
||||
sampler.addInstance("label0", "label0 - utterance 1");
|
||||
sampler.addInstance("label0", "label0 - utterance 2");
|
||||
sampler.addInstance("label0", "label0 - utterance 3");
|
||||
sampler.addInstance("label1", "label1 - utterance 0");
|
||||
sampler.addInstance("label1", "label1 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 0");
|
||||
sampler.addInstance("label2", "label2 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 2");
|
||||
sampler.addInstance("label2", "label2 - utterance 3");
|
||||
sampler.addInstance("label2", "label2 - utterance 4");
|
||||
sampler.addInstance("label2", "label2 - utterance 5");
|
||||
sampler.resetLabelsAndMap();
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances();
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
export function exampleFunctionBootstrapSamplerKeyMapWithArrayInput(): void {
|
||||
const instances: TMapStringKeyGenericArray<number> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericArray<number>();
|
||||
instances.set("a", [0, 1, 2, 3]);
|
||||
instances.set("b", [4, 5, 6]);
|
||||
instances.set("c", [7, 8, 9]);
|
||||
const sampler: BootstrapSamplerKeyMap<number> =
|
||||
new BootstrapSamplerKeyMap<number>(instances);
|
||||
const typeInstances = typeof instances;
|
||||
Utility.debuggingLog(
|
||||
`typeInstances=${typeInstances}`);
|
||||
Utility.debuggingLog(
|
||||
`typeInstances=${typeInstances}`);
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances();
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionBootstrapSamplerKeyMap();
|
||||
exampleFunctionBootstrapSamplerKeyMapWithArrayInput();
|
||||
}
|
||||
|
|
@ -1,45 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ReservoirArraySampler } from "./ReservoirArraySampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionReservoirArraySampler(): void {
|
||||
const sampler: ReservoirArraySampler<number> = new ReservoirArraySampler<number>();
|
||||
for (let index = 0; index < 100; index++) {
|
||||
sampler.addInstance(index);
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstancesSampling() = " + sampler.getNumberInstancesSampling());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getInstancesBeginIndex() = " + sampler.getInstancesBeginIndex());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getInstancesEndIndex() = " + sampler.getInstancesEndIndex());
|
||||
sampler.resetInstancesBeginEndIndex(10, 30);
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstancesSampling() = " + sampler.getNumberInstancesSampling());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getInstancesBeginIndex() = " + sampler.getInstancesBeginIndex());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getInstancesEndIndex() = " + sampler.getInstancesEndIndex());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances(10);
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionReservoirArraySampler();
|
||||
}
|
||||
|
|
@ -1,82 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArray } from "../../data_structure/IDictionaryStringIdGenericArray";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { ReservoirSampler } from "./ReservoirSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionReservoirSampler(): void {
|
||||
const sampler: ReservoirSampler<string> = new ReservoirSampler<string>();
|
||||
sampler.addInstance("label0", "label0 - utterance 0");
|
||||
sampler.addInstance("label0", "label0 - utterance 1");
|
||||
sampler.addInstance("label0", "label0 - utterance 2");
|
||||
sampler.addInstance("label0", "label0 - utterance 3");
|
||||
sampler.addInstance("label1", "label1 - utterance 0");
|
||||
sampler.addInstance("label1", "label1 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 0");
|
||||
sampler.addInstance("label2", "label2 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 2");
|
||||
sampler.addInstance("label2", "label2 - utterance 3");
|
||||
sampler.addInstance("label2", "label2 - utterance 4");
|
||||
sampler.addInstance("label2", "label2 - utterance 5");
|
||||
sampler.resetLabelsAndMap();
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances(6, 2);
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
export function exampleFunctionReservoirSamplerWithArrayInput(): void {
|
||||
const instances: IDictionaryStringIdGenericArray<number> = {};
|
||||
instances.a = [0, 1, 2, 3];
|
||||
instances.b = [4, 5, 6];
|
||||
instances.c = [7, 8, 9];
|
||||
const sampler: ReservoirSampler<number> = new ReservoirSampler<number>(instances);
|
||||
const typeInstances = typeof instances;
|
||||
Utility.debuggingLog(
|
||||
`typeInstances=${typeInstances}`);
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances(6, 2);
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionReservoirSampler();
|
||||
exampleFunctionReservoirSamplerWithArrayInput();
|
||||
}
|
||||
|
|
@ -1,87 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { TMapStringKeyGenericArray } from "../../data_structure/TMapStringKeyGenericArray";
|
||||
import { TMapStringKeyGenericValue } from "../../data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
import { ReservoirSamplerKeyMap } from "./ReservoirSamplerKeyMap";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export function exampleFunctionReservoirSamplerKeyMap(): void {
|
||||
const sampler: ReservoirSamplerKeyMap<string> =
|
||||
new ReservoirSamplerKeyMap<string>(DictionaryMapUtility.newTMapStringKeyGenericArray<string>());
|
||||
sampler.addInstance("label0", "label0 - utterance 0");
|
||||
sampler.addInstance("label0", "label0 - utterance 1");
|
||||
sampler.addInstance("label0", "label0 - utterance 2");
|
||||
sampler.addInstance("label0", "label0 - utterance 3");
|
||||
sampler.addInstance("label1", "label1 - utterance 0");
|
||||
sampler.addInstance("label1", "label1 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 0");
|
||||
sampler.addInstance("label2", "label2 - utterance 1");
|
||||
sampler.addInstance("label2", "label2 - utterance 2");
|
||||
sampler.addInstance("label2", "label2 - utterance 3");
|
||||
sampler.addInstance("label2", "label2 - utterance 4");
|
||||
sampler.addInstance("label2", "label2 - utterance 5");
|
||||
sampler.resetLabelsAndMap();
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances(6, 2);
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
export function exampleFunctionReservoirSamplerKeyMapWithArrayInput(): void {
|
||||
const instances: TMapStringKeyGenericArray<number> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericArray<number>();
|
||||
instances.set("a", [0, 1, 2, 3]);
|
||||
instances.set("b", [4, 5, 6]);
|
||||
instances.set("c", [7, 8, 9]);
|
||||
const sampler: ReservoirSamplerKeyMap<number> =
|
||||
new ReservoirSamplerKeyMap<number>(instances);
|
||||
const typeInstances = typeof instances;
|
||||
Utility.debuggingLog(
|
||||
`typeInstances=${typeInstances}`);
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
Utility.debuggingLog(
|
||||
Object.keys(sampler.getInstances()));
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberInstances() = " + sampler.getNumberInstances());
|
||||
Utility.debuggingLog(
|
||||
"sampler.getNumberLabels() = " + sampler.getNumberLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabels());
|
||||
Utility.debuggingLog(
|
||||
sampler.getLabelMap());
|
||||
Utility.debuggingLog(
|
||||
sampler.getInstances());
|
||||
const sampledInstances = sampler.sampleInstances(6, 2);
|
||||
for (const instance of sampledInstances) {
|
||||
Utility.debuggingLog(
|
||||
instance);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionReservoirSamplerKeyMap();
|
||||
exampleFunctionReservoirSamplerKeyMapWithArrayInput();
|
||||
}
|
||||
|
|
@ -1,104 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArray } from "../../data_structure/IDictionaryStringIdGenericArray";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseBootstrapSampler } from "./AbstractBaseBootstrapSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class BootstrapSampler<T> extends
|
||||
AbstractBaseBootstrapSampler<T, IDictionaryStringIdGenericValue<number>, IDictionaryStringIdGenericArray<T>> {
|
||||
|
||||
public constructor(
|
||||
instances: IDictionaryStringIdGenericArray<T> = {},
|
||||
toResetLabelsAndMap: boolean = true,
|
||||
sampleSizeConfiguration: number = 1) {
|
||||
super(instances, toResetLabelsAndMap, sampleSizeConfiguration);
|
||||
}
|
||||
|
||||
public addInstance(label: string, instance: T): void {
|
||||
if (label in this.instances) {
|
||||
this.instances[label].push(instance);
|
||||
} else {
|
||||
this.instances[label] = [instance];
|
||||
}
|
||||
}
|
||||
|
||||
public resetLabelsAndMap(): void {
|
||||
this.labels =
|
||||
Object.keys(this.instances);
|
||||
this.labelMap =
|
||||
DictionaryMapUtility.buildStringIdNumberValueDictionaryFromUniqueStringArray(this.labels);
|
||||
DictionaryMapUtility.validateStringArrayAndStringIdNumberValueDictionary(
|
||||
this.labels,
|
||||
this.labelMap);
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
let numberInstances: number = 0;
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
numberInstances += instanceArray.length;
|
||||
}
|
||||
}
|
||||
return numberInstances;
|
||||
}
|
||||
|
||||
public computeSamplingNumberInstancesPerLabel(label: string = ""): number {
|
||||
// ---- NOTE: can be overridden in child class.
|
||||
let numberInstancesPerLabelReduce: number =
|
||||
this.labels.reduce(
|
||||
(maxValue: number, key: string) =>
|
||||
this.instances[key].length > maxValue ? this.instances[key].length : maxValue,
|
||||
0);
|
||||
Utility.debuggingLog(`numberInstancesPerLabel=` +
|
||||
`${numberInstancesPerLabelReduce}`);
|
||||
Utility.debuggingLog(`numberInstancesPerLabelSpread-BEFORE-adjustment=` +
|
||||
`${numberInstancesPerLabelReduce}`);
|
||||
numberInstancesPerLabelReduce =
|
||||
Utility.getRoundInteger(numberInstancesPerLabelReduce * this.getSampleSizeConfiguration());
|
||||
return numberInstancesPerLabelReduce;
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- const numberInstancesPerLabelSpreadArray: number[] =
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- [...this.instances].map((x) => x[1].length);
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- Utility.debuggingLog(`numberInstancesPerLabelSpreadArray=` +
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- `${[...numberInstancesPerLabelSpreadArray]}`);
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- const numberInstancesPerLabelSpread: number =
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- Math.max(...[...this.instances].map((x) => x[1].length));
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- Utility.debuggingLog(`numberInstancesPerLabelSpread=` +
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- `${numberInstancesPerLabelSpread}`);
|
||||
// ---- NOTE-NEED-UNIT-TEST-TO-CHECK ---- return numberInstancesPerLabelSpread;
|
||||
}
|
||||
|
||||
public createSampleInstancesGenerator(): Generator<T, void, unknown> {
|
||||
return this.sampleInstances();
|
||||
}
|
||||
|
||||
public *sampleInstances() {
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): key=` +
|
||||
// ---- `${key}`);
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
const numberInstancesPerLabel: number = instanceArray.length;
|
||||
if (numberInstancesPerLabel > 0) {
|
||||
const numberSamplingInstancesPerLabel: number =
|
||||
this.computeSamplingNumberInstancesPerLabel(key);
|
||||
for (let i: number = 0; i < numberSamplingInstancesPerLabel; i++) {
|
||||
const indexRandom =
|
||||
Utility.getRandomInt(numberInstancesPerLabel);
|
||||
yield instanceArray[indexRandom];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,50 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArray } from "../../data_structure/IDictionaryStringIdGenericArray";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { BootstrapSampler } from "./BootstrapSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class BootstrapSamplerDistribution<T> extends
|
||||
BootstrapSampler<T> {
|
||||
|
||||
protected distribution: IDictionaryStringIdGenericValue<number> = {};
|
||||
|
||||
public constructor(
|
||||
distribution: IDictionaryStringIdGenericValue<number> = {},
|
||||
instances: IDictionaryStringIdGenericArray<T> = {},
|
||||
toResetLabelsAndMap: boolean = true,
|
||||
sampleSizeConfiguration: number = 1) {
|
||||
super(instances, toResetLabelsAndMap, sampleSizeConfiguration);
|
||||
this.distribution = distribution;
|
||||
}
|
||||
|
||||
public getDistribution(): IDictionaryStringIdGenericValue<number> {
|
||||
return this.distribution;
|
||||
}
|
||||
|
||||
public computeSamplingNumberInstancesPerLabel(label: string = ""): number {
|
||||
const distribution: IDictionaryStringIdGenericValue<number> =
|
||||
this.getDistribution();
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(distribution)) {
|
||||
return super.computeSamplingNumberInstancesPerLabel(label);
|
||||
}
|
||||
if (!(label in distribution)) {
|
||||
return 0;
|
||||
}
|
||||
let numberInstancesPerLabel: number =
|
||||
distribution[label];
|
||||
Utility.debuggingLog(`numberInstancesPerLabel-BEFORE-adjustment=` +
|
||||
`${numberInstancesPerLabel}`);
|
||||
numberInstancesPerLabel =
|
||||
Utility.getRoundInteger(numberInstancesPerLabel * this.getSampleSizeConfiguration());
|
||||
return numberInstancesPerLabel;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,109 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapStringKeyGenericArray } from "../../data_structure/TMapStringKeyGenericArray";
|
||||
import { TMapStringKeyGenericValue } from "../../data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseBootstrapSampler } from "./AbstractBaseBootstrapSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class BootstrapSamplerKeyMap<T> extends
|
||||
AbstractBaseBootstrapSampler<T, TMapStringKeyGenericValue<number>, TMapStringKeyGenericArray<T>> {
|
||||
|
||||
public constructor(
|
||||
instances: TMapStringKeyGenericArray<T> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericArray<T>(),
|
||||
toResetLabelsAndMap: boolean = true,
|
||||
sampleSizeConfiguration: number = 1) {
|
||||
super(instances, toResetLabelsAndMap, sampleSizeConfiguration);
|
||||
}
|
||||
|
||||
public addInstance(label: string, instance: T): void {
|
||||
if (this.instances.has(label)) {
|
||||
(this.instances.get(label) as T[]).push(instance);
|
||||
} else {
|
||||
this.instances.set(label, [instance]);
|
||||
}
|
||||
}
|
||||
|
||||
public resetLabelsAndMap(): void {
|
||||
this.labels =
|
||||
Object.keys(this.instances);
|
||||
this.labelMap =
|
||||
DictionaryMapUtility.buildStringKeyNumberValueMapFromUniqueStringArray(this.labels);
|
||||
DictionaryMapUtility.validateStringArrayAndStringKeyNumberValueMap(
|
||||
this.labels,
|
||||
this.labelMap);
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
let numberInstances: number = 0;
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
numberInstances += instanceArray.length;
|
||||
}
|
||||
}
|
||||
return numberInstances;
|
||||
}
|
||||
|
||||
public computeSamplingNumberInstancesPerLabel(label: string = ""): number {
|
||||
// ---- NOTE: can be overridden in child class.
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- const numberInstancesPerLabelReduce: number =
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- this.labels.reduce(
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- (maxValue: number, key: string) =>
|
||||
// tslint:disable-next-line: max-line-length
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- (this.instances.get(key) as T[]).length > maxValue ? (this.instances.get(key) as T[]).length : maxValue,
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- 0);
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- Utility.debuggingLog(`numberInstancesPerLabelReduce=` +
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- `${numberInstancesPerLabelReduce}`);
|
||||
// ---- NOTE-DOES-NOT-WORK-NEED-TO-CHECK ---- return numberInstancesPerLabelReduce;
|
||||
const numberInstancesPerLabelSpreadArray: number[] =
|
||||
[...this.instances].map((x) => x[1].length);
|
||||
Utility.debuggingLog(`numberInstancesPerLabelSpreadArray=` +
|
||||
`${[...numberInstancesPerLabelSpreadArray]}`);
|
||||
let numberInstancesPerLabelSpread: number =
|
||||
Math.max(...[...this.instances].map((x) => x[1].length));
|
||||
Utility.debuggingLog(`numberInstancesPerLabelSpread-BEFORE-adjustment=` +
|
||||
`${numberInstancesPerLabelSpread}`);
|
||||
numberInstancesPerLabelSpread =
|
||||
Utility.getRoundInteger(numberInstancesPerLabelSpread * this.getSampleSizeConfiguration());
|
||||
Utility.debuggingLog(`numberInstancesPerLabelSpread=` +
|
||||
`${numberInstancesPerLabelSpread}`);
|
||||
return numberInstancesPerLabelSpread;
|
||||
}
|
||||
|
||||
public createSampleInstancesGenerator(): Generator<T, void, unknown> {
|
||||
return this.sampleInstances();
|
||||
}
|
||||
|
||||
public *sampleInstances() {
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
const key: string = entry[0];
|
||||
const instanceArray: T[] = entry[1];
|
||||
const numberInstancesPerLabel: number = instanceArray.length;
|
||||
if (numberInstancesPerLabel > 0) {
|
||||
const numberSamplingInstancesPerLabel: number =
|
||||
this.computeSamplingNumberInstancesPerLabel(key);
|
||||
for (let i: number = 0; i < numberSamplingInstancesPerLabel; i++) {
|
||||
const indexRandom =
|
||||
Utility.getRandomInt(numberInstancesPerLabel);
|
||||
yield instanceArray[indexRandom];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,53 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapStringKeyGenericArray } from "../../data_structure/TMapStringKeyGenericArray";
|
||||
import { TMapStringKeyGenericValue } from "../../data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { BootstrapSamplerKeyMap } from "./BootstrapSamplerKeyMap";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class BootstrapSamplerKeyMapDistribution<T> extends
|
||||
BootstrapSamplerKeyMap<T> {
|
||||
|
||||
protected distribution: TMapStringKeyGenericValue<number> =
|
||||
DictionaryMapUtility.newTMapStringKeyGenericValue<number>();
|
||||
|
||||
public constructor(
|
||||
distribution: TMapStringKeyGenericValue<number>,
|
||||
instances: TMapStringKeyGenericArray<T> = DictionaryMapUtility.newTMapStringKeyGenericArray<T>(),
|
||||
toResetLabelsAndMap: boolean = true,
|
||||
sampleSizeConfiguration: number = 1) {
|
||||
super(instances, toResetLabelsAndMap, sampleSizeConfiguration);
|
||||
this.distribution = distribution;
|
||||
}
|
||||
|
||||
public getDistribution(): TMapStringKeyGenericValue<number> {
|
||||
return this.distribution;
|
||||
}
|
||||
|
||||
public computeSamplingNumberInstancesPerLabel(label: string = ""): number {
|
||||
const distribution: TMapStringKeyGenericValue<number> =
|
||||
this.getDistribution();
|
||||
if (DictionaryMapUtility.isEmptyStringKeyGenericValueMap(distribution)) {
|
||||
return super.computeSamplingNumberInstancesPerLabel(label);
|
||||
}
|
||||
if (!distribution.has(label)) {
|
||||
return 0;
|
||||
}
|
||||
let numberInstancesPerLabel: number =
|
||||
distribution.get(label) as number;
|
||||
Utility.debuggingLog(`numberInstancesPerLabel-BEFORE-adjustment=` +
|
||||
`${numberInstancesPerLabel}`);
|
||||
numberInstancesPerLabel =
|
||||
Utility.getRoundInteger(numberInstancesPerLabel * this.getSampleSizeConfiguration());
|
||||
Utility.debuggingLog(`numberInstancesPerLabel=` +
|
||||
`${numberInstancesPerLabel}`);
|
||||
return numberInstancesPerLabel;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,89 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class ReservoirArraySampler<T> {
|
||||
|
||||
protected instances: T[] = [];
|
||||
protected instancesBeginIndex: number = 0;
|
||||
protected instancesEndIndex: number = -1;
|
||||
|
||||
public constructor(
|
||||
instances: T[] = [],
|
||||
instancesBeginIndex: number = 0,
|
||||
instancesEndIndex: number = -1) {
|
||||
this.instances = instances;
|
||||
this.resetInstancesBeginEndIndex(
|
||||
instancesBeginIndex,
|
||||
instancesEndIndex);
|
||||
}
|
||||
|
||||
public getInstances(): T[] {
|
||||
return this.instances;
|
||||
}
|
||||
public getInstancesBeginIndex(): number {
|
||||
return this.instancesBeginIndex;
|
||||
}
|
||||
public getInstancesEndIndex(): number {
|
||||
return this.instancesEndIndex;
|
||||
}
|
||||
|
||||
public resetInstancesBeginEndIndex(
|
||||
instancesBeginIndex: number = 0,
|
||||
instancesEndIndex: number = -1): void {
|
||||
this.instancesBeginIndex = instancesBeginIndex;
|
||||
if (this.instancesBeginIndex < 0) {
|
||||
this.instancesBeginIndex = 0;
|
||||
}
|
||||
this.instancesEndIndex = instancesEndIndex;
|
||||
if (this.instancesEndIndex < 0) {
|
||||
this.instancesEndIndex = this.instances.length;
|
||||
}
|
||||
if (this.instancesEndIndex > this.instances.length) {
|
||||
this.instancesEndIndex = this.instances.length;
|
||||
}
|
||||
}
|
||||
|
||||
public addInstance(instance: T) {
|
||||
this.instances.push(instance);
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
return this.getInstances().length;
|
||||
}
|
||||
public getNumberInstancesSampling(): number {
|
||||
return (this.getInstancesEndIndex() - this.getInstancesBeginIndex());
|
||||
}
|
||||
|
||||
public *sampleInstances(
|
||||
sampleSize: number) {
|
||||
const numberInstancesSampling = this.getNumberInstancesSampling();
|
||||
const instances: T[] = this.getInstances();
|
||||
const instancesBeginIndex: number = this.getInstancesBeginIndex();
|
||||
const instancesEndIndex: number = this.getInstancesEndIndex();
|
||||
if (sampleSize >= numberInstancesSampling) {
|
||||
for (let index = instancesBeginIndex; index < instancesEndIndex; index++) {
|
||||
yield instances[index];
|
||||
}
|
||||
} else {
|
||||
const sampledInstances: T[] = new Array<T>(sampleSize);
|
||||
let indexSampled: number = 0;
|
||||
for (let index = instancesBeginIndex; index < instancesEndIndex; index++) {
|
||||
if (indexSampled < sampleSize) {
|
||||
sampledInstances[indexSampled++] = instances[index];
|
||||
} else {
|
||||
const indexRandom = Utility.getRandomInt(++indexSampled);
|
||||
if (indexRandom < sampleSize) {
|
||||
sampledInstances[indexRandom] = instances[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const instance of sampledInstances) {
|
||||
yield instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,159 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArray } from "../../data_structure/IDictionaryStringIdGenericArray";
|
||||
import { IDictionaryStringIdGenericValue } from "../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseReservoirSampler } from "./AbstractBaseReservoirSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class ReservoirSampler<T> extends
|
||||
AbstractBaseReservoirSampler<T, IDictionaryStringIdGenericValue<number>, IDictionaryStringIdGenericArray<T>> {
|
||||
|
||||
public constructor(
|
||||
instances: IDictionaryStringIdGenericArray<T> = {},
|
||||
toResetLabelsAndMap: boolean = true) {
|
||||
super(instances, toResetLabelsAndMap);
|
||||
}
|
||||
|
||||
public addInstance(label: string, instance: T) {
|
||||
if (label in this.instances) {
|
||||
this.instances[label].push(instance);
|
||||
} else {
|
||||
this.instances[label] = [instance];
|
||||
}
|
||||
}
|
||||
|
||||
public resetLabelsAndMap(): void {
|
||||
this.labels =
|
||||
Object.keys(this.instances);
|
||||
this.labelMap =
|
||||
DictionaryMapUtility.buildStringIdNumberValueDictionaryFromUniqueStringArray(this.labels);
|
||||
DictionaryMapUtility.validateStringArrayAndStringIdNumberValueDictionary(
|
||||
this.labels,
|
||||
this.labelMap);
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
let numberInstances: number = 0;
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
numberInstances += instanceArray.length;
|
||||
}
|
||||
}
|
||||
return numberInstances;
|
||||
}
|
||||
|
||||
public createSampleInstancesGenerator(
|
||||
sampleSize: number,
|
||||
minNumberPerLabel: number = 1,
|
||||
toShuffle: boolean = true): Generator<T, void, unknown> {
|
||||
return this.sampleInstances(
|
||||
sampleSize,
|
||||
minNumberPerLabel,
|
||||
toShuffle);
|
||||
}
|
||||
|
||||
public *sampleInstances(
|
||||
sampleSize: number,
|
||||
minNumberPerLabel: number = 1,
|
||||
toShuffle: boolean = true) {
|
||||
const numberInstances = this.getNumberInstances();
|
||||
if (numberInstances > 0) {
|
||||
const numberLabels = this.getNumberLabels();
|
||||
if (minNumberPerLabel <= 0) {
|
||||
minNumberPerLabel = 1;
|
||||
// ---- NOTE: need at least one sample per label.
|
||||
}
|
||||
let sumMinNumberPerLabel = minNumberPerLabel * numberLabels;
|
||||
if (sumMinNumberPerLabel > sampleSize) {
|
||||
minNumberPerLabel = Math.floor(sampleSize / numberLabels);
|
||||
sumMinNumberPerLabel = minNumberPerLabel * numberLabels;
|
||||
}
|
||||
if (minNumberPerLabel < 1) {
|
||||
minNumberPerLabel = 1;
|
||||
sumMinNumberPerLabel = numberLabels;
|
||||
}
|
||||
if (sampleSize < numberLabels) {
|
||||
sampleSize = numberLabels;
|
||||
// ---- NOTE: need at least one sample per label.
|
||||
minNumberPerLabel = 1;
|
||||
sumMinNumberPerLabel = numberLabels;
|
||||
}
|
||||
if (toShuffle) {
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
Utility.shuffle(instanceArray);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sampleSize >= numberInstances) {
|
||||
// ---- NOT: return every instance if sampleSize is bigger than the size of the instances.
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
for (const instance of instanceArray) {
|
||||
yield instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let numberSampled: number = 0;
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
const instanceArrayLength: number = instanceArray.length;
|
||||
for (let index: number = 0; index < minNumberPerLabel; index++) {
|
||||
if (index >= instanceArrayLength) {
|
||||
break;
|
||||
}
|
||||
yield instanceArray[index];
|
||||
numberSampled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
sampleSize -= numberSampled;
|
||||
if (sampleSize > 0) {
|
||||
let indexSampled: number = 0;
|
||||
const sampledInstances: T[] = new Array(sampleSize);
|
||||
for (const key in this.instances) {
|
||||
if (key) {
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = this.instances[key];
|
||||
const instanceArrayLength: number = instanceArray.length;
|
||||
for (let index: number = minNumberPerLabel; index < instanceArrayLength; index++) {
|
||||
if (indexSampled < sampleSize) {
|
||||
sampledInstances[indexSampled++] = instanceArray[index];
|
||||
} else {
|
||||
const indexRandom = Utility.getRandomInt(++indexSampled);
|
||||
if (indexRandom < sampleSize) {
|
||||
sampledInstances[indexRandom] = instanceArray[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const instance of sampledInstances) {
|
||||
yield instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,169 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { TMapStringKeyGenericArray } from "../../data_structure/TMapStringKeyGenericArray";
|
||||
import { TMapStringKeyGenericValue } from "../../data_structure/TMapStringKeyGenericValue";
|
||||
|
||||
import { DictionaryMapUtility } from "../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseReservoirSampler } from "./AbstractBaseReservoirSampler";
|
||||
|
||||
import { Utility } from "../../utility/Utility";
|
||||
|
||||
export class ReservoirSamplerKeyMap<T> extends
|
||||
AbstractBaseReservoirSampler<T, TMapStringKeyGenericValue<number>, TMapStringKeyGenericArray<T>> {
|
||||
|
||||
public constructor(
|
||||
instances: TMapStringKeyGenericArray<T> = DictionaryMapUtility.newTMapStringKeyGenericArray<T>(),
|
||||
toResetLabelsAndMap: boolean = true) {
|
||||
super(instances, toResetLabelsAndMap);
|
||||
}
|
||||
|
||||
public addInstance(label: string, instance: T) {
|
||||
if (this.instances.has(label)) {
|
||||
(this.instances.get(label) as T[]).push(instance);
|
||||
} else {
|
||||
this.instances.set(label, [instance]);
|
||||
}
|
||||
}
|
||||
|
||||
public resetLabelsAndMap(): void {
|
||||
this.labels =
|
||||
Object.keys(this.instances);
|
||||
this.labelMap =
|
||||
DictionaryMapUtility.buildStringKeyNumberValueMapFromUniqueStringArray(this.labels);
|
||||
DictionaryMapUtility.validateStringArrayAndStringKeyNumberValueMap(
|
||||
this.labels,
|
||||
this.labelMap);
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
let numberInstances: number = 0;
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
numberInstances += instanceArray.length;
|
||||
}
|
||||
}
|
||||
return numberInstances;
|
||||
}
|
||||
|
||||
public createSampleInstancesGenerator(
|
||||
sampleSize: number,
|
||||
minNumberPerLabel: number = 1,
|
||||
toShuffle: boolean = true): Generator<T, void, unknown> {
|
||||
return this.sampleInstances(
|
||||
sampleSize,
|
||||
minNumberPerLabel,
|
||||
toShuffle);
|
||||
}
|
||||
|
||||
public *sampleInstances(
|
||||
sampleSize: number,
|
||||
minNumberPerLabel: number = 1,
|
||||
toShuffle: boolean = true) {
|
||||
const numberInstances = this.getNumberInstances();
|
||||
if (numberInstances > 0) {
|
||||
const numberLabels = this.getNumberLabels();
|
||||
if (minNumberPerLabel <= 0) {
|
||||
minNumberPerLabel = 1;
|
||||
// ---- NOTE: need at least one sample per label.
|
||||
}
|
||||
let sumMinNumberPerLabel = minNumberPerLabel * numberLabels;
|
||||
if (sumMinNumberPerLabel > sampleSize) {
|
||||
minNumberPerLabel = Math.floor(sampleSize / numberLabels);
|
||||
sumMinNumberPerLabel = minNumberPerLabel * numberLabels;
|
||||
}
|
||||
if (minNumberPerLabel < 1) {
|
||||
minNumberPerLabel = 1;
|
||||
sumMinNumberPerLabel = numberLabels;
|
||||
}
|
||||
if (sampleSize < numberLabels) {
|
||||
sampleSize = numberLabels;
|
||||
// ---- NOTE: need at least one sample per label.
|
||||
minNumberPerLabel = 1;
|
||||
sumMinNumberPerLabel = numberLabels;
|
||||
}
|
||||
if (toShuffle) {
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
Utility.shuffle(instanceArray);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (sampleSize >= numberInstances) {
|
||||
// ---- NOT: return every instance if sampleSize is bigger than the size of the instances.
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
for (const instance of instanceArray) {
|
||||
yield instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let numberSampled: number = 0;
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
const instanceArrayLength: number = instanceArray.length;
|
||||
for (let index: number = 0; index < minNumberPerLabel; index++) {
|
||||
if (index >= instanceArrayLength) {
|
||||
break;
|
||||
}
|
||||
yield instanceArray[index];
|
||||
numberSampled++;
|
||||
}
|
||||
}
|
||||
}
|
||||
sampleSize -= numberSampled;
|
||||
if (sampleSize > 0) {
|
||||
let indexSampled: number = 0;
|
||||
const sampledInstances: T[] = new Array(sampleSize);
|
||||
for (const entry of this.instances) {
|
||||
if (entry) {
|
||||
// ---- Utility.debuggingLog(`sampleInstances(): entry=` +
|
||||
// ---- `${entry}`);
|
||||
// ---- side effect is to remove TSLint warning for
|
||||
// ---- 'in' statements must be filtered with an if statement.
|
||||
const instanceArray: T[] = entry[1];
|
||||
const instanceArrayLength: number = instanceArray.length;
|
||||
for (let index: number = minNumberPerLabel; index < instanceArrayLength; index++) {
|
||||
if (indexSampled < sampleSize) {
|
||||
sampledInstances[indexSampled++] = instanceArray[index];
|
||||
} else {
|
||||
const indexRandom = Utility.getRandomInt(++indexSampled);
|
||||
if (indexRandom < sampleSize) {
|
||||
sampledInstances[indexRandom] = instanceArray[index];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const instance of sampledInstances) {
|
||||
yield instance;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,25 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { Data } from "../../../data/Data";
|
||||
|
||||
import { AbstractBaseEvaluator } from "./AbstractBaseEvaluator";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseDataProfileEvaluator extends AbstractBaseEvaluator {
|
||||
|
||||
protected data: Data;
|
||||
|
||||
constructor(
|
||||
data: Data) {
|
||||
super();
|
||||
this.data = data;
|
||||
}
|
||||
|
||||
public getData(): Data {
|
||||
return this.data;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,71 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseEvaluator {
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
return {};
|
||||
}
|
||||
public abstract generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[],
|
||||
columnDelimiter: string,
|
||||
recordDelimiter: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string>): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
};
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
return {};
|
||||
}
|
||||
public abstract generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any>): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
};
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {};
|
||||
}
|
||||
public abstract generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string>):
|
||||
IDictionaryStringIdGenericValue<string>;
|
||||
|
||||
public dumpEvaluationJsonReportToFile(
|
||||
outputReportFilename: string,
|
||||
outputReportContent: any,
|
||||
encoding: string): string {
|
||||
if (Utility.isEmptyString(outputReportFilename)) {
|
||||
Utility.debuggingThrow("outputReportFilename is empty");
|
||||
}
|
||||
return Utility.dumpFile(
|
||||
outputReportFilename,
|
||||
Utility.jsonStringify(outputReportContent),
|
||||
encoding);
|
||||
}
|
||||
public dumpEvaluationDirectReportToFile(
|
||||
outputReportFilename: string,
|
||||
outputReportContent: any,
|
||||
encoding: string): string {
|
||||
if (Utility.isEmptyString(outputReportFilename)) {
|
||||
Utility.debuggingThrow("outputReportFilename is empty");
|
||||
}
|
||||
return Utility.dumpFile(
|
||||
outputReportFilename,
|
||||
outputReportContent,
|
||||
encoding);
|
||||
}
|
||||
}
|
||||
|
|
@ -1,141 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { AbstractBaseEvaluator } from "./AbstractBaseEvaluator";
|
||||
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export abstract class AbstractBaseModelFeaturizerEvaluator extends AbstractBaseEvaluator {
|
||||
|
||||
protected modelFilename: string = "";
|
||||
protected featurizerFilename: string = "";
|
||||
|
||||
protected modelNullable: SoftmaxRegressionSparse|null = null;
|
||||
protected featurizerNullable: NgramSubwordFeaturizer|null = null;
|
||||
|
||||
protected labels: string[] = [];
|
||||
protected labelMap: { [id: string]: number } = {};
|
||||
|
||||
constructor(
|
||||
modelFilename: string,
|
||||
featurizerFilename: string,
|
||||
modelNullable: SoftmaxRegressionSparse|null,
|
||||
featurizerNullable: NgramSubwordFeaturizer|null,
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super();
|
||||
if (!Utility.isEmptyString(modelFilename)) {
|
||||
this.modelNullable = this.deserializeModel(modelFilename);
|
||||
this.modelFilename = modelFilename;
|
||||
}
|
||||
if (!Utility.isEmptyString(featurizerFilename)) {
|
||||
this.featurizerNullable = this.deserializeFeaturizer(featurizerFilename);
|
||||
this.featurizerFilename = featurizerFilename;
|
||||
}
|
||||
if (this.modelNullable === null) {
|
||||
this.modelNullable = modelNullable;
|
||||
}
|
||||
if (this.featurizerNullable === null) {
|
||||
this.featurizerNullable = featurizerNullable;
|
||||
}
|
||||
if (this.featurizerNullable !== null) {
|
||||
this.labels = this.getFeaturizer().getLabels();
|
||||
}
|
||||
if (this.featurizerNullable !== null) {
|
||||
this.labelMap = this.getFeaturizer().getLabelMap();
|
||||
}
|
||||
if (Utility.isEmptyStringArray(this.labels)) {
|
||||
this.labels = labels;
|
||||
}
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(this.labelMap)) {
|
||||
this.labelMap = labelMap;
|
||||
}
|
||||
if ((!Utility.isEmptyStringArray(this.labels)) && (this.labelMap !== null)) {
|
||||
DictionaryMapUtility.validateStringArrayAndStringIdNumberValueDictionary(this.labels, this.labelMap);
|
||||
}
|
||||
}
|
||||
|
||||
public getModelFilename(): string {
|
||||
return this.modelFilename;
|
||||
}
|
||||
public getFeaturizerFilename(): string {
|
||||
return this.featurizerFilename;
|
||||
}
|
||||
|
||||
public getModelNullable(): SoftmaxRegressionSparse|null {
|
||||
return this.modelNullable;
|
||||
}
|
||||
public getFeaturizerNullable(): NgramSubwordFeaturizer|null {
|
||||
return this.featurizerNullable;
|
||||
}
|
||||
|
||||
public getModel(): SoftmaxRegressionSparse {
|
||||
const modelNullable: SoftmaxRegressionSparse|null =
|
||||
this.getModelNullable();
|
||||
if (modelNullable === null) {
|
||||
Utility.debuggingThrow("modelNullable is null");
|
||||
}
|
||||
return modelNullable as SoftmaxRegressionSparse;
|
||||
}
|
||||
public getFeaturizer(): NgramSubwordFeaturizer {
|
||||
const featurizerNullable: NgramSubwordFeaturizer|null =
|
||||
this.getFeaturizerNullable();
|
||||
if (featurizerNullable === null) {
|
||||
Utility.debuggingThrow("featurizerNullable is null");
|
||||
}
|
||||
return featurizerNullable as NgramSubwordFeaturizer;
|
||||
}
|
||||
|
||||
public getLabels(): string[] {
|
||||
if (Utility.isEmptyStringArray(this.labels)) {
|
||||
Utility.debuggingThrow("this.labels is empty");
|
||||
}
|
||||
return this.labels;
|
||||
}
|
||||
public getLabelMap(): { [id: string]: number } {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(this.labelMap)) {
|
||||
Utility.debuggingThrow("this.labelMap is empty");
|
||||
}
|
||||
return this.labelMap;
|
||||
}
|
||||
|
||||
public getNumberLabels(): number {
|
||||
return this.getLabels().length;
|
||||
}
|
||||
|
||||
public deserializeModel(
|
||||
modelFilename: string): SoftmaxRegressionSparse {
|
||||
if (Utility.isEmptyString(modelFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`modelFilename is empty`);
|
||||
}
|
||||
const modelFileDeserializedJsonString: string = Utility.loadFile(
|
||||
modelFilename);
|
||||
const model: SoftmaxRegressionSparse =
|
||||
new SoftmaxRegressionSparse();
|
||||
model.deserializeFromJsonString(modelFileDeserializedJsonString);
|
||||
return model;
|
||||
}
|
||||
|
||||
public deserializeFeaturizer(
|
||||
featurizerFilename: string): NgramSubwordFeaturizer {
|
||||
if (Utility.isEmptyString(featurizerFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`featurizerFilename is empty`);
|
||||
}
|
||||
const featurizerFileDeserializedJsonString: string = Utility.loadFile(
|
||||
featurizerFilename);
|
||||
const featurizer: NgramSubwordFeaturizer =
|
||||
new NgramSubwordFeaturizer();
|
||||
featurizer.deserializeFromJsonString(featurizerFileDeserializedJsonString);
|
||||
return featurizer;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,591 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import assert = require("assert");
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { CrossValidator } from "./CrossValidator";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { AppSoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
import { ColumnarData } from "../../../data/ColumnarData";
|
||||
import { LuData } from "../../../data/LuData";
|
||||
import { Data } from "../../../data/Data";
|
||||
import { DataUtility } from "../../../data/DataUtility";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
/**
|
||||
* This function consumes a Data object as input and run cross validation (CV) to evaluate models trained from
|
||||
* the input label/text (intent/utterance) instance set.
|
||||
*
|
||||
* @param data - input Data object as input.
|
||||
* @param numberOfCrossValidationFolds - number of cross validation (CV) folds.
|
||||
* @param learnerParameterEpochs - CV Softmax Regression Learner parameter - number of epochs
|
||||
* @param learnerParameterMiniBatchSize - CV Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - CV Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - CV Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - CV Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - CV Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - CV Softmax Regression learner parameter - flag
|
||||
*/
|
||||
export function mainCrossValidatorWithData(
|
||||
data: Data,
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true): CrossValidator {
|
||||
// -----------------------------------------------------------------------
|
||||
if (!numberOfCrossValidationFolds) {
|
||||
numberOfCrossValidationFolds = CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
const intents: string[] =
|
||||
data.getIntents();
|
||||
const utterances: string[] =
|
||||
data.getUtterances();
|
||||
const intentLabelIndexArray: number[] =
|
||||
data.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
data.getUtteranceFeatureIndexArrays();
|
||||
assert(intentLabelIndexArray, "intentLabelIndexArray is undefined.");
|
||||
assert(utteranceFeatureIndexArrays, "utteranceFeatureIndexArrays is undefined.");
|
||||
const crossValidator: CrossValidator =
|
||||
new CrossValidator(
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
const crossValidationResult: {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix,
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = crossValidator.crossValidate(
|
||||
data.getFeaturizerLabels(),
|
||||
data.getFeaturizerLabelMap(),
|
||||
data.getFeaturizer().getNumberLabels(),
|
||||
data.getFeaturizer().getNumberFeatures(),
|
||||
intents,
|
||||
utterances,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
data.getIntentInstanceIndexMapArray());
|
||||
Utility.debuggingLog(
|
||||
`crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()}`);
|
||||
return crossValidator;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
/**
|
||||
* This function consumes a LU file content as input and run cross validation (CV) to evaluate models trained from
|
||||
* the input label/text (intent/utterance) instance set.
|
||||
*
|
||||
* @param luContent - input LU file content as input.
|
||||
* @param numberOfCrossValidationFolds - number of cross validation (CV) folds.
|
||||
* @param learnerParameterEpochs - CV Softmax Regression Learner parameter - number of epochs
|
||||
* @param learnerParameterMiniBatchSize - CV Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - CV Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - CV Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - CV Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - CV Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - CV Softmax Regression learner parameter - flag
|
||||
*/
|
||||
export async function mainCrossValidatorWithLuContent(
|
||||
luContent: string,
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true): Promise<CrossValidator> {
|
||||
// -----------------------------------------------------------------------
|
||||
const luData: LuData =
|
||||
await LuData.createLuData(
|
||||
luContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (!numberOfCrossValidationFolds) {
|
||||
numberOfCrossValidationFolds = CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
const intents: string[] =
|
||||
luData.getIntents();
|
||||
const utterances: string[] =
|
||||
luData.getUtterances();
|
||||
const intentLabelIndexArray: number[] =
|
||||
luData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
luData.getUtteranceFeatureIndexArrays();
|
||||
assert(intentLabelIndexArray, "intentLabelIndexArray is undefined.");
|
||||
assert(utteranceFeatureIndexArrays, "utteranceFeatureIndexArrays is undefined.");
|
||||
const crossValidator: CrossValidator =
|
||||
new CrossValidator(
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
const crossValidationResult: {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix,
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = crossValidator.crossValidate(
|
||||
luData.getFeaturizerLabels(),
|
||||
luData.getFeaturizerLabelMap(),
|
||||
luData.getFeaturizer().getNumberLabels(),
|
||||
luData.getFeaturizer().getNumberFeatures(),
|
||||
intents,
|
||||
utterances,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
luData.getIntentInstanceIndexMapArray());
|
||||
Utility.debuggingLog(
|
||||
`crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()}`);
|
||||
return crossValidator;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
/**
|
||||
* This function consumes a columnar TSV file content as input and run cross validation (CV) to
|
||||
* evaluate models trained from the input label/text (intent/utterance) instance set.
|
||||
*
|
||||
* @param columnarContent - content of a TSV columnar file in string form as input.
|
||||
* @param labelColumnIndex - label/intent column index.
|
||||
* @param textColumnIndex - text/utterace column index.
|
||||
* @param weightColumnIndex - weight column index.
|
||||
* @param linesToSkip - number of header lines skipped before processing each line as an instance record.
|
||||
* @param numberOfCrossValidationFolds - number of cross validation (CV) folds.
|
||||
* @param learnerParameterEpochs - CV Softmax Regression Learner parameter - number of epochs
|
||||
* @param learnerParameterMiniBatchSize - CV Softmax Regression learner parameter - mini-batch size.
|
||||
* @param learnerParameterL1Regularization - CV Softmax Regression learner parameter - L1 regularization.
|
||||
* @param learnerParameterL2Regularization - CV Softmax Regression learner parameter - L2 regularization.
|
||||
* @param learnerParameterLossEarlyStopRatio - CV Softmax Regression learner parameter - early stop ratio.
|
||||
* @param learnerParameterLearningRate - CV Softmax Regression learner parameter - learning rate.
|
||||
* @param learnerParameterToCalculateOverallLossAfterEpoch - CV Softmax Regression learner parameter - flag
|
||||
*/
|
||||
export function mainCrossValidatorWithColumnarContent(
|
||||
columnarContent: string,
|
||||
labelColumnIndex: number,
|
||||
textColumnIndex: number,
|
||||
weightColumnIndex: number,
|
||||
linesToSkip: number,
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true): CrossValidator {
|
||||
// -----------------------------------------------------------------------
|
||||
const columnarData: ColumnarData =
|
||||
ColumnarData.createColumnarData(
|
||||
columnarContent,
|
||||
new NgramSubwordFeaturizer(),
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip,
|
||||
true);
|
||||
// -----------------------------------------------------------------------
|
||||
if (!numberOfCrossValidationFolds) {
|
||||
numberOfCrossValidationFolds = CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
const intents: string[] =
|
||||
columnarData.getIntents();
|
||||
const utterances: string[] =
|
||||
columnarData.getUtterances();
|
||||
const intentLabelIndexArray: number[] =
|
||||
columnarData.getIntentLabelIndexArray();
|
||||
const utteranceFeatureIndexArrays: number[][] =
|
||||
columnarData.getUtteranceFeatureIndexArrays();
|
||||
assert(intentLabelIndexArray, "intentLabelIndexArray is undefined.");
|
||||
assert(utteranceFeatureIndexArrays, "utteranceFeatureIndexArrays is undefined.");
|
||||
const crossValidator: CrossValidator =
|
||||
new CrossValidator(
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
const crossValidationResult: {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix,
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = crossValidator.crossValidate(
|
||||
columnarData.getFeaturizerLabels(),
|
||||
columnarData.getFeaturizerLabelMap(),
|
||||
columnarData.getFeaturizer().getNumberLabels(),
|
||||
columnarData.getFeaturizer().getNumberFeatures(),
|
||||
intents,
|
||||
utterances,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays,
|
||||
columnarData.getIntentInstanceIndexMapArray());
|
||||
Utility.debuggingLog(
|
||||
`crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMicroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getMacroAverageMetrics()}` +
|
||||
`,crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()=` +
|
||||
`${crossValidationResult.confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()}`);
|
||||
return crossValidator;
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
export async function mainCrossValidator(): Promise<{
|
||||
"evaluationJsonReportResult": {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
},
|
||||
"evaluationDataArraysReportResult": {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
},
|
||||
}> {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppCrossValidator",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "an input data file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-t", "--filetype"],
|
||||
{
|
||||
help: "data file type",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-m", "--outputModelFilename"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "output model file",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-x", "--outputFeaturizerFilename"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "output serialized featurizer file",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
parser.addArgument(
|
||||
["-nc", "--numberOfCrossValidationFolds"],
|
||||
{
|
||||
defaultValue: CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
help: "initial number of data instances per category for auto active learning",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-le", "--learnerParameterEpochs"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
help: "number of epochs",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-lb", "--learnerParameterMiniBatchSize"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
help: "mini batch size",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ll1", "--learnerParameterL1Regularization"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
help: "l1 regularization coefficient",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ll2", "--learnerParameterL2Regularization"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
help: "l2 regularization coefficient",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-lesr", "--learnerParameterLossEarlyStopRatio"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
help: "loss early stop ratio",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-llr", "--learnerParameterLearningRate"],
|
||||
{
|
||||
defaultValue: AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
help: "learning rate",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ltl", "--learnerParameterToCalculateOverallLossAfterEpoch"],
|
||||
{
|
||||
defaultValue: true,
|
||||
help: "whether to calcualte loss after each epoch",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--linesToSkip"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string =
|
||||
args.filename;
|
||||
if (!Utility.exists(filename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input dataset file ${filename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const filetype: string =
|
||||
args.filetype;
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(filename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const numberOfCrossValidationFolds: number =
|
||||
+args.numberOfCrossValidationFolds;
|
||||
const learnerParameterEpochs: number =
|
||||
+args.learnerParameterEpochs;
|
||||
const learnerParameterMiniBatchSize: number =
|
||||
+args.learnerParameterMiniBatchSize;
|
||||
const learnerParameterL1Regularization: number =
|
||||
+args.learnerParameterL1Regularization;
|
||||
const learnerParameterL2Regularization: number =
|
||||
+args.learnerParameterL2Regularization;
|
||||
const learnerParameterLossEarlyStopRatio: number =
|
||||
+args.learnerParameterLossEarlyStopRatio;
|
||||
const learnerParameterLearningRate: number =
|
||||
+args.learnerParameterLearningRate;
|
||||
const learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
args.learnerParameterToCalculateOverallLossAfterEpoch;
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
Utility.debuggingLog(
|
||||
`numberOfCrossValidationFolds=${numberOfCrossValidationFolds}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterEpochs=${learnerParameterEpochs}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterMiniBatchSize=${learnerParameterMiniBatchSize}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterL1Regularization=${learnerParameterL1Regularization}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterL2Regularization=${learnerParameterL2Regularization}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterLossEarlyStopRatio=${learnerParameterLossEarlyStopRatio}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterLearningRate=${learnerParameterLearningRate}`);
|
||||
Utility.debuggingLog(
|
||||
`learnerParameterToCalculateOverallLossAfterEpoch=${learnerParameterToCalculateOverallLossAfterEpoch}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const outputModelFilename: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.outputModelFilename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const outputFeaturizerFilename: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.outputFeaturizerFilename;
|
||||
// -----------------------------------------------------------------------
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const linesToSkip: number = +args.linesToSkip;
|
||||
Utility.debuggingLog(
|
||||
`labelColumnIndex=${labelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`textColumnIndex=${textColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`weightColumnIndex=${weightColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`linesToSkip=${linesToSkip}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const data: Data = await DataUtility.LoadData(
|
||||
filename,
|
||||
null,
|
||||
true,
|
||||
filetype,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip);
|
||||
// -----------------------------------------------------------------------
|
||||
const crossValidator: CrossValidator =
|
||||
mainCrossValidatorWithData(
|
||||
data,
|
||||
numberOfCrossValidationFolds,
|
||||
learnerParameterEpochs,
|
||||
learnerParameterMiniBatchSize,
|
||||
learnerParameterL1Regularization,
|
||||
learnerParameterL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio,
|
||||
learnerParameterLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch);
|
||||
// -----------------------------------------------------------------------
|
||||
const evaluationJsonReportResult: {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} = crossValidator.generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = crossValidator.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
return { evaluationJsonReportResult, evaluationDataArraysReportResult };
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainCrossValidator().then(() => { return; });
|
||||
}
|
||||
|
|
@ -1,588 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { AppSoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { IMathematicsHelper } from "../../../mathematics/mathematics_helper/IMathematicsHelper";
|
||||
import { MathematicsHelper } from "../../../mathematics/mathematics_helper/MathematicsHelper";
|
||||
|
||||
import { IConfusionMatrix } from "../../../mathematics/confusion_matrix/IConfusionMatrix";
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseEvaluator } from "../abstract_base_evaluator/AbstractBaseEvaluator";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export class CrossValidator extends AbstractBaseEvaluator {
|
||||
|
||||
public static readonly MathematicsHelperObject: IMathematicsHelper =
|
||||
MathematicsHelper.GetMathematicsHelperObject();
|
||||
|
||||
public static defaultNumberOfCrossValidationFolds: number = 5;
|
||||
|
||||
protected numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds;
|
||||
|
||||
protected labelsCachedAfterCrossValidation: string[] = [];
|
||||
protected labelMapCachedAfterCrossValidation: { [id: string]: number } = {};
|
||||
protected numberLabelsCachedAfterCrossValidation: number = -1;
|
||||
protected numberFeaturesCachedAfterCrossValidation: number = -1;
|
||||
protected intentsCachedAfterCrossValidation: string[] = [];
|
||||
protected utterancesCachedAfterCrossValidation: string[] = [];
|
||||
protected labelIndexArrayCachedAfterCrossValidation: number[] = [];
|
||||
protected featureIndexArraysCachedAfterCrossValidation: number[][] = [];
|
||||
protected labelInstanceIndexMapArrayCachedAfterCrossValidation: Map<string, number[]> = new Map<string, number[]>();
|
||||
|
||||
protected learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs;
|
||||
protected learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize;
|
||||
protected learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization;
|
||||
protected learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization;
|
||||
protected learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio;
|
||||
protected learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate;
|
||||
protected learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true;
|
||||
|
||||
protected crossValidationResultCachedAfterCrossValidation: {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = {
|
||||
confusionMatrixCrossValidation: new ConfusionMatrix([], {}),
|
||||
thresholdReporterCrossValidation: new ThresholdReporter("", "", null, null, [], {}),
|
||||
predictionLabels: [],
|
||||
predictionLabelIndexes: [],
|
||||
instanceIndexes: [],
|
||||
groundTruthLabels: [],
|
||||
groundTruthLabelIndexes: [],
|
||||
predictions: [] };
|
||||
|
||||
constructor(
|
||||
numberOfCrossValidationFolds: number =
|
||||
CrossValidator.defaultNumberOfCrossValidationFolds,
|
||||
learnerParameterEpochs: number =
|
||||
AppSoftmaxRegressionSparse.defaultEpochs,
|
||||
learnerParameterMiniBatchSize: number =
|
||||
AppSoftmaxRegressionSparse.defaultMiniBatchSize,
|
||||
learnerParameterL1Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL1Regularization,
|
||||
learnerParameterL2Regularization: number =
|
||||
AppSoftmaxRegressionSparse.defaultL2Regularization,
|
||||
learnerParameterLossEarlyStopRatio: number =
|
||||
AppSoftmaxRegressionSparse.defaultLossEarlyStopRatio,
|
||||
learnerParameterLearningRate: number =
|
||||
AppSoftmaxRegressionSparse.defaultLearningRate,
|
||||
learnerParameterToCalculateOverallLossAfterEpoch: boolean =
|
||||
true) {
|
||||
super();
|
||||
this.numberOfCrossValidationFolds =
|
||||
numberOfCrossValidationFolds;
|
||||
this.learnerParameterEpochs =
|
||||
learnerParameterEpochs;
|
||||
this.learnerParameterMiniBatchSize =
|
||||
learnerParameterMiniBatchSize;
|
||||
this.learnerParameterL1Regularization =
|
||||
learnerParameterL1Regularization;
|
||||
this.learnerParameterL2Regularization =
|
||||
learnerParameterL2Regularization;
|
||||
this.learnerParameterLossEarlyStopRatio =
|
||||
learnerParameterLossEarlyStopRatio;
|
||||
this.learnerParameterLearningRate =
|
||||
learnerParameterLearningRate;
|
||||
this.learnerParameterToCalculateOverallLossAfterEpoch =
|
||||
learnerParameterToCalculateOverallLossAfterEpoch;
|
||||
}
|
||||
|
||||
public getCrossValidationResult(): {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
return this.crossValidationResultCachedAfterCrossValidation;
|
||||
}
|
||||
|
||||
public getNumberOfCrossValidationFolds(): number {
|
||||
return this.numberOfCrossValidationFolds;
|
||||
}
|
||||
|
||||
public createLearner(
|
||||
numberLabels: number,
|
||||
numberFeatures: number): SoftmaxRegressionSparse {
|
||||
const numberInputUnits: number = numberFeatures;
|
||||
const numberOutputUnits: number = numberLabels;
|
||||
const softmax: SoftmaxRegressionSparse = new SoftmaxRegressionSparse(
|
||||
numberInputUnits,
|
||||
numberOutputUnits,
|
||||
this.learnerParameterL1Regularization,
|
||||
this.learnerParameterL2Regularization,
|
||||
this.learnerParameterLossEarlyStopRatio);
|
||||
return softmax;
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
{
|
||||
const predictionLabels: string[] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.predictionLabels;
|
||||
const predictionLabelIndexes: number[] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.predictionLabelIndexes;
|
||||
const instanceIndexes: number[] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.instanceIndexes;
|
||||
const groundTruthLabels: string[] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.groundTruthLabels;
|
||||
const groundTruthLabelIndexes: number[] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.groundTruthLabelIndexes;
|
||||
const predictions: number[][] =
|
||||
this.crossValidationResultCachedAfterCrossValidation.predictions;
|
||||
const outputEvaluationReportDataArraysScoreRecords: string[][] = [];
|
||||
for (let index: number = 0; index < this.intentsCachedAfterCrossValidation.length; index++) {
|
||||
const instanceIndex: number = instanceIndexes[index];
|
||||
const intent: string = this.intentsCachedAfterCrossValidation[instanceIndex];
|
||||
const utterance: string = this.utterancesCachedAfterCrossValidation[instanceIndex];
|
||||
const groundTruthLabel: string = groundTruthLabels[index];
|
||||
const groundTruthLabelIndex: number = groundTruthLabelIndexes[index];
|
||||
const predictionLabel: string = predictionLabels[index];
|
||||
const predictionLabelIndex: number = predictionLabelIndexes[index];
|
||||
const outputEvaluationReportDataArraysScoreRecord: string[] = [];
|
||||
outputEvaluationReportDataArraysScoreRecord.push(intent);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(utterance);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(instanceIndex.toString());
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabelIndex.toString());
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabelIndex.toString());
|
||||
const prediction: number[] = predictions[index];
|
||||
for (const labelIndex of Array(prediction.length).keys()) {
|
||||
outputEvaluationReportDataArraysScoreRecord.push(prediction[labelIndex].toString());
|
||||
}
|
||||
outputEvaluationReportDataArraysScoreRecords.push(
|
||||
outputEvaluationReportDataArraysScoreRecord);
|
||||
}
|
||||
outputEvaluationReportDataArrays.CrossValidationScoreRecords =
|
||||
outputEvaluationReportDataArraysScoreRecords;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
const outputFilenames: string[] = [];
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_CrossValidationScoreRecords.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportDataArrays.CrossValidationScoreRecords,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
outputFilenames.push(outputFilename);
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_CrossValidationScoreRecordsLabel.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
[this.labelsCachedAfterCrossValidation],
|
||||
[],
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
outputFilenames.push(outputFilename);
|
||||
}
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
const outputEvaluationReport: IDictionaryStringIdGenericValue<any> = {};
|
||||
const confusionMatrixCrossValidation: ConfusionMatrix =
|
||||
this.crossValidationResultCachedAfterCrossValidation.confusionMatrixCrossValidation;
|
||||
const confusionMatrixMetricStructure: {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } } =
|
||||
confusionMatrixCrossValidation.generateConfusionMatrixMetricStructure();
|
||||
Utility.debuggingLog(
|
||||
`confusionMatrixCrossValidation.getMicroAverageMetrics()=` +
|
||||
`${confusionMatrixCrossValidation.getMicroAverageMetrics()}` +
|
||||
`,confusionMatrixCrossValidation.getMacroAverageMetrics()=` +
|
||||
`${confusionMatrixCrossValidation.getMacroAverageMetrics()}` +
|
||||
`,confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()=` +
|
||||
`${confusionMatrixCrossValidation.getWeightedMacroAverageMetrics()}`);
|
||||
outputEvaluationReport.confusionMatrixMetricStructure =
|
||||
confusionMatrixMetricStructure;
|
||||
return outputEvaluationReport;
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any> = {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(outputEvaluationReportJson)) {
|
||||
outputEvaluationReportJson =
|
||||
this.generateEvaluationJsonReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_CrossValidationConfusionMatrixReport.json`;
|
||||
outputFilename = this.dumpEvaluationJsonReportToFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportJson.confusionMatrixMetricStructure,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportJson, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
|
||||
public crossValidate(
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number },
|
||||
numberLabels: number,
|
||||
numberFeatures: number,
|
||||
intents: string[],
|
||||
utterances: string[],
|
||||
labelIndexArray: number[],
|
||||
featureIndexArrays: number[][],
|
||||
labelInstanceIndexMapArray: Map<string, number[]>): {
|
||||
"confusionMatrixCrossValidation": ConfusionMatrix,
|
||||
"thresholdReporterCrossValidation": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
// -------------------------------------------------------------------
|
||||
Utility.debuggingLog(`labelIndexArray.length=` +
|
||||
`${labelIndexArray.length}`);
|
||||
Utility.debuggingLog(`featureIndexArrays.length=` +
|
||||
`${featureIndexArrays.length}`);
|
||||
Utility.debuggingLog(`labelInstanceIndexMapArray.size=` +
|
||||
`${labelInstanceIndexMapArray.size}`);
|
||||
// -------------------------------------------------------------------
|
||||
this.labelsCachedAfterCrossValidation = labels;
|
||||
this.labelMapCachedAfterCrossValidation = labelMap;
|
||||
this.numberLabelsCachedAfterCrossValidation = numberLabels;
|
||||
this.numberFeaturesCachedAfterCrossValidation = numberFeatures;
|
||||
this.intentsCachedAfterCrossValidation = intents;
|
||||
this.utterancesCachedAfterCrossValidation = utterances;
|
||||
this.labelIndexArrayCachedAfterCrossValidation = labelIndexArray;
|
||||
this.featureIndexArraysCachedAfterCrossValidation = featureIndexArrays;
|
||||
this.labelInstanceIndexMapArrayCachedAfterCrossValidation = labelInstanceIndexMapArray;
|
||||
// -------------------------------------------------------------------
|
||||
const confusionMatrixCrossValidation =
|
||||
new ConfusionMatrix(labels, labelMap);
|
||||
const thresholdReporterCrossValidation: ThresholdReporter =
|
||||
new ThresholdReporter("", "", null, null, labels, labelMap);
|
||||
const predictions: number[][] =
|
||||
[];
|
||||
const predictionLabels: string[] =
|
||||
[];
|
||||
const predictionLabelIndexes: number[] =
|
||||
[];
|
||||
const instanceIndexes: number[] =
|
||||
[];
|
||||
const groundTruthLabels: string[] =
|
||||
[];
|
||||
const groundTruthLabelIndexes: number[] =
|
||||
[];
|
||||
const numberOfCrossValidationFolds: number =
|
||||
this.getNumberOfCrossValidationFolds();
|
||||
let numberAcrossFoldsPredictions: number = 0;
|
||||
let numberAcrossFoldsPredictionLabelMatches: number = 0;
|
||||
let numberAcrossFoldsPredictionLabelIndexMatches: number = 0;
|
||||
for (let fold: number = 0; fold < numberOfCrossValidationFolds; fold++) {
|
||||
// ---------------------------------------------------------------
|
||||
const learner: SoftmaxRegressionSparse =
|
||||
this.createLearner(numberLabels, numberFeatures);
|
||||
// ---------------------------------------------------------------
|
||||
const cvLabelDenseIndexArrayForTraining: number[] = [];
|
||||
const cvFeatureSparseIndexArraysForTraining: number[][] = [];
|
||||
const cvInstanceIndexDenseArrayForTesting: number[] = [];
|
||||
const cvLabelDenseArrayForTesting: string[] = [];
|
||||
const cvLabelDenseIndexArrayForTesting: number[] = [];
|
||||
const cvFeatureSparseIndexArraysForTesting: number[][] = [];
|
||||
for (const label of labelInstanceIndexMapArray.keys()) {
|
||||
const instanceIndexArrayPerLabel: number[] =
|
||||
labelInstanceIndexMapArray.get(label) as number[];
|
||||
const numberOfInstancesPerLabel: number =
|
||||
instanceIndexArrayPerLabel.length;
|
||||
const numberOfInstancesPerFold: number =
|
||||
Math.floor(numberOfInstancesPerLabel / numberOfCrossValidationFolds) + 1;
|
||||
const beginIndexPerFold: number =
|
||||
numberOfInstancesPerFold * fold;
|
||||
let endIndexPerFold: number =
|
||||
beginIndexPerFold + numberOfInstancesPerFold;
|
||||
if (beginIndexPerFold >= numberOfInstancesPerLabel) {
|
||||
continue;
|
||||
}
|
||||
if (endIndexPerFold > numberOfInstancesPerLabel) {
|
||||
endIndexPerFold = numberOfInstancesPerLabel;
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`label=${label}` +
|
||||
`,fold=${fold}` +
|
||||
`,numberOfCrossValidationFolds=${numberOfCrossValidationFolds}` +
|
||||
`,numberOfInstancesPerLabel=${numberOfInstancesPerLabel}` +
|
||||
`,beginIndexPerFold=${beginIndexPerFold}` +
|
||||
`,endIndexPerFold=${endIndexPerFold}`);
|
||||
for (let index = 0; index < beginIndexPerFold; index++) {
|
||||
const instanceIndex: number = instanceIndexArrayPerLabel[index];
|
||||
if ((instanceIndex === undefined) ||
|
||||
(instanceIndex < 0) ||
|
||||
(instanceIndex >= labelIndexArray.length) ||
|
||||
(instanceIndex >= featureIndexArrays.length)) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceLabelIndex: number =
|
||||
labelIndexArray[instanceIndex];
|
||||
if (instanceLabelIndex === undefined) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,instanceLabelIndex=${instanceLabelIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceFeatureIndexArray: number[] =
|
||||
featureIndexArrays[instanceIndex];
|
||||
cvLabelDenseIndexArrayForTraining.push(instanceLabelIndex);
|
||||
cvFeatureSparseIndexArraysForTraining.push(instanceFeatureIndexArray);
|
||||
}
|
||||
for (let index = beginIndexPerFold; index < endIndexPerFold; index++) {
|
||||
const instanceIndex: number = instanceIndexArrayPerLabel[index];
|
||||
if ((instanceIndex === undefined) ||
|
||||
(instanceIndex < 0) ||
|
||||
(instanceIndex >= labelIndexArray.length) ||
|
||||
(instanceIndex >= featureIndexArrays.length)) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceLabel: string =
|
||||
intents[instanceIndex];
|
||||
const instanceLabelIndex: number =
|
||||
labelIndexArray[instanceIndex];
|
||||
if (instanceLabelIndex === undefined) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,instanceLabelIndex=${instanceLabelIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceFeatureIndexArray: number[] =
|
||||
featureIndexArrays[instanceIndex];
|
||||
cvInstanceIndexDenseArrayForTesting.push(instanceIndex);
|
||||
cvLabelDenseArrayForTesting.push(instanceLabel);
|
||||
cvLabelDenseIndexArrayForTesting.push(instanceLabelIndex);
|
||||
cvFeatureSparseIndexArraysForTesting.push(instanceFeatureIndexArray);
|
||||
}
|
||||
for (let index = endIndexPerFold; index < numberOfInstancesPerLabel; index++) {
|
||||
const instanceIndex: number = instanceIndexArrayPerLabel[index];
|
||||
if ((instanceIndex === undefined) ||
|
||||
(instanceIndex < 0) ||
|
||||
(instanceIndex >= labelIndexArray.length) ||
|
||||
(instanceIndex >= featureIndexArrays.length)) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceLabelIndex: number =
|
||||
labelIndexArray[instanceIndex];
|
||||
if (instanceLabelIndex === undefined) {
|
||||
Utility.debuggingThrow(
|
||||
`label=${label}` +
|
||||
`,index=${index}` +
|
||||
`,instanceIndex=${instanceIndex}` +
|
||||
`,instanceLabelIndex=${instanceLabelIndex}` +
|
||||
`,labelIndexArray.length=${labelIndexArray.length}` +
|
||||
`,featureIndexArrays.length=${featureIndexArrays.length}`);
|
||||
}
|
||||
const instanceFeatureIndexArray: number[] =
|
||||
featureIndexArrays[instanceIndex];
|
||||
cvLabelDenseIndexArrayForTraining.push(instanceLabelIndex);
|
||||
cvFeatureSparseIndexArraysForTraining.push(instanceFeatureIndexArray);
|
||||
}
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`fold=${fold}` +
|
||||
`,numberOfCrossValidationFolds=${numberOfCrossValidationFolds}` +
|
||||
`,cvLabelDenseIndexArrayForTraining.length=${cvLabelDenseIndexArrayForTraining.length}` +
|
||||
`,cvLabelDenseIndexArrayForTesting.length=${cvLabelDenseIndexArrayForTesting.length}`);
|
||||
// ---------------------------------------------------------------
|
||||
if (this.learnerParameterMiniBatchSize <= 0) {
|
||||
learner.fit(
|
||||
cvFeatureSparseIndexArraysForTraining,
|
||||
cvLabelDenseIndexArrayForTraining,
|
||||
this.learnerParameterEpochs,
|
||||
this.learnerParameterLearningRate,
|
||||
this.learnerParameterToCalculateOverallLossAfterEpoch,
|
||||
false);
|
||||
} else {
|
||||
learner.fitMinibatching(
|
||||
cvFeatureSparseIndexArraysForTraining,
|
||||
cvLabelDenseIndexArrayForTraining,
|
||||
this.learnerParameterEpochs,
|
||||
this.learnerParameterMiniBatchSize,
|
||||
this.learnerParameterLearningRate,
|
||||
this.learnerParameterToCalculateOverallLossAfterEpoch,
|
||||
false,
|
||||
false);
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
for (let index = 0; index < cvLabelDenseIndexArrayForTesting.length; index++) {
|
||||
const instanceIndex: number =
|
||||
cvInstanceIndexDenseArrayForTesting[index];
|
||||
instanceIndexes.push(instanceIndex);
|
||||
const groundTruthLabel: string =
|
||||
cvLabelDenseArrayForTesting[index];
|
||||
groundTruthLabels.push(groundTruthLabel);
|
||||
const groundTruthLabelIndex: number =
|
||||
cvLabelDenseIndexArrayForTesting[index];
|
||||
groundTruthLabelIndexes.push(groundTruthLabelIndex);
|
||||
const instanceFeatureIndexArray: number[] =
|
||||
cvFeatureSparseIndexArraysForTesting[index];
|
||||
const prediction: number[] =
|
||||
learner.predict([instanceFeatureIndexArray])[0];
|
||||
predictions.push(prediction);
|
||||
const argMax: { "indexMax": number, "max": number } =
|
||||
CrossValidator.MathematicsHelperObject.getIndexOnFirstMaxEntry(prediction);
|
||||
const predictionLabelIndex: number =
|
||||
argMax.indexMax;
|
||||
predictionLabelIndexes.push(predictionLabelIndex);
|
||||
const predictionLabel: string =
|
||||
labels[predictionLabelIndex];
|
||||
predictionLabels.push(predictionLabel);
|
||||
confusionMatrixCrossValidation.addInstanceByLabelIndex(
|
||||
groundTruthLabelIndex,
|
||||
predictionLabelIndex);
|
||||
thresholdReporterCrossValidation.addInstance(
|
||||
prediction,
|
||||
groundTruthLabelIndex,
|
||||
"",
|
||||
`${index}`,
|
||||
1);
|
||||
{
|
||||
numberAcrossFoldsPredictions++;
|
||||
if (predictionLabel === groundTruthLabel) {
|
||||
numberAcrossFoldsPredictionLabelMatches++;
|
||||
}
|
||||
if (predictionLabelIndex === groundTruthLabelIndex) {
|
||||
numberAcrossFoldsPredictionLabelIndexMatches++;
|
||||
}
|
||||
}
|
||||
}
|
||||
// ---------------------------------------------------------------
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
{
|
||||
Utility.debuggingLog(
|
||||
`numberAcrossFoldsPredictions=${numberAcrossFoldsPredictions}` +
|
||||
`,numberAcrossFoldsPredictionLabelMatches=${numberAcrossFoldsPredictionLabelMatches}` +
|
||||
`,numberAcrossFoldsPredictionLabelIndexMatches=${numberAcrossFoldsPredictionLabelIndexMatches}` +
|
||||
`,numberAcrossFoldsPredictionLabelMatchRatio=${numberAcrossFoldsPredictionLabelMatches / numberAcrossFoldsPredictions}` +
|
||||
`,numberAcrossFoldsPredictionLabelIndexMatchRatio=${numberAcrossFoldsPredictionLabelIndexMatches / numberAcrossFoldsPredictions}`);
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
this.crossValidationResultCachedAfterCrossValidation = {
|
||||
confusionMatrixCrossValidation,
|
||||
thresholdReporterCrossValidation,
|
||||
predictionLabels,
|
||||
predictionLabelIndexes,
|
||||
instanceIndexes,
|
||||
groundTruthLabels,
|
||||
groundTruthLabelIndexes,
|
||||
predictions };
|
||||
return this.crossValidationResultCachedAfterCrossValidation;
|
||||
// -------------------------------------------------------------------
|
||||
}
|
||||
}
|
||||
|
|
@ -1,209 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import assert = require("assert");
|
||||
|
||||
// tslint:disable-next-line: no-var-requires
|
||||
const readlineSync = require("readline-sync");
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { Predictor } from "./Predictor";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export function mainPredictor(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppPredictor",
|
||||
version: "0.0.1",
|
||||
});
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-f", "--filename"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "an input data file",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: true,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-t", "--filetype"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "data file type",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
parser.addArgument(
|
||||
["-m", "--modelFilename"],
|
||||
{
|
||||
help: "model file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-x", "--featurizerFilename"],
|
||||
{
|
||||
help: "serialized featurizer file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const filename: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- if (!Utility.exists(filename)) {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingThrow(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `The input dataset file ${filename} does not exist!` +
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ` process.cwd()=${process.cwd()}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- }
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const filetype: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.filetype;
|
||||
const modelFilename: string =
|
||||
args.modelFilename;
|
||||
if (!Utility.exists(modelFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input model file ${modelFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const featurizerFilename: string =
|
||||
args.featurizerFilename;
|
||||
if (!Utility.exists(featurizerFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input featurizer file ${featurizerFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(modelFilename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `filename=${filename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
Utility.debuggingLog(
|
||||
`modelFilename=${modelFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`featurizerFilename=${featurizerFilename}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const predictor: Predictor =
|
||||
new Predictor(
|
||||
modelFilename,
|
||||
featurizerFilename);
|
||||
// -----------------------------------------------------------------------
|
||||
const modelNullable: SoftmaxRegressionSparse|null =
|
||||
predictor.getModel();
|
||||
if (modelNullable === null) {
|
||||
Utility.debuggingThrow("model is null");
|
||||
}
|
||||
const featurizerNullable: NgramSubwordFeaturizer|null =
|
||||
predictor.getFeaturizer();
|
||||
if (featurizerNullable === null) {
|
||||
Utility.debuggingThrow("featurizer is null");
|
||||
}
|
||||
const model: SoftmaxRegressionSparse =
|
||||
modelNullable as SoftmaxRegressionSparse;
|
||||
const featurizer: NgramSubwordFeaturizer =
|
||||
featurizerNullable as NgramSubwordFeaturizer;
|
||||
// -------------------------------------------------------------------
|
||||
const labels: string[] = featurizer.getLabels();
|
||||
const labelMap: { [id: string]: number } = featurizer.getLabelMap();
|
||||
// const numberLabels: number = featurizer.getNumberLabels();
|
||||
// const numberFeatures: number = featurizer.getNumberFeatures();
|
||||
// -----------------------------------------------------------------------
|
||||
const confusionMatrixPrediction: ConfusionMatrix =
|
||||
new ConfusionMatrix(labels, labelMap);
|
||||
const thresholdReporterPredict: ThresholdReporter =
|
||||
new ThresholdReporter("", "", null, null, labels, labelMap);
|
||||
// -----------------------------------------------------------------------
|
||||
while (true) {
|
||||
const utterance: string =
|
||||
readlineSync.question("Please enter an utterance? [empty to end the loop] ");
|
||||
Utility.debuggingLog(
|
||||
`utterance=${utterance}`);
|
||||
if (Utility.isEmptyString(utterance)) {
|
||||
break;
|
||||
}
|
||||
const intent: string =
|
||||
readlineSync.question("Please enter intent for the utterance? ");
|
||||
Utility.debuggingLog(
|
||||
`intent=${intent}`);
|
||||
const predictionResult: {
|
||||
"confusionMatrixPrediction": ConfusionMatrix
|
||||
"thresholdReporterPrediction": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = predictor.predict(
|
||||
utterance,
|
||||
intent);
|
||||
Utility.debuggingLog(
|
||||
`groundTruthLabel=${predictionResult.groundTruthLabels[predictionResult.groundTruthLabels.length - 1]}` +
|
||||
`, predictionLabel=${predictionResult.predictionLabels[predictionResult.predictionLabels.length - 1]}` +
|
||||
`, groundTruthLabelIndex=${predictionResult.groundTruthLabelIndexes[predictionResult.groundTruthLabelIndexes.length - 1]}, ` +
|
||||
`, predictionLabelIndex=${predictionResult.predictionLabelIndexes[predictionResult.predictionLabelIndexes.length - 1]}`);
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const evaluationJsonReportResult: {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} = predictor.generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = predictor.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainPredictor();
|
||||
}
|
||||
|
|
@ -1,326 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { IMathematicsHelper } from "../../../mathematics/mathematics_helper/IMathematicsHelper";
|
||||
import { MathematicsHelper } from "../../../mathematics/mathematics_helper/MathematicsHelper";
|
||||
|
||||
import { IConfusionMatrix } from "../../../mathematics/confusion_matrix/IConfusionMatrix";
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseModelFeaturizerEvaluator } from "../abstract_base_evaluator/AbstractBaseModelFeaturizerEvaluator";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export class Predictor extends AbstractBaseModelFeaturizerEvaluator {
|
||||
|
||||
public static readonly MathematicsHelperObject: IMathematicsHelper =
|
||||
MathematicsHelper.GetMathematicsHelperObject();
|
||||
|
||||
protected labels: string[] = [];
|
||||
protected labelMap: { [id: string]: number } = {};
|
||||
|
||||
protected intents: string[] = [];
|
||||
protected utterances: string[] = [];
|
||||
protected labelIndexArray: number[] = [];
|
||||
protected featureIndexArrays: number[][] = [];
|
||||
|
||||
protected confusionMatrixPrediction: ConfusionMatrix =
|
||||
new ConfusionMatrix([], {});
|
||||
protected thresholdReporterPrediction: ThresholdReporter =
|
||||
new ThresholdReporter("", "", null, null, [], {});
|
||||
protected predictionLabels: string[] = [];
|
||||
protected predictionLabelIndexes: number[] = [];
|
||||
protected instanceIndexes: number[] = [];
|
||||
protected groundTruthLabels: string[] = [];
|
||||
protected groundTruthLabelIndexes: number[] = [];
|
||||
protected predictions: number[][] = [];
|
||||
|
||||
constructor(
|
||||
modelFilename: string,
|
||||
featurizerFilename: string) {
|
||||
super(modelFilename, featurizerFilename, null, null, [], {});
|
||||
this.labels = this.getFeaturizer().getLabels();
|
||||
this.labelMap = this.getFeaturizer().getLabelMap();
|
||||
this.confusionMatrixPrediction =
|
||||
new ConfusionMatrix(this.labels, this.labelMap);
|
||||
this.thresholdReporterPrediction =
|
||||
new ThresholdReporter("", "", null, null, this.labels, this.labelMap);
|
||||
}
|
||||
|
||||
public getPredictionResult(): {
|
||||
"confusionMatrixPrediction": ConfusionMatrix
|
||||
"thresholdReporterPrediction": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
return {
|
||||
confusionMatrixPrediction: this.confusionMatrixPrediction,
|
||||
thresholdReporterPrediction: this.thresholdReporterPrediction,
|
||||
predictionLabels: this.predictionLabels,
|
||||
predictionLabelIndexes: this.predictionLabelIndexes,
|
||||
instanceIndexes: this.instanceIndexes,
|
||||
groundTruthLabels: this.groundTruthLabels,
|
||||
groundTruthLabelIndexes: this.groundTruthLabelIndexes,
|
||||
predictions: this.predictions,
|
||||
};
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
const predictionResult: {
|
||||
"confusionMatrixPrediction": ConfusionMatrix
|
||||
"thresholdReporterPrediction": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } =
|
||||
this.getPredictionResult();
|
||||
{
|
||||
const predictionLabels: string[] = predictionResult.predictionLabels;
|
||||
const predictionLabelIndexes: number[] = predictionResult.predictionLabelIndexes;
|
||||
const instanceIndexes: number[] = predictionResult.instanceIndexes;
|
||||
const groundTruthLabels: string[] = predictionResult.predictionLabels;
|
||||
const groundTruthLabelIndexes: number[] = predictionResult.predictionLabelIndexes;
|
||||
const predictions: number[][] = predictionResult.predictions;
|
||||
const outputEvaluationReportDataArraysScoreRecords: string[][] = [];
|
||||
for (let index: number = 0; index < this.intents.length; index++) {
|
||||
const instanceIndex: number = instanceIndexes[index];
|
||||
const intent: string = this.intents[instanceIndex];
|
||||
const utterance: string = this.utterances[instanceIndex];
|
||||
const groundTruthLabel: string = groundTruthLabels[index];
|
||||
const groundTruthLabelIndex: number = groundTruthLabelIndexes[index];
|
||||
const predictionLabel: string = predictionLabels[index];
|
||||
const predictionLabelIndex: number = predictionLabelIndexes[index];
|
||||
const outputEvaluationReportDataArraysScoreRecord: string[] = [];
|
||||
outputEvaluationReportDataArraysScoreRecord.push(intent);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(utterance);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabelIndex.toString());
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabelIndex.toString());
|
||||
const prediction: number[] = predictions[index];
|
||||
for (const labelIndex of Array(prediction.length).keys()) {
|
||||
outputEvaluationReportDataArraysScoreRecord.push(prediction[labelIndex].toString());
|
||||
}
|
||||
outputEvaluationReportDataArraysScoreRecords.push(
|
||||
outputEvaluationReportDataArraysScoreRecord);
|
||||
}
|
||||
outputEvaluationReportDataArrays.PredictionScoreRecords =
|
||||
outputEvaluationReportDataArraysScoreRecords;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_PredictionScoreRecords.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportDataArrays.PredictionScoreRecords,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
const outputEvaluationReport: IDictionaryStringIdGenericValue<any> = {};
|
||||
const predictionResult: {
|
||||
"confusionMatrixPrediction": ConfusionMatrix
|
||||
"thresholdReporterPrediction": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } =
|
||||
this.getPredictionResult();
|
||||
const confusionMatrixPrediction: ConfusionMatrix =
|
||||
predictionResult.confusionMatrixPrediction;
|
||||
const confusionMatrixMetricStructure: {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } } =
|
||||
confusionMatrixPrediction.generateConfusionMatrixMetricStructure();
|
||||
Utility.debuggingLog(
|
||||
`confusionMatrixPrediction.getMicroAverageMetrics()=` +
|
||||
`${confusionMatrixPrediction.getMicroAverageMetrics()}` +
|
||||
`,confusionMatrixPrediction.getMacroAverageMetrics()=` +
|
||||
`${confusionMatrixPrediction.getMacroAverageMetrics()}` +
|
||||
`,confusionMatrixPrediction.getWeightedMacroAverageMetrics()=` +
|
||||
`${confusionMatrixPrediction.getWeightedMacroAverageMetrics()}`);
|
||||
outputEvaluationReport.confusionMatrixMetricStructure =
|
||||
confusionMatrixMetricStructure;
|
||||
return outputEvaluationReport;
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any> = {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(outputEvaluationReportJson)) {
|
||||
outputEvaluationReportJson =
|
||||
this.generateEvaluationJsonReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_PredictionConfusionMatrixReport.json`;
|
||||
outputFilename = this.dumpEvaluationJsonReportToFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportJson.confusionMatrixMetricStructure,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportJson, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
|
||||
public predict(
|
||||
utterance: string,
|
||||
intent: string): {
|
||||
"confusionMatrixPrediction": ConfusionMatrix
|
||||
"thresholdReporterPrediction": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
// -------------------------------------------------------------------
|
||||
this.intents.push(intent);
|
||||
this.utterances.push(utterance);
|
||||
// -------------------------------------------------------------------
|
||||
const model: SoftmaxRegressionSparse =
|
||||
this.getModel();
|
||||
const featurizer: NgramSubwordFeaturizer =
|
||||
this.getFeaturizer();
|
||||
// -------------------------------------------------------------------
|
||||
const labels: string[] = this.getLabels();
|
||||
const labelMap: { [id: string]: number } = this.getLabelMap();
|
||||
// const numberLabels: number = featurizer.getNumberLabels();
|
||||
// const numberFeatures: number = featurizer.getNumberFeatures();
|
||||
// -------------------------------------------------------------------
|
||||
const featureIndexArray: number[] =
|
||||
featurizer.createFeatureSparseIndexArray(utterance);
|
||||
if (Utility.isEmptyNumberArray(featureIndexArray)) {
|
||||
Utility.debuggingThrow(
|
||||
`featureIndexArray is empty`);
|
||||
}
|
||||
const predictions: number[][] =
|
||||
model.predict([featureIndexArray]);
|
||||
{
|
||||
// ---------------------------------------------------------------
|
||||
const prediction: number[] = predictions[0];
|
||||
// ---------------------------------------------------------------
|
||||
const argMax: { "indexMax": number, "max": number } =
|
||||
Predictor.MathematicsHelperObject.getIndexOnFirstMaxEntry(prediction);
|
||||
const predictionLabelIndex: number =
|
||||
argMax.indexMax;
|
||||
let groundTruthLabelIndex: number = labelMap[intent];
|
||||
if (!Utility.isEmptyString(intent)) {
|
||||
if (!groundTruthLabelIndex) {
|
||||
groundTruthLabelIndex = -1;
|
||||
}
|
||||
if (groundTruthLabelIndex >= 0) {
|
||||
this.confusionMatrixPrediction.addInstanceByLabelIndex(
|
||||
groundTruthLabelIndex,
|
||||
predictionLabelIndex);
|
||||
this.thresholdReporterPrediction.addInstance(
|
||||
prediction,
|
||||
groundTruthLabelIndex,
|
||||
utterance,
|
||||
`${this.thresholdReporterPrediction.getNumberInstances()}`,
|
||||
1);
|
||||
}
|
||||
}
|
||||
const predictionLabel: string = labels[predictionLabelIndex];
|
||||
const groundTruthLabel: string = (groundTruthLabelIndex >= 0) ? labels[groundTruthLabelIndex] : "";
|
||||
// ---------------------------------------------------------------
|
||||
this.labelIndexArray.push(groundTruthLabelIndex);
|
||||
this.featureIndexArrays.push(featureIndexArray);
|
||||
// ---------------------------------------------------------------
|
||||
this.predictionLabels.push(predictionLabel);
|
||||
this.predictionLabelIndexes.push(predictionLabelIndex);
|
||||
this.instanceIndexes.push(this.instanceIndexes.length);
|
||||
this.groundTruthLabels.push(groundTruthLabel);
|
||||
this.groundTruthLabelIndexes.push(groundTruthLabelIndex);
|
||||
this.predictions.push(prediction);
|
||||
// ---------------------------------------------------------------
|
||||
return this.getPredictionResult();
|
||||
// ---------------------------------------------------------------
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,175 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
import { DataProfileReporter } from "../report/DataProfileReporter";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DataUtility } from "../../../data/DataUtility";
|
||||
import { Data } from "../../../data/Data";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
export function mainDataProfileReporter(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppDataProfileReporter",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "an input data file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-t", "--filetype"],
|
||||
{
|
||||
help: "data file type",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file name",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--linesToSkip"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string =
|
||||
args.filename;
|
||||
if (!Utility.exists(filename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input score file ${filename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const filetype: string =
|
||||
args.filetype;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const featurizerFilename: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.featurizerFilename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- if (!Utility.exists(featurizerFilename)) {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingThrow(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `The input featurizer file ${featurizerFilename} ` +
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `does not exist! process.cwd()=${process.cwd()}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- }
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(filename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const linesToSkip: number = +args.linesToSkip;
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `featurizerFilename=${featurizerFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
Utility.debuggingLog(
|
||||
`labelColumnIndex=${labelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`textColumnIndex=${textColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`weightColumnIndex=${weightColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`linesToSkip=${linesToSkip}`);
|
||||
// -----------------------------------------------------------------------
|
||||
DataUtility.LoadData(
|
||||
filename,
|
||||
null,
|
||||
true,
|
||||
filetype,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip).then((data) => {
|
||||
// ---------------------------------------------------------------
|
||||
const dataProfileReporter: DataProfileReporter =
|
||||
new DataProfileReporter(data);
|
||||
// ---------------------------------------------------------------
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = dataProfileReporter.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// ---------------------------------------------------------------
|
||||
});
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainDataProfileReporter();
|
||||
}
|
||||
|
|
@ -1,193 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
import { ModelMetaDataProfileReporter } from "./ModelMetaDataProfileReporter";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { DataUtility } from "../../../data/DataUtility";
|
||||
|
||||
import { Data } from "../../../data/Data";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export function mainModelMetaDataProfileReporter(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppModelMetaDataProfileReporter",
|
||||
version: "0.0.1",
|
||||
});
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-f", "--filename"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "an input data file",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: true,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-t", "--filetype"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "data file type",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
parser.addArgument(
|
||||
["-m", "--modelFilename"],
|
||||
{
|
||||
help: "model file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-x", "--featurizerFilename"],
|
||||
{
|
||||
help: "serialized featurizer file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file name",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-li", "--labelColumnIndex"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- defaultValue: 0,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "label column index",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-ti", "--textColumnIndex"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- defaultValue: 1,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "text/utterance column index",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-wi", "--weightColumnIndex"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- defaultValue: -1,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "weight column index",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- parser.addArgument(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- ["-ls", "--linesToSkip"],
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- defaultValue: 0,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- help: "number of lines to skip from the input file",
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- required: false,
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- },
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- );
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const filename: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.filename;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- if (!Utility.exists(filename)) {
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingThrow(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `The input score file ${filename} does not exist! ` +
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `process.cwd()=${process.cwd()}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- }
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const filetype: string =
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- args.filetype;
|
||||
const modelFilename: string =
|
||||
args.modelFilename;
|
||||
if (!Utility.exists(modelFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input model file ${modelFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const featurizerFilename: string =
|
||||
args.featurizerFilename;
|
||||
if (!Utility.exists(featurizerFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input featurizer file ${featurizerFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(modelFilename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const textColumnIndex: number = +args.textColumnIndex;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- const linesToSkip: number = +args.linesToSkip;
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `filename=${filename}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `filetype=${filetype}`);
|
||||
Utility.debuggingLog(
|
||||
`modelFilename=${modelFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`featurizerFilename=${featurizerFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `labelColumnIndex=${labelColumnIndex}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `textColumnIndex=${textColumnIndex}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `weightColumnIndex=${weightColumnIndex}`);
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- Utility.debuggingLog(
|
||||
// ---- NOTE-TODO-PLACEHOLDER ---- `linesToSkip=${linesToSkip}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const modelMetaDataProfileReporter: ModelMetaDataProfileReporter =
|
||||
new ModelMetaDataProfileReporter(
|
||||
modelFilename,
|
||||
featurizerFilename,
|
||||
null,
|
||||
null,
|
||||
[],
|
||||
{});
|
||||
// -----------------------------------------------------------------------
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = modelMetaDataProfileReporter.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainModelMetaDataProfileReporter();
|
||||
}
|
||||
|
|
@ -1,243 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
export function mainThresholdReporter(): void {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppThresholdReporter",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--scoreFilename"],
|
||||
{
|
||||
help: "an input score file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-si", "--scoreColumnBeginIndex"],
|
||||
{
|
||||
help: "score column begin index",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-l", "--labelFilename"],
|
||||
{
|
||||
defaultValue: "",
|
||||
help: "an input label file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-x", "--featurizerFilename"],
|
||||
{
|
||||
defaultValue: "",
|
||||
help: "serialized featurizer file, we can use the label information from a featurizer if provided",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ii", "--identifierColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "identifier column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-pli", "--predictedLabelColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "predicted label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-rti", "--revisedTextColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "revised text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--lineIndexToStart"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const scoreFilename: string =
|
||||
args.scoreFilename;
|
||||
if (!Utility.exists(scoreFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input score file ${scoreFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const featurizerFilename: string =
|
||||
args.featurizerFilename;
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- if (!Utility.exists(featurizerFilename)) {
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- Utility.debuggingThrow(
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- `The input featurizer file ${featurizerFilename}` +
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- ` does not exist! ` +
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- `process.cwd()=${process.cwd()}`);
|
||||
// ---- NOTE-MAY-NOT-NEED-A-FEATURIZER-FOR-labelMap ---- }
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(scoreFilename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`scoreFilename=${scoreFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`featurizerFilename=${featurizerFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const labelFilename: string =
|
||||
args.labelFilename;
|
||||
Utility.debuggingLog(
|
||||
`labelFilename=${labelFilename}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const identifierColumnIndex: number = +args.identifierColumnIndex;
|
||||
const scoreColumnBeginIndex: number = +args.scoreColumnBeginIndex;
|
||||
const predictedLabelColumnIndex: number = +args.predictedLabelColumnIndex;
|
||||
const revisedTextColumnIndex: number = +args.revisedTextColumnIndex;
|
||||
const lineIndexToStart: number = +args.lineIndexToStart;
|
||||
Utility.debuggingLog(
|
||||
`labelColumnIndex=${labelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`textColumnIndex=${textColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`weightColumnIndex=${weightColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`identifierColumnIndex=${identifierColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`scoreColumnBeginIndex=${scoreColumnBeginIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`predictedLabelColumnIndex=${predictedLabelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`revisedTextColumnIndex=${revisedTextColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`lineIndexToStart=${lineIndexToStart}`);
|
||||
// -----------------------------------------------------------------------
|
||||
let labels: string[] = [];
|
||||
let labelMap: { [id: string]: number } = {};
|
||||
if (!Utility.isEmptyString(labelFilename)) {
|
||||
const labelsAndLabelMap: { "stringArray": string[], "stringMap": { [id: string]: number } } =
|
||||
DictionaryMapUtility.buildStringIdNumberValueDictionaryFromUniqueStringArrayFile(labelFilename);
|
||||
labels = labelsAndLabelMap.stringArray;
|
||||
labelMap = labelsAndLabelMap.stringMap;
|
||||
}
|
||||
// -----------------------------------------------------------------------
|
||||
const thresholdReporter: ThresholdReporter =
|
||||
new ThresholdReporter(
|
||||
"",
|
||||
featurizerFilename,
|
||||
null,
|
||||
null,
|
||||
labels,
|
||||
labelMap);
|
||||
// -----------------------------------------------------------------------
|
||||
thresholdReporter.loadScoreFileAndPopulate(
|
||||
scoreFilename,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
scoreColumnBeginIndex,
|
||||
identifierColumnIndex,
|
||||
predictedLabelColumnIndex,
|
||||
revisedTextColumnIndex,
|
||||
lineIndexToStart);
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = thresholdReporter.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainThresholdReporter();
|
||||
}
|
||||
|
|
@ -1,130 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { AbstractBaseDataProfileEvaluator } from "../abstract_base_evaluator/AbstractBaseDataProfileEvaluator";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ListArrayUtility } from "../../../utility/ListArrayUtility";
|
||||
|
||||
import { Data } from "../../../data/Data";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export class DataProfileReporter extends AbstractBaseDataProfileEvaluator {
|
||||
|
||||
constructor(
|
||||
data: Data) {
|
||||
super(data);
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
{
|
||||
const outputEvaluationReportLabelDistributionDataArrays: string[][] = [];
|
||||
const intentInstanceIndexMapArray: Map<string, number[]> =
|
||||
this.getData().getIntentInstanceIndexMapArray();
|
||||
for (const entry of intentInstanceIndexMapArray.entries()) {
|
||||
const entryKey: string = entry[0];
|
||||
const entryValueLength: number = entry[1].length;
|
||||
outputEvaluationReportLabelDistributionDataArrays.push([entryKey, entryValueLength.toString()]);
|
||||
}
|
||||
outputEvaluationReportDataArrays.DataProfileIntentLabelDistribution =
|
||||
outputEvaluationReportLabelDistributionDataArrays;
|
||||
}
|
||||
{
|
||||
const outputEvaluationReportLabelDistributionDataArrays: string[][] = [];
|
||||
const entityTypeInstanceIndexMapArray: Map<string, number[]> =
|
||||
this.getData().getEntityTypeInstanceIndexMapArray();
|
||||
for (const entry of entityTypeInstanceIndexMapArray.entries()) {
|
||||
const entryKey: string = entry[0];
|
||||
const entryValueLength: number = entry[1].length;
|
||||
outputEvaluationReportLabelDistributionDataArrays.push([entryKey, entryValueLength.toString()]);
|
||||
}
|
||||
outputEvaluationReportDataArrays.DataProfileEntityTypeLabelDistribution =
|
||||
outputEvaluationReportLabelDistributionDataArrays;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
const outputFilenames: string[] = [];
|
||||
{
|
||||
let outputFilenameIntentLabelDistribution: string =
|
||||
`${outputReportFilenamePrefix}_DataProfileIntentLabelDistribution.txt`;
|
||||
outputFilenameIntentLabelDistribution = Utility.storeDataArraysToTsvFile(
|
||||
outputFilenameIntentLabelDistribution,
|
||||
outputEvaluationReportDataArrays.DataProfileIntentLabelDistribution,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
outputFilenames.push(outputFilenameIntentLabelDistribution);
|
||||
}
|
||||
{
|
||||
let outputFilenameEntityTypeLabelDistribution: string =
|
||||
`${outputReportFilenamePrefix}_DataProfileEntityTypeLabelDistribution.txt`;
|
||||
outputFilenameEntityTypeLabelDistribution = Utility.storeDataArraysToTsvFile(
|
||||
outputFilenameEntityTypeLabelDistribution,
|
||||
outputEvaluationReportDataArrays.DataProfileEntityTypeLabelDistribution,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
outputFilenames.push(outputFilenameEntityTypeLabelDistribution);
|
||||
}
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any>= {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
const outputFilenames: string[] = [];
|
||||
// ---- NOTE: nothing to report here.
|
||||
return {
|
||||
outputEvaluationReportJson,
|
||||
outputFilenames,
|
||||
};
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,119 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { AbstractBaseModelFeaturizerEvaluator } from "../abstract_base_evaluator/AbstractBaseModelFeaturizerEvaluator";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ListArrayUtility } from "../../../utility/ListArrayUtility";
|
||||
|
||||
import { Data } from "../../../data/Data";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export class ModelMetaDataProfileReporter extends AbstractBaseModelFeaturizerEvaluator {
|
||||
|
||||
constructor(
|
||||
modelFilename: string,
|
||||
featurizerFilename: string,
|
||||
modelNullable: SoftmaxRegressionSparse|null,
|
||||
featurizerNullable: NgramSubwordFeaturizer|null,
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number }) {
|
||||
super(
|
||||
modelFilename,
|
||||
featurizerFilename,
|
||||
modelNullable,
|
||||
featurizerNullable,
|
||||
labels,
|
||||
labelMap);
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
{
|
||||
const outputEvaluationReportDataArraysLabelBiases: string[][] = [];
|
||||
const labels: string[] =
|
||||
this.getLabels();
|
||||
const modelBiases: number[] =
|
||||
this.getModel().getModelBiases();
|
||||
for (let labelIndex: number = 0; labelIndex < modelBiases.length; labelIndex++) {
|
||||
const label: string = labels[labelIndex];
|
||||
const bias: number = modelBiases[labelIndex];
|
||||
outputEvaluationReportDataArraysLabelBiases.push([label, bias.toString()]);
|
||||
}
|
||||
outputEvaluationReportDataArrays.ModelMetaDataProfileLabelBiases =
|
||||
outputEvaluationReportDataArraysLabelBiases;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_ModelMetaDataProfileLabelBiases.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportDataArrays.ModelMetaDataProfileLabelBiases,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any>= {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
const outputFilenames: string[] = [];
|
||||
// ---- NOTE: nothing to report here.
|
||||
return {
|
||||
outputEvaluationReportJson,
|
||||
outputFilenames,
|
||||
};
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
}
|
||||
|
|
@ -1,494 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { AbstractBaseModelFeaturizerEvaluator } from "../abstract_base_evaluator/AbstractBaseModelFeaturizerEvaluator";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ListArrayUtility } from "../../../utility/ListArrayUtility";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
const NumberOfLabelsPerBatchToReport: number = 8;
|
||||
const EnableDebuggingInstrumentation: boolean = false;
|
||||
|
||||
export class ThresholdReporter extends AbstractBaseModelFeaturizerEvaluator {
|
||||
|
||||
protected instancePredictedScoreArrays: number[][] = [];
|
||||
protected instanceGroudTruthLabelIds: number[] = [];
|
||||
protected instanceFeatureTexts: string[] = [];
|
||||
protected instanceIdentifiers: string[] = [];
|
||||
protected instanceWeights: number[] = [];
|
||||
|
||||
protected targetLabelBatchesToReport: string[][] = [];
|
||||
protected numberOfLabelsPerBatch: number = NumberOfLabelsPerBatchToReport;
|
||||
protected explicitTotal: number = -1;
|
||||
protected explicitTotalPositives: number = -1;
|
||||
protected explicitTotalNegatives: number = -1;
|
||||
protected fMeasureBeta: number = 1;
|
||||
protected effectivenessMeasureAlpha: number = 0.5;
|
||||
protected rejectRate: number = -1;
|
||||
protected alpha: number = 1;
|
||||
protected beta: number = 1;
|
||||
protected A: number = 1;
|
||||
protected B: number = 1;
|
||||
protected explicitMaxPositives: number = -1;
|
||||
|
||||
constructor(
|
||||
modelFilename: string,
|
||||
featurizerFilename: string,
|
||||
modelNullable: SoftmaxRegressionSparse|null,
|
||||
featurizerNullable: NgramSubwordFeaturizer|null,
|
||||
labels: string[],
|
||||
labelMap: { [id: string]: number },
|
||||
targetLabelBatchesToReport: string[][] = [],
|
||||
numberOfLabelsPerBatch: number = NumberOfLabelsPerBatchToReport,
|
||||
explicitTotal: number = -1,
|
||||
explicitTotalPositives: number = -1,
|
||||
explicitTotalNegatives: number = -1,
|
||||
fMeasureBeta: number = 1,
|
||||
effectivenessMeasureAlpha: number = 0.5,
|
||||
rejectRate: number = -1,
|
||||
alpha: number = 1,
|
||||
beta: number = 1,
|
||||
A: number = 1,
|
||||
B: number = 1,
|
||||
explicitMaxPositives: number = -1) {
|
||||
super(modelFilename, featurizerFilename, modelNullable, featurizerNullable, labels, labelMap);
|
||||
this.targetLabelBatchesToReport = targetLabelBatchesToReport;
|
||||
this.numberOfLabelsPerBatch = numberOfLabelsPerBatch;
|
||||
this.explicitTotal = explicitTotal;
|
||||
this.explicitTotalPositives = explicitTotalPositives;
|
||||
this.explicitTotalNegatives = explicitTotalNegatives;
|
||||
this.fMeasureBeta = fMeasureBeta;
|
||||
this.effectivenessMeasureAlpha = effectivenessMeasureAlpha;
|
||||
this.rejectRate = rejectRate;
|
||||
this.alpha = alpha;
|
||||
this.beta = beta;
|
||||
this.A = A;
|
||||
this.B = B;
|
||||
this.explicitMaxPositives = explicitMaxPositives;
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
{
|
||||
const outputEvaluationReportDataArrayConfusionMatrixThresholdMetrics: string[][] =
|
||||
this.reportToDataArrays();
|
||||
outputEvaluationReportDataArrays.ConfusionMatrixThresholdMetrics =
|
||||
outputEvaluationReportDataArrayConfusionMatrixThresholdMetrics;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_ConfusionMatrixThresholdMetrics.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportDataArrays.ConfusionMatrixThresholdMetrics,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any> = {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
const outputFilenames: string[] = [];
|
||||
// ---- NOTE: nothing to report here.
|
||||
return {
|
||||
outputEvaluationReportJson,
|
||||
outputFilenames,
|
||||
};
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
|
||||
public getNumberInstances(): number {
|
||||
return this.instancePredictedScoreArrays.length;
|
||||
}
|
||||
|
||||
public addInstance(
|
||||
instancePredictedScoreArray: number[],
|
||||
instanceGroudTruthLabelId: number,
|
||||
instanceFeatureText: string,
|
||||
instanceIdentifier: string,
|
||||
instanceWeight: number): void {
|
||||
this.instancePredictedScoreArrays.push(instancePredictedScoreArray);
|
||||
this.instanceGroudTruthLabelIds.push(instanceGroudTruthLabelId);
|
||||
this.instanceFeatureTexts.push(instanceFeatureText);
|
||||
this.instanceIdentifiers.push(instanceIdentifier);
|
||||
this.instanceWeights.push(instanceWeight);
|
||||
}
|
||||
|
||||
public reportToDataArrays(): string[][] {
|
||||
const reportResults: Array<Array<{
|
||||
"targetLabel": string,
|
||||
"scoreThresholds": number[],
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"metricNames": string[],
|
||||
"metricNameMap": IDictionaryStringIdGenericValue<number>,
|
||||
"metricValues": number[][],
|
||||
}>> = this.report(
|
||||
this.targetLabelBatchesToReport,
|
||||
this.numberOfLabelsPerBatch,
|
||||
this.explicitTotal,
|
||||
this.explicitTotalPositives,
|
||||
this.explicitTotalNegatives,
|
||||
this.fMeasureBeta,
|
||||
this.effectivenessMeasureAlpha,
|
||||
this.rejectRate,
|
||||
this.alpha,
|
||||
this.beta,
|
||||
this.A,
|
||||
this.B,
|
||||
this.explicitMaxPositives);
|
||||
let cell11MetricIndexForDebugInstrumentation: number = -1;
|
||||
let ratioCell11MetricIndexForDebugInstrumentation: number = -1;
|
||||
let cell11MetricIndexInOutputArrayForDebugInstrumentation: number = -1;
|
||||
let ratioCell11MetricIndexInOutputArrayForDebugInstrumentation: number = -1;
|
||||
const outputEvaluationReportDataArrayHeader: string[] = [];
|
||||
const outputEvaluationReportDataArrays: string[][] = [];
|
||||
for (const reportResultsPerBatch of reportResults) {
|
||||
for (const reportResultPerLabel of reportResultsPerBatch) {
|
||||
const tarrgetLabel: string = reportResultPerLabel.targetLabel;
|
||||
const scoreThresholds: number[] = reportResultPerLabel.scoreThresholds;
|
||||
const metricNames: string[] = reportResultPerLabel.metricNames;
|
||||
const metricNameMap: IDictionaryStringIdGenericValue<number> = reportResultPerLabel.metricNameMap;
|
||||
const metricValues: number[][] = reportResultPerLabel.metricValues;
|
||||
if (EnableDebuggingInstrumentation) { // ---- NOTE-DEBUG-INSTRUMENTATION ----
|
||||
// const binaryConfusionMatrices: BinaryConfusionMatrix[] =
|
||||
// reportResultPerLabel.binaryConfusionMatrices;
|
||||
cell11MetricIndexForDebugInstrumentation =
|
||||
metricNameMap.Cell11;
|
||||
ratioCell11MetricIndexForDebugInstrumentation =
|
||||
metricNameMap.RatioCell11;
|
||||
cell11MetricIndexInOutputArrayForDebugInstrumentation =
|
||||
cell11MetricIndexForDebugInstrumentation + 2;
|
||||
ratioCell11MetricIndexInOutputArrayForDebugInstrumentation =
|
||||
ratioCell11MetricIndexForDebugInstrumentation + 2;
|
||||
}
|
||||
if (Utility.isEmptyStringArray(outputEvaluationReportDataArrayHeader)) {
|
||||
outputEvaluationReportDataArrayHeader.push("TargetLabel");
|
||||
outputEvaluationReportDataArrayHeader.push("ScoreThreshold");
|
||||
for (const metricName of metricNames) {
|
||||
outputEvaluationReportDataArrayHeader.push(metricName);
|
||||
}
|
||||
outputEvaluationReportDataArrays.push(outputEvaluationReportDataArrayHeader);
|
||||
}
|
||||
const numberInstances: number = scoreThresholds.length;
|
||||
for (let indexInstance: number = 0; indexInstance < numberInstances; indexInstance++) {
|
||||
const outputEvaluationReportDataArray: string[] = [];
|
||||
const scoreThreshold: number = scoreThresholds[indexInstance];
|
||||
const metricValueArray: number[] = metricValues[indexInstance];
|
||||
outputEvaluationReportDataArray.push(tarrgetLabel);
|
||||
outputEvaluationReportDataArray.push(scoreThreshold.toString());
|
||||
for (const metricValue of metricValueArray) {
|
||||
outputEvaluationReportDataArray.push(metricValue.toString());
|
||||
}
|
||||
outputEvaluationReportDataArrays.push(outputEvaluationReportDataArray);
|
||||
if (EnableDebuggingInstrumentation) { // ---- NOTE-DEBUG-INSTRUMENTATION ----
|
||||
const cell1MetricValue: number =
|
||||
metricValueArray[cell11MetricIndexForDebugInstrumentation];
|
||||
const ratioCell1MetricValue: number =
|
||||
metricValueArray[ratioCell11MetricIndexForDebugInstrumentation];
|
||||
const cell1MetricValueInString: string =
|
||||
outputEvaluationReportDataArray[cell11MetricIndexInOutputArrayForDebugInstrumentation];
|
||||
const ratioCell1MetricValueInString: string =
|
||||
outputEvaluationReportDataArray[ratioCell11MetricIndexInOutputArrayForDebugInstrumentation];
|
||||
if (cell1MetricValue > 0) {
|
||||
Utility.debuggingLog(
|
||||
`cell1MetricValue=${cell1MetricValue}`);
|
||||
Utility.debuggingLog(
|
||||
`cell1MetricValueInString=${cell1MetricValueInString}`);
|
||||
Utility.debuggingLog(
|
||||
`ratioCell1MetricValue=${ratioCell1MetricValue}`);
|
||||
Utility.debuggingLog(
|
||||
`ratioCell1MetricValueInString=${ratioCell1MetricValueInString}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public report(
|
||||
targetLabelBatchesToReport: string[][] = [],
|
||||
numberOfLabelsPerBatch: number = NumberOfLabelsPerBatchToReport,
|
||||
explicitTotal: number = -1,
|
||||
explicitTotalPositives: number = -1,
|
||||
explicitTotalNegatives: number = -1,
|
||||
fMeasureBeta: number = 1,
|
||||
effectivenessMeasureAlpha: number = 0.5,
|
||||
rejectRate: number = -1,
|
||||
alpha: number = 1,
|
||||
beta: number = 1,
|
||||
A: number = 1,
|
||||
B: number = 1,
|
||||
explicitMaxPositives: number = -1): Array<Array<{
|
||||
"targetLabel": string,
|
||||
"scoreThresholds": number[],
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"metricNames": string[],
|
||||
"metricNameMap": IDictionaryStringIdGenericValue<number>,
|
||||
"metricValues": number[][],
|
||||
}>> {
|
||||
this.validate();
|
||||
const labels: string[] = this.getLabels();
|
||||
const numberLabels: number = labels.length;
|
||||
if (Utility.isEmptyStringArrays(targetLabelBatchesToReport)) {
|
||||
targetLabelBatchesToReport = [];
|
||||
if (numberOfLabelsPerBatch <= 0) {
|
||||
numberOfLabelsPerBatch = NumberOfLabelsPerBatchToReport;
|
||||
}
|
||||
let numberLabelBatches: number = Math.floor(numberLabels / numberOfLabelsPerBatch);
|
||||
if ((numberLabelBatches * numberOfLabelsPerBatch) < numberLabels) {
|
||||
numberLabelBatches++;
|
||||
}
|
||||
for (let batch = 0; batch < numberLabelBatches; batch++) {
|
||||
const labelIndexBegin = batch * numberOfLabelsPerBatch;
|
||||
let labelIndexEnd = labelIndexBegin + numberOfLabelsPerBatch;
|
||||
if (labelIndexEnd > numberLabels) {
|
||||
labelIndexEnd = numberLabels;
|
||||
}
|
||||
if (labelIndexEnd > labelIndexBegin) {
|
||||
const targetLabelsPerBatch: string[] = [];
|
||||
for (let labelIndex = labelIndexBegin; labelIndex < labelIndexEnd; labelIndex++) {
|
||||
targetLabelsPerBatch.push(labels[labelIndex]);
|
||||
}
|
||||
targetLabelBatchesToReport.push(targetLabelsPerBatch);
|
||||
}
|
||||
}
|
||||
}
|
||||
const labelMap: { [id: string]: number } = this.getLabelMap();
|
||||
let metricNames: string[] = [];
|
||||
let metricNameMap: IDictionaryStringIdGenericValue<number> = {};
|
||||
const numberInstances: number = this.instanceGroudTruthLabelIds.length;
|
||||
const reportResults: Array<Array<{
|
||||
"targetLabel": string,
|
||||
"scoreThresholds": number[],
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"metricNames": string[],
|
||||
"metricNameMap": IDictionaryStringIdGenericValue<number>,
|
||||
"metricValues": number[][],
|
||||
}>> = [];
|
||||
for (const targetLabelsPerBatchToReport of targetLabelBatchesToReport) {
|
||||
const reportResultsPerBatch: Array<{
|
||||
"targetLabel": string,
|
||||
"scoreThresholds": number[],
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"metricNames": string[],
|
||||
"metricNameMap": IDictionaryStringIdGenericValue<number>,
|
||||
"metricValues": number[][],
|
||||
}> = [];
|
||||
for (const targetLabel of targetLabelsPerBatchToReport) {
|
||||
const targetLabelIndex: number =
|
||||
labelMap[targetLabel];
|
||||
if (targetLabelIndex === undefined) {
|
||||
Utility.debuggingThrow(
|
||||
`targetLabel|${targetLabel}| is not defined.`);
|
||||
}
|
||||
let targetLabelIndexScores: number[] = this.instancePredictedScoreArrays.map(
|
||||
(scoreArray) => scoreArray[targetLabelIndex]);
|
||||
const targetLabelPositives: number = this.instanceGroudTruthLabelIds.reduce(
|
||||
(total, instanceGroudTruthLabelId) =>
|
||||
(instanceGroudTruthLabelId === targetLabelIndex ? total + 1 : total), 0);
|
||||
let orderSequence: number[] = ListArrayUtility.sortGenerateOrderSequence(
|
||||
targetLabelIndexScores);
|
||||
{
|
||||
orderSequence = orderSequence.reverse();
|
||||
targetLabelIndexScores = targetLabelIndexScores.reverse();
|
||||
}
|
||||
const binaryConfusionMatrixBase: BinaryConfusionMatrix = new BinaryConfusionMatrix(
|
||||
numberInstances,
|
||||
0,
|
||||
targetLabelPositives,
|
||||
0);
|
||||
if (Utility.isEmptyStringArray(metricNames)) {
|
||||
metricNames = binaryConfusionMatrixBase.getMetricNames();
|
||||
}
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(metricNameMap)) {
|
||||
metricNameMap = binaryConfusionMatrixBase.getMetricNameMap();
|
||||
}
|
||||
const binaryConfusionMatrices: BinaryConfusionMatrix[] = orderSequence.map(
|
||||
(instanceIndex) => binaryConfusionMatrixBase.moveFromPredictedNegativeToPositive(
|
||||
this.instanceGroudTruthLabelIds[instanceIndex] === targetLabelIndex));
|
||||
const metricValues: number[][] =
|
||||
binaryConfusionMatrices.map((binaryConfusionMatrix) => binaryConfusionMatrix.getMetricValues(
|
||||
explicitTotal,
|
||||
explicitTotalPositives,
|
||||
explicitTotalNegatives,
|
||||
fMeasureBeta,
|
||||
effectivenessMeasureAlpha,
|
||||
rejectRate,
|
||||
alpha,
|
||||
beta,
|
||||
A,
|
||||
B,
|
||||
explicitMaxPositives));
|
||||
if (EnableDebuggingInstrumentation) { // ---- NOTE-DEBUG-INSTRUMENTATION ----
|
||||
const totalMetricIndex: number =
|
||||
metricNameMap.Total;
|
||||
const cell11MetricIndex: number =
|
||||
metricNameMap.Cell11;
|
||||
const ratioCell11MetricIndex: number =
|
||||
metricNameMap.RatioCell11;
|
||||
const totalMatricValues: number[] =
|
||||
metricValues.map((x: number[]) => x[totalMetricIndex]);
|
||||
const cell11MatricValues: number[] =
|
||||
metricValues.map((x: number[]) => x[cell11MetricIndex]);
|
||||
const ratioCell11MatricValues: number[] =
|
||||
metricValues.map((x: number[]) => x[ratioCell11MetricIndex]);
|
||||
Utility.debuggingLog(
|
||||
`cell11MatricValues=${cell11MatricValues}`);
|
||||
Utility.debuggingLog(
|
||||
`ratioCell11MatricValues=${ratioCell11MatricValues}`);
|
||||
Utility.debuggingLog(
|
||||
`totalMatricValues=${totalMatricValues}`);
|
||||
}
|
||||
const reportResultPerLabel: {
|
||||
"targetLabel": string,
|
||||
"scoreThresholds": number[],
|
||||
"binaryConfusionMatrices": BinaryConfusionMatrix[],
|
||||
"metricNames": string[],
|
||||
"metricNameMap": IDictionaryStringIdGenericValue<number>,
|
||||
"metricValues": number[][],
|
||||
} = {
|
||||
targetLabel,
|
||||
scoreThresholds: targetLabelIndexScores,
|
||||
binaryConfusionMatrices,
|
||||
metricNames,
|
||||
metricNameMap,
|
||||
metricValues,
|
||||
};
|
||||
reportResultsPerBatch.push(reportResultPerLabel);
|
||||
}
|
||||
reportResults.push(reportResultsPerBatch);
|
||||
}
|
||||
return reportResults;
|
||||
}
|
||||
|
||||
public validate(): void {
|
||||
if (Utility.isEmptyArray(this.instancePredictedScoreArrays)) {
|
||||
Utility.debuggingThrow("'this.instancePredictedScoreArrays' array is empty");
|
||||
}
|
||||
if (Utility.isEmptyNumberArray(this.instanceGroudTruthLabelIds)) {
|
||||
Utility.debuggingThrow("'this.instanceGroudTruthLabelIds' array is empty");
|
||||
}
|
||||
if (Utility.isEmptyStringArray(this.instanceFeatureTexts)) {
|
||||
Utility.debuggingThrow("'this.instanceFeatureTexts' array is empty");
|
||||
}
|
||||
if (Utility.isEmptyStringArray(this.instanceIdentifiers)) {
|
||||
Utility.debuggingThrow("'this.instanceIdentifiers' array is empty");
|
||||
}
|
||||
if (Utility.isEmptyNumberArray(this.instanceWeights)) {
|
||||
Utility.debuggingThrow("'this.instanceWeights' array is empty");
|
||||
}
|
||||
const numberInstances: number = this.getNumberInstances();
|
||||
if (this.instanceGroudTruthLabelIds.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`this.instanceGroudTruthLabelIds.length|${this.instanceGroudTruthLabelIds.length}|!==numberInstances|${numberInstances}|`);
|
||||
}
|
||||
if (this.instanceFeatureTexts.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`this.instanceFeatureTexts.length|${this.instanceFeatureTexts.length}|!==numberInstances|${numberInstances}|`);
|
||||
}
|
||||
if (this.instanceIdentifiers.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`this.instanceIdentifiers.length|${this.instanceIdentifiers.length}|!==numberInstances|${numberInstances}|`);
|
||||
}
|
||||
if (this.instanceWeights.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`this.instanceWeights.length|${this.instanceWeights.length}|!==numberInstances|${numberInstances}|`);
|
||||
}
|
||||
}
|
||||
|
||||
public loadScoreFileAndPopulate(
|
||||
scoreFilename: string,
|
||||
labelColumnIndex: number = 0,
|
||||
textColumnIndex: number = 1,
|
||||
weightColumnIndex: number = -1,
|
||||
scoreColumnBeginIndex: number = 2,
|
||||
identifierColumnIndex: number = -1,
|
||||
predictedLabelColumnIndex: number = -1,
|
||||
revisedTextColumnIndex: number = -1,
|
||||
lineIndexToStart: number = 0): void {
|
||||
const scoreDataStructure: {
|
||||
"labels": string[],
|
||||
"texts": string[],
|
||||
"weights": number[],
|
||||
"identifiers": string[],
|
||||
"scoreArrays": number[][],
|
||||
"predictedLabels": string[],
|
||||
"revisedTexts": string[] } =
|
||||
Utility.loadLabelTextScoreFile(
|
||||
scoreFilename,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
scoreColumnBeginIndex,
|
||||
this.getNumberLabels(),
|
||||
identifierColumnIndex,
|
||||
predictedLabelColumnIndex,
|
||||
revisedTextColumnIndex,
|
||||
lineIndexToStart);
|
||||
const labelMap: { [id: string]: number } =
|
||||
this.getLabelMap();
|
||||
const numberInstances: number = scoreDataStructure.labels.length;
|
||||
for (let i = 0; i < numberInstances; i++) {
|
||||
const label: string = scoreDataStructure.labels[i];
|
||||
const text: string = scoreDataStructure.texts[i];
|
||||
const identifier: string = scoreDataStructure.identifiers[i];
|
||||
const scoreArray: number[] = scoreDataStructure.scoreArrays[i];
|
||||
const labelId: number = labelMap[label];
|
||||
const weight: number = scoreDataStructure.weights[i];
|
||||
this.addInstance(scoreArray, labelId, text, identifier, weight);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -1,236 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import { ArgumentParser } from "argparse";
|
||||
|
||||
import { Tester } from "./Tester";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
|
||||
// tslint:disable-next-line: max-line-length
|
||||
// import { AppSoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
|
||||
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
|
||||
// import { ColumnarData } from "../../../data/ColumnarData";
|
||||
// import { LuData } from "../../../data/LuData";
|
||||
import { Data } from "../../../data/Data";
|
||||
import { DataUtility } from "../../../data/DataUtility";
|
||||
|
||||
// import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
import { LuData } from "../../../data/LuData";
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
import { ColumnarData } from "../../../data/ColumnarData";
|
||||
|
||||
export async function mainTester(): Promise<void> {
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeBeginInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
const parser = new ArgumentParser({
|
||||
addHelp: true,
|
||||
description: "AppTester",
|
||||
version: "0.0.1",
|
||||
});
|
||||
parser.addArgument(
|
||||
["-f", "--filename"],
|
||||
{
|
||||
help: "an input data file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-t", "--filetype"],
|
||||
{
|
||||
help: "data file type",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-m", "--modelFilename"],
|
||||
{
|
||||
help: "model file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-x", "--featurizerFilename"],
|
||||
{
|
||||
help: "serialized featurizer file",
|
||||
required: true,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-o", "--outputReportFilenamePrefix"],
|
||||
{
|
||||
help: "output report file prefix",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-d", "--debug"],
|
||||
{
|
||||
help: "enable printing debug information",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-li", "--labelColumnIndex"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "label column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ti", "--textColumnIndex"],
|
||||
{
|
||||
defaultValue: 1,
|
||||
help: "text/utterance column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-wi", "--weightColumnIndex"],
|
||||
{
|
||||
defaultValue: -1,
|
||||
help: "weight column index",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
parser.addArgument(
|
||||
["-ls", "--linesToSkip"],
|
||||
{
|
||||
defaultValue: 0,
|
||||
help: "number of lines to skip from the input file",
|
||||
required: false,
|
||||
},
|
||||
);
|
||||
const parsedKnownArgs: any[] = parser.parseKnownArgs();
|
||||
const args: any = parsedKnownArgs[0];
|
||||
const unknownArgs: any = parsedKnownArgs[1];
|
||||
Utility.debuggingLog(
|
||||
`args=${Utility.jsonStringify(args)}`);
|
||||
Utility.debuggingLog(
|
||||
`unknownArgs=${Utility.jsonStringify(unknownArgs)}`);
|
||||
const debugFlag: boolean = Utility.toBoolean(args.debug);
|
||||
Utility.toPrintDebuggingLogToConsole = debugFlag;
|
||||
// ---- NOTE-FOR-DEBUGGING ---- console.dir(args);
|
||||
// -----------------------------------------------------------------------
|
||||
const filename: string =
|
||||
args.filename;
|
||||
if (!Utility.exists(filename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input dataset file ${filename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const filetype: string =
|
||||
args.filetype;
|
||||
const modelFilename: string =
|
||||
args.modelFilename;
|
||||
if (!Utility.exists(modelFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input model file ${modelFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
const featurizerFilename: string =
|
||||
args.featurizerFilename;
|
||||
if (!Utility.exists(featurizerFilename)) {
|
||||
Utility.debuggingThrow(
|
||||
`The input featurizer file ${featurizerFilename} does not exist! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
let outputReportFilenamePrefix: string = args.outputReportFilenamePrefix;
|
||||
if (Utility.isEmptyString(outputReportFilenamePrefix)) {
|
||||
outputReportFilenamePrefix = Utility.getFilenameWithoutExtension(filename);
|
||||
// Utility.debuggingThrow(
|
||||
// `The output file ${outputReportFilenamePrefix} is empty! process.cwd()=${process.cwd()}`);
|
||||
}
|
||||
Utility.debuggingLog(
|
||||
`filename=${filename}`);
|
||||
Utility.debuggingLog(
|
||||
`outputReportFilenamePrefix=${outputReportFilenamePrefix}`);
|
||||
Utility.debuggingLog(
|
||||
`modelFilename=${modelFilename}`);
|
||||
Utility.debuggingLog(
|
||||
`featurizerFilename=${featurizerFilename}`);
|
||||
// const outputModelFilename: string =
|
||||
// args.outputModelFilename;
|
||||
// -----------------------------------------------------------------------
|
||||
const labelColumnIndex: number = +args.labelColumnIndex;
|
||||
const textColumnIndex: number = +args.textColumnIndex;
|
||||
const weightColumnIndex: number = +args.weightColumnIndex;
|
||||
const linesToSkip: number = +args.linesToSkip;
|
||||
Utility.debuggingLog(
|
||||
`labelColumnIndex=${labelColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`textColumnIndex=${textColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`weightColumnIndex=${weightColumnIndex}`);
|
||||
Utility.debuggingLog(
|
||||
`linesToSkip=${linesToSkip}`);
|
||||
// -----------------------------------------------------------------------
|
||||
const tester: Tester =
|
||||
new Tester(
|
||||
modelFilename,
|
||||
featurizerFilename);
|
||||
const featurizer: NgramSubwordFeaturizer =
|
||||
tester.getFeaturizer();
|
||||
// Utility.debuggingLog(
|
||||
// `featurizer.getLabelMap()=${Utility.jsonStringify(featurizer.getLabelMap())}`);
|
||||
// -----------------------------------------------------------------------
|
||||
let intentsUtterancesWeights: {
|
||||
"intents": string[],
|
||||
"utterances": string[],
|
||||
"weights": number[] } = {
|
||||
intents: [],
|
||||
utterances: [],
|
||||
weights: [] };
|
||||
let intentLabelIndexArray: number[] = [];
|
||||
let utteranceFeatureIndexArrays: number[][] = [];
|
||||
const data: Data = await DataUtility.LoadData(
|
||||
filename,
|
||||
featurizer,
|
||||
false,
|
||||
filetype,
|
||||
labelColumnIndex,
|
||||
textColumnIndex,
|
||||
weightColumnIndex,
|
||||
linesToSkip);
|
||||
intentsUtterancesWeights = data.getIntentsUtterancesWeights();
|
||||
intentLabelIndexArray = data.getIntentLabelIndexArray();
|
||||
utteranceFeatureIndexArrays = data.getUtteranceFeatureIndexArrays();
|
||||
// -----------------------------------------------------------------------
|
||||
tester.test(
|
||||
intentsUtterancesWeights.intents,
|
||||
intentsUtterancesWeights.utterances,
|
||||
intentsUtterancesWeights.weights,
|
||||
intentLabelIndexArray,
|
||||
utteranceFeatureIndexArrays);
|
||||
// -----------------------------------------------------------------------
|
||||
const evaluationJsonReportResult: {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} = tester.generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
const evaluationDataArraysReportResult: {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} = tester.generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix);
|
||||
// -----------------------------------------------------------------------
|
||||
const dateTimeEndInString: string = (new Date()).toISOString();
|
||||
// -----------------------------------------------------------------------
|
||||
Utility.debuggingLog(
|
||||
`dateTimeBeginInString=${dateTimeBeginInString}`);
|
||||
Utility.debuggingLog(
|
||||
`dateTimeEndInString=${dateTimeEndInString}`);
|
||||
// -----------------------------------------------------------------------
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
mainTester();
|
||||
}
|
||||
|
|
@ -1,354 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
// tslint:disable-next-line: max-line-length
|
||||
// import { AppSoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/AppSoftmaxRegressionSparse";
|
||||
import { SoftmaxRegressionSparse } from "../../supervised/classifier/neural_network/learner/SoftmaxRegressionSparse";
|
||||
|
||||
import { IMathematicsHelper } from "../../../mathematics/mathematics_helper/IMathematicsHelper";
|
||||
import { MathematicsHelper } from "../../../mathematics/mathematics_helper/MathematicsHelper";
|
||||
|
||||
import { IConfusionMatrix } from "../../../mathematics/confusion_matrix/IConfusionMatrix";
|
||||
import { ConfusionMatrix } from "../../../mathematics/confusion_matrix/ConfusionMatrix";
|
||||
import { BinaryConfusionMatrix } from "../../../mathematics/confusion_matrix/BinaryConfusionMatrix";
|
||||
|
||||
import { ThresholdReporter } from "../report/ThresholdReporter";
|
||||
|
||||
// import { ColumnarData } from "../../../data/ColumnarData";
|
||||
// import { LuData } from "../../../data/LuData";
|
||||
|
||||
import { NgramSubwordFeaturizer } from "../../language_understanding/featurizer/NgramSubwordFeaturizer";
|
||||
|
||||
import { IDictionaryStringIdGenericArrays } from "../../../data_structure/IDictionaryStringIdGenericArrays";
|
||||
import { IDictionaryStringIdGenericValue } from "../../../data_structure/IDictionaryStringIdGenericValue";
|
||||
import { DictionaryMapUtility } from "../../../data_structure/DictionaryMapUtility";
|
||||
|
||||
import { AbstractBaseModelFeaturizerEvaluator } from "../abstract_base_evaluator/AbstractBaseModelFeaturizerEvaluator";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export class Tester extends AbstractBaseModelFeaturizerEvaluator {
|
||||
|
||||
public static readonly MathematicsHelperObject: IMathematicsHelper =
|
||||
MathematicsHelper.GetMathematicsHelperObject();
|
||||
|
||||
protected intentsCachedAfterTest: string[] = [];
|
||||
protected utterancesCachedAfterTest: string[] = [];
|
||||
protected labelIndexArrayCachedAfterTest: number[] = [];
|
||||
protected featureIndexArraysCachedAfterTest: number[][] = [];
|
||||
|
||||
protected testResultCachedAfterTest: {
|
||||
"confusionMatrixTest": ConfusionMatrix
|
||||
"thresholdReporterTest": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } = {
|
||||
confusionMatrixTest: new ConfusionMatrix([], {}),
|
||||
thresholdReporterTest: new ThresholdReporter("", "", null, null, [], {}),
|
||||
predictionLabels: [],
|
||||
predictionLabelIndexes: [],
|
||||
instanceIndexes: [],
|
||||
groundTruthLabels: [],
|
||||
groundTruthLabelIndexes: [],
|
||||
predictions: [] };
|
||||
|
||||
public constructor(
|
||||
modelFilename: string,
|
||||
featurizerFilename: string) {
|
||||
super(modelFilename, featurizerFilename, null, null, [], {});
|
||||
}
|
||||
|
||||
public getTestResult(): {
|
||||
"confusionMatrixTest": ConfusionMatrix
|
||||
"thresholdReporterTest": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
return this.testResultCachedAfterTest;
|
||||
}
|
||||
|
||||
public generateEvaluationDataArraysReport(): IDictionaryStringIdGenericArrays<string> {
|
||||
const outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {};
|
||||
{
|
||||
const predictionLabels: string[] = this.testResultCachedAfterTest.predictionLabels;
|
||||
const predictionLabelIndexes: number[] = this.testResultCachedAfterTest.predictionLabelIndexes;
|
||||
const instanceIndexes: number[] = this.testResultCachedAfterTest.instanceIndexes;
|
||||
const groundTruthLabels: string[] = this.testResultCachedAfterTest.groundTruthLabels;
|
||||
const groundTruthLabelIndexes: number[] = this.testResultCachedAfterTest.groundTruthLabelIndexes;
|
||||
const predictions: number[][] = this.testResultCachedAfterTest.predictions;
|
||||
const outputEvaluationReportDataArraysScoreRecords: string[][] = [];
|
||||
for (let index: number = 0; index < this.intentsCachedAfterTest.length; index++) {
|
||||
const instanceIndex: number = instanceIndexes[index];
|
||||
const intent: string = this.intentsCachedAfterTest[instanceIndex];
|
||||
const utterance: string = this.utterancesCachedAfterTest[instanceIndex];
|
||||
const groundTruthLabel: string = groundTruthLabels[index];
|
||||
const groundTruthLabelIndex: number = groundTruthLabelIndexes[index];
|
||||
const predictionLabel: string = predictionLabels[index];
|
||||
const predictionLabelIndex: number = predictionLabelIndexes[index];
|
||||
const outputEvaluationReportDataArraysScoreRecord: string[] = [];
|
||||
outputEvaluationReportDataArraysScoreRecord.push(intent);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(utterance);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(instanceIndex.toString());
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(groundTruthLabelIndex.toString());
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabel);
|
||||
outputEvaluationReportDataArraysScoreRecord.push(predictionLabelIndex.toString());
|
||||
const prediction: number[] = predictions[index];
|
||||
for (const labelIndex of Array(prediction.length).keys()) {
|
||||
outputEvaluationReportDataArraysScoreRecord.push(prediction[labelIndex].toString());
|
||||
}
|
||||
outputEvaluationReportDataArraysScoreRecords.push(
|
||||
outputEvaluationReportDataArraysScoreRecord);
|
||||
}
|
||||
outputEvaluationReportDataArrays.TestScoreRecords =
|
||||
outputEvaluationReportDataArraysScoreRecords;
|
||||
}
|
||||
return outputEvaluationReportDataArrays;
|
||||
}
|
||||
public generateEvaluationDataArraysReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
outputDataArraryHeaders: string[] = [],
|
||||
columnDelimiter: string = "\t",
|
||||
recordDelimiter: string = "\n",
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportDataArrays: IDictionaryStringIdGenericArrays<string> = {}): {
|
||||
"outputEvaluationReportDataArrays": IDictionaryStringIdGenericArrays<string>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericArraysDictionary(outputEvaluationReportDataArrays)) {
|
||||
outputEvaluationReportDataArrays =
|
||||
this.generateEvaluationDataArraysReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_TestScoreRecords.txt`;
|
||||
outputFilename = Utility.storeDataArraysToTsvFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportDataArrays.TestScoreRecords,
|
||||
outputDataArraryHeaders,
|
||||
columnDelimiter,
|
||||
recordDelimiter,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportDataArrays, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationJsonReport(): IDictionaryStringIdGenericValue<any> {
|
||||
const outputEvaluationReport: IDictionaryStringIdGenericValue<any> = {};
|
||||
const confusionMatrixTest: ConfusionMatrix =
|
||||
this.testResultCachedAfterTest.confusionMatrixTest;
|
||||
const confusionMatrixMetricStructure: {
|
||||
"confusionMatrix": IConfusionMatrix,
|
||||
"labelBinaryConfusionMatrixBasicMetricMap": { [id: string]: { [id: string]: number } },
|
||||
"labelBinaryConfusionMatrixMap": { [id: string]: BinaryConfusionMatrix },
|
||||
"microAverageMetrics": {
|
||||
"accuracy": number,
|
||||
"truePositives": number,
|
||||
"falsePositives": number,
|
||||
"falseNegatives": number,
|
||||
"support": number },
|
||||
"macroAverageMetrics": {
|
||||
"averagePrecision": number,
|
||||
"averageRecall": number,
|
||||
"averageF1Score": number,
|
||||
"averageAccuracy": number,
|
||||
"averageTruePositives": number,
|
||||
"averageFalsePositives": number,
|
||||
"averageTrueNegatives": number,
|
||||
"averageFalseNegatives": number,
|
||||
"averageSupport": number,
|
||||
"support": number },
|
||||
"weightedMacroAverageMetrics": {
|
||||
"weightedAveragePrecision": number,
|
||||
"weightedAverageRecall": number,
|
||||
"weightedAverageF1Score": number,
|
||||
"weightedAverageAccuracy": number,
|
||||
"weightedAverageSupport": number,
|
||||
"support": number } } =
|
||||
confusionMatrixTest.generateConfusionMatrixMetricStructure();
|
||||
Utility.debuggingLog(
|
||||
`confusionMatrixTest.getMicroAverageMetrics()=` +
|
||||
`${confusionMatrixTest.getMicroAverageMetrics()}` +
|
||||
`,confusionMatrixTest.getMacroAverageMetrics()=` +
|
||||
`${confusionMatrixTest.getMacroAverageMetrics()}` +
|
||||
`,confusionMatrixTest.getWeightedMacroAverageMetrics()=` +
|
||||
`${confusionMatrixTest.getWeightedMacroAverageMetrics()}`);
|
||||
outputEvaluationReport.confusionMatrixMetricStructure =
|
||||
confusionMatrixMetricStructure;
|
||||
return outputEvaluationReport;
|
||||
}
|
||||
public generateEvaluationJsonReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string = "utf8",
|
||||
outputEvaluationReportJson: IDictionaryStringIdGenericValue<any> = {}): {
|
||||
"outputEvaluationReportJson": IDictionaryStringIdGenericValue<any>,
|
||||
"outputFilenames": string[],
|
||||
} {
|
||||
if (DictionaryMapUtility.isEmptyStringIdGenericValueDictionary(outputEvaluationReportJson)) {
|
||||
outputEvaluationReportJson =
|
||||
this.generateEvaluationJsonReport();
|
||||
}
|
||||
{
|
||||
let outputFilename: string =
|
||||
`${outputReportFilenamePrefix}_TestConfusionMatrixReport.json`;
|
||||
outputFilename = this.dumpEvaluationJsonReportToFile(
|
||||
outputFilename,
|
||||
outputEvaluationReportJson.confusionMatrixMetricStructure,
|
||||
encoding);
|
||||
const outputFilenames: string[] =
|
||||
[outputFilename];
|
||||
return { outputEvaluationReportJson, outputFilenames };
|
||||
}
|
||||
}
|
||||
|
||||
public generateEvaluationDirectReport(): IDictionaryStringIdGenericValue<string> {
|
||||
return {}; // ---- NOTE: nothing to report here.
|
||||
}
|
||||
public generateEvaluationDirectReportToFiles(
|
||||
outputReportFilenamePrefix: string,
|
||||
encoding: string,
|
||||
outputEvaluationReportDirect: IDictionaryStringIdGenericValue<string> = {}):
|
||||
IDictionaryStringIdGenericValue<string> {
|
||||
// ---- NOTE: nothing to report here.
|
||||
return outputEvaluationReportDirect;
|
||||
}
|
||||
|
||||
public test(
|
||||
intents: string[],
|
||||
utterances: string[],
|
||||
weights: number[],
|
||||
labelIndexArray: number[],
|
||||
featureIndexArrays: number[][]): {
|
||||
"confusionMatrixTest": ConfusionMatrix,
|
||||
"thresholdReporterTest": ThresholdReporter,
|
||||
"predictionLabels": string[],
|
||||
"predictionLabelIndexes": number[],
|
||||
"instanceIndexes": number[],
|
||||
"groundTruthLabels": string[],
|
||||
"groundTruthLabelIndexes": number[],
|
||||
"predictions": number[][] } {
|
||||
// -------------------------------------------------------------------
|
||||
this.intentsCachedAfterTest = intents;
|
||||
this.utterancesCachedAfterTest = utterances;
|
||||
this.labelIndexArrayCachedAfterTest = labelIndexArray;
|
||||
this.featureIndexArraysCachedAfterTest = featureIndexArrays;
|
||||
// -------------------------------------------------------------------
|
||||
const model: SoftmaxRegressionSparse =
|
||||
this.getModel();
|
||||
const featurizer: NgramSubwordFeaturizer =
|
||||
this.getFeaturizer();
|
||||
// -------------------------------------------------------------------
|
||||
const labels: string[] = this.getLabels();
|
||||
const labelMap: { [id: string]: number } = this.getLabelMap();
|
||||
// const numberLabels: number = featurizer.getNumberLabels();
|
||||
// const numberFeatures: number = featurizer.getNumberFeatures();
|
||||
// -------------------------------------------------------------------
|
||||
Utility.debuggingLog(`labelIndexArray.length=` +
|
||||
`${labelIndexArray.length}`);
|
||||
Utility.debuggingLog(`featureIndexArrays.length=` +
|
||||
`${featureIndexArrays.length}`);
|
||||
// -------------------------------------------------------------------
|
||||
const confusionMatrixTest: ConfusionMatrix =
|
||||
new ConfusionMatrix(labels, labelMap);
|
||||
const thresholdReporterTest: ThresholdReporter =
|
||||
new ThresholdReporter("", "", null, null, labels, labelMap);
|
||||
const numberInstances: number =
|
||||
intents.length;
|
||||
if (utterances.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`utterances.length|${utterances.length}|!==` +
|
||||
`numberInstances|${numberInstances}|`);
|
||||
}
|
||||
const isWeightsArrayNotEmpty: boolean = !Utility.isEmptyNumberArray(weights);
|
||||
if (isWeightsArrayNotEmpty) {
|
||||
if (weights.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`weights.length|${weights.length}|!==` +
|
||||
`numberInstances|${numberInstances}|`);
|
||||
}
|
||||
}
|
||||
if (labelIndexArray.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`labelIndexArray.length|${labelIndexArray.length}|!==` +
|
||||
`numberInstances|${numberInstances}|`);
|
||||
}
|
||||
if (featureIndexArrays.length !== numberInstances) {
|
||||
Utility.debuggingThrow(
|
||||
`featureIndexArrays.length|${featureIndexArrays.length}|!==` +
|
||||
`numberInstances|${numberInstances}|`);
|
||||
}
|
||||
const predictions: number[][] =
|
||||
model.predict(featureIndexArrays);
|
||||
const predictionLabels: string[] =
|
||||
new Array<string>(numberInstances);
|
||||
const predictionLabelIndexes: number[] =
|
||||
new Array<number>(numberInstances);
|
||||
const instanceIndexes: number[] =
|
||||
new Array<number>(numberInstances);
|
||||
const groundTruthLabels: string[] =
|
||||
new Array<string>(numberInstances);
|
||||
const groundTruthLabelIndexes: number[] =
|
||||
new Array<number>(numberInstances);
|
||||
for (let index: number = 0; index < numberInstances; index++) {
|
||||
// ---------------------------------------------------------------
|
||||
// const intent: string = intents[index];
|
||||
const utterance: string = utterances[index];
|
||||
let weight: number = 1;
|
||||
if (isWeightsArrayNotEmpty) {
|
||||
weight = weights[index];
|
||||
}
|
||||
const labelIndex: number = labelIndexArray[index];
|
||||
// const featureIndexArray: number[] = featureIndexArrays[index];
|
||||
const prediction: number[] = predictions[index];
|
||||
// ---------------------------------------------------------------
|
||||
const groundTruthLabel: string = labels[labelIndex];
|
||||
instanceIndexes[index] = index;
|
||||
groundTruthLabels[index] = groundTruthLabel;
|
||||
groundTruthLabelIndexes[index] = labelIndex;
|
||||
// ---------------------------------------------------------------
|
||||
const argMax: { "indexMax": number, "max": number } =
|
||||
Tester.MathematicsHelperObject.getIndexOnFirstMaxEntry(prediction);
|
||||
const predictionLabelIndex: number =
|
||||
argMax.indexMax;
|
||||
predictionLabelIndexes[index] =
|
||||
predictionLabelIndex;
|
||||
const predictionLabel: string =
|
||||
labels[predictionLabelIndex];
|
||||
predictionLabels[index] =
|
||||
predictionLabel;
|
||||
confusionMatrixTest.addInstanceByLabelIndex(
|
||||
labelIndex,
|
||||
predictionLabelIndex,
|
||||
weight);
|
||||
thresholdReporterTest.addInstance(
|
||||
prediction,
|
||||
labelIndex,
|
||||
utterance,
|
||||
`${index}`,
|
||||
weight);
|
||||
// ---------------------------------------------------------------
|
||||
}
|
||||
// -------------------------------------------------------------------
|
||||
this.testResultCachedAfterTest = {
|
||||
confusionMatrixTest,
|
||||
thresholdReporterTest,
|
||||
predictionLabels,
|
||||
predictionLabelIndexes,
|
||||
instanceIndexes,
|
||||
groundTruthLabels,
|
||||
groundTruthLabelIndexes,
|
||||
predictions };
|
||||
return this.testResultCachedAfterTest;
|
||||
// -------------------------------------------------------------------
|
||||
}
|
||||
}
|
||||
|
|
@ -1,26 +0,0 @@
|
|||
/**
|
||||
* Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
* Licensed under the MIT License.
|
||||
*/
|
||||
|
||||
import assert = require("assert");
|
||||
|
||||
import { NgramSubwordFeaturizer } from "./NgramSubwordFeaturizer";
|
||||
|
||||
import { Utility } from "../../../utility/Utility";
|
||||
|
||||
export function exampleFunctionNgramSubwordFeaturizer(): void {
|
||||
const featurizer: NgramSubwordFeaturizer = new NgramSubwordFeaturizer();
|
||||
const input = "sdsd p k-90yl m,.13 t!@ qefq# qef@Eg ljqefq";
|
||||
const result = featurizer.featurize(input);
|
||||
Utility.debuggingLog(input);
|
||||
Utility.debuggingLog(result);
|
||||
Utility.debuggingLog("hashing code = " + Utility.getPositiveStringHashCode(input));
|
||||
assert.throws(() => {
|
||||
Utility.debuggingLog(`hashing index = ${featurizer.getHashingFeatureIndex(input)}`); },
|
||||
`featurizer.getNumberHashingFeatures()=${featurizer.getNumberHashingFeatures()}`);
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
exampleFunctionNgramSubwordFeaturizer();
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче