From 9ffbc33efd692cac217386c231264c1909ab2792 Mon Sep 17 00:00:00 2001 From: v-rucdu Date: Thu, 31 Dec 2020 18:02:44 +0530 Subject: [PATCH] Logic to validate schema of DataConnector json --- .../dataConnectorValidations.yaml | 12 + .script/dataConnectorValidator.ts | 31 ++ .../dataConnectorValidator.test.ts | 25 + ...ngRequiredPropertyDataConnectorSchema.json | 89 ++++ .../testFiles/validDataConnectorSchema.json | 90 ++++ .script/utils/dataConnector.ts | 189 ++++++++ .../utils/schemas/DataConnectorSchema.json | 442 ++++++++++++++++++ azure-pipelines.yml | 1 + 8 files changed, 879 insertions(+) create mode 100644 .azure-pipelines/dataConnectorValidations.yaml create mode 100644 .script/dataConnectorValidator.ts create mode 100644 .script/tests/dataConnectorValidatorTest/dataConnectorValidator.test.ts create mode 100644 .script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json create mode 100644 .script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json create mode 100644 .script/utils/dataConnector.ts create mode 100644 .script/utils/schemas/DataConnectorSchema.json diff --git a/.azure-pipelines/dataConnectorValidations.yaml b/.azure-pipelines/dataConnectorValidations.yaml new file mode 100644 index 0000000000..6336c8bc38 --- /dev/null +++ b/.azure-pipelines/dataConnectorValidations.yaml @@ -0,0 +1,12 @@ +jobs: +- job: "DataConnectorValidations" + pool: + vmImage: 'Ubuntu 16.04' + steps: + - task: Npm@1 + displayName: 'npm install' + inputs: + verbose: false + command: 'install' + - script: 'npm run tsc && node .script/dataConnectorValidator.js' + displayName: 'Data Connector Validations' \ No newline at end of file diff --git a/.script/dataConnectorValidator.ts b/.script/dataConnectorValidator.ts new file mode 100644 index 0000000000..f9fc656a7e --- /dev/null +++ b/.script/dataConnectorValidator.ts @@ -0,0 +1,31 @@ +import fs from "fs"; +import { runCheckOverChangedFiles } from "./utils/changedFilesValidator"; +import { ExitCode } from "./utils/exitCode"; +import { isValidSchema } from "./utils/jsonSchemaChecker"; +import * as logger from "./utils/logger"; + +export async function IsValidDataConnectorSchema(filePath: string): Promise { + let dataConnector = JSON.parse(fs.readFileSync(filePath, "utf8")); + let schema = JSON.parse(fs.readFileSync(".script/utils/schemas/DataConnectorSchema.json", "utf8")); + + isValidSchema(dataConnector, schema); + + return ExitCode.SUCCESS; +} + +let fileTypeSuffixes = ["*.json"]; +let filePathFolderPrefixes = ["DataConnectors"]; +let fileKinds = ["Added", "Modified"]; +let CheckOptions = { + onCheckFile: (filePath: string) => { + return IsValidDataConnectorSchema(filePath); + }, + onExecError: async (e: any, filePath: string) => { + console.log(`Data Connector Validation Failed. File path: ${filePath}. Error message: ${e.message}`); + }, + onFinalFailed: async () => { + logger.logError("An error occurred, please open an issue"); + }, +}; + +runCheckOverChangedFiles(CheckOptions, fileKinds, fileTypeSuffixes, filePathFolderPrefixes); diff --git a/.script/tests/dataConnectorValidatorTest/dataConnectorValidator.test.ts b/.script/tests/dataConnectorValidatorTest/dataConnectorValidator.test.ts new file mode 100644 index 0000000000..7a491fb4fe --- /dev/null +++ b/.script/tests/dataConnectorValidatorTest/dataConnectorValidator.test.ts @@ -0,0 +1,25 @@ +import chai, { expect } from "chai"; +import chaiAsPromised from "chai-as-promised"; +import { ExitCode } from "../../utils/exitCode"; +import { IsValidDataConnectorSchema } from "../../dataConnectorValidator"; + +chai.use(chaiAsPromised); + +describe("dataConnectorValidator", () => { + it("should pass when dataConnectorSchema.json is valid", async () => { + await checkValid(".script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json"); + }); + + it("should throw an exception when dataConnectorSchema.json is missing a required property", async () => { + await checkInvalid(".script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json", "SchemaError"); + }); + + async function checkValid(filePath: string): Promise { + let result = await IsValidDataConnectorSchema(filePath); + expect(result).to.equal(ExitCode.SUCCESS); + } + + async function checkInvalid(filePath: string, expectedError: string): Promise { + expect(IsValidDataConnectorSchema(filePath)).eventually.rejectedWith(Error).and.have.property("name", expectedError); + } +}); diff --git a/.script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json b/.script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json new file mode 100644 index 0000000000..82293e7b84 --- /dev/null +++ b/.script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json @@ -0,0 +1,89 @@ +{ + "id": "NXLogDnsLogs", + "title": "NXLog DNS Logs", + "descriptionMarkdown": "The NXLog DNS Logs data connector uses Event Tracing for Windows ([ETW](https://docs.microsoft.com/windows/apps/trace-processing/overview)) for collecting both Audit and Analytical DNS Server events. The [NXLog *im_etw* module](https://nxlog.co/documentation/nxlog-user-guide/im_etw.html) reads event tracing data directly for maximum efficiency, without the need to capture the event trace into an .etl file. This REST API connector can forward DNS Server events to Azure Sentinel in real-time.", + "graphQueries": [ + { + "metricName": "Total data received", + "legend": "DNS_Logs_CL", + "baseQuery": "DNS_Logs_CL" + } + ], + "sampleQueries": [ + { + "description" : "EventID frequency", + "query": "DNS_Logs_CL| summarize EventCount = count() by EventID = EventID_d| order by EventID| render barchart" + }, + { + "description" : "Event Type counts", + "query": "DNS_Logs_CL| where strlen(DNS_LogType_s) > 1| summarize EventCount = count() by EventType = DNS_LogType_s| order by EventType asc | render barchart" + } + ], + "dataTypes": [ + { + "name": "DNS_Logs_CL", + "lastDataReceivedQuery": "DNS_Logs_CL | summarize Time = max(TimeGenerated) | where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "DNS_Logs_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1 + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key)", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ] + }, + "instructionSteps": [ + { + "title": "", + "description": "Follow the step-by-step instructions in the *NXLog User Guide* Integration Topic [Microsoft Azure Sentinel](https://nxlog.co/documentation/nxlog-user-guide/sentinel.html) to configure this connector.", + "instructions": [ + { + "parameters": { + "fillWith": [ + "WorkspaceId" + ], + "label": "Workspace ID" + }, + "type": "CopyableLabel" + }, + { + "parameters": { + "fillWith": [ + "PrimaryKey" + ], + "label": "Primary Key" + }, + "type": "CopyableLabel" + } + ] + } + ] +} \ No newline at end of file diff --git a/.script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json b/.script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json new file mode 100644 index 0000000000..024974b384 --- /dev/null +++ b/.script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json @@ -0,0 +1,90 @@ +{ + "id": "NXLogDnsLogs", + "title": "NXLog DNS Logs", + "publisher": "NXLog", + "descriptionMarkdown": "The NXLog DNS Logs data connector uses Event Tracing for Windows ([ETW](https://docs.microsoft.com/windows/apps/trace-processing/overview)) for collecting both Audit and Analytical DNS Server events. The [NXLog *im_etw* module](https://nxlog.co/documentation/nxlog-user-guide/im_etw.html) reads event tracing data directly for maximum efficiency, without the need to capture the event trace into an .etl file. This REST API connector can forward DNS Server events to Azure Sentinel in real-time.", + "graphQueries": [ + { + "metricName": "Total data received", + "legend": "DNS_Logs_CL", + "baseQuery": "DNS_Logs_CL" + } + ], + "sampleQueries": [ + { + "description" : "EventID frequency", + "query": "DNS_Logs_CL| summarize EventCount = count() by EventID = EventID_d| order by EventID| render barchart" + }, + { + "description" : "Event Type counts", + "query": "DNS_Logs_CL| where strlen(DNS_LogType_s) > 1| summarize EventCount = count() by EventType = DNS_LogType_s| order by EventType asc | render barchart" + } + ], + "dataTypes": [ + { + "name": "DNS_Logs_CL", + "lastDataReceivedQuery": "DNS_Logs_CL | summarize Time = max(TimeGenerated) | where isnotempty(Time)" + } + ], + "connectivityCriterias": [ + { + "type": "IsConnectedQuery", + "value": [ + "DNS_Logs_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)" + ] + } + ], + "availability": { + "status": 1 + }, + "permissions": { + "resourceProvider": [ + { + "provider": "Microsoft.OperationalInsights/workspaces", + "permissionsDisplayText": "read and write permissions are required.", + "providerDisplayName": "Workspace", + "scope": "Workspace", + "requiredPermissions": { + "write": true, + "read": true, + "delete": true + } + }, + { + "provider": "Microsoft.OperationalInsights/workspaces/sharedKeys", + "permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key)", + "providerDisplayName": "Keys", + "scope": "Workspace", + "requiredPermissions": { + "action": true + } + } + ] + }, + "instructionSteps": [ + { + "title": "", + "description": "Follow the step-by-step instructions in the *NXLog User Guide* Integration Topic [Microsoft Azure Sentinel](https://nxlog.co/documentation/nxlog-user-guide/sentinel.html) to configure this connector.", + "instructions": [ + { + "parameters": { + "fillWith": [ + "WorkspaceId" + ], + "label": "Workspace ID" + }, + "type": "CopyableLabel" + }, + { + "parameters": { + "fillWith": [ + "PrimaryKey" + ], + "label": "Primary Key" + }, + "type": "CopyableLabel" + } + ] + } + ] +} \ No newline at end of file diff --git a/.script/utils/dataConnector.ts b/.script/utils/dataConnector.ts new file mode 100644 index 0000000000..147794dc7e --- /dev/null +++ b/.script/utils/dataConnector.ts @@ -0,0 +1,189 @@ +export interface DataConnectorModel { + id: string; + title: string; + publisher: string; + descriptionMarkdown: string; + graphQueries: GraphQuery[]; + sampleQueries: SampleQuery[]; + dataTypes: DataType[]; + availability: ConnectorAvailability; + instructionSteps: InstructionStep[]; + permissions: RequiredConnectorPermissions; + isConnectivityCriteriasMatchSome?: boolean; + connectivityCriterias: ConnectivityCriteriaModel[]; + additionalRequirementBanner?: string; + logo?: string ; +} + +export interface GraphQuery { + metricName: string; + legend: string; + baseQuery: string; +} + +export interface SampleQuery { + description?: string; + query: string; +} + +export interface DataType { + name: string; + lastDataReceivedQuery: string; +} + +export enum TenantPermissions { + GlobalAdmin = "GlobalAdmin", + SecurityAdmin = "SecurityAdmin", + SecurityReader = "SecurityReader", + InformationProtection = "InformationProtection", +} + +export enum RequiredLicense { + OfficeATP = "OfficeATP", + Office365 = "Office365", + AadP1P2 = "AadP1P2", + Mcas = "Mcas", + Aatp = "Aatp", + Asc = "Asc", + Mdatp = "Mdatp", + Mtp = "Mtp", + IoT = "IoT", +} + +export interface CustomPreCondition { + name: string; + description: string; +} + +export interface RequiredPermissionSet { + read?: boolean; + write?: boolean; + delete?: boolean; + action?: boolean; +} + +export interface ResourceProviderPermissions { + provider: string; + providerDisplayName: string; + permissionsDisplayText: string; + requiredPermissions: RequiredPermissionSet; + scope: string; +} + +export interface RequiredConnectorPermissions { + tenant?: TenantPermissions[]; + + licenses?: RequiredLicense[]; + + customs?: CustomPreCondition[]; + + resourceProvider?: ResourceProviderPermissions[]; +} + +export enum ConnectorAvailabilityStatus { + Available = 1, + FeatureFlag, + ComingSoon, + Internal, +} + +export enum ExplicitFeatureState { + PrivatePreview, + Disabled, +} + +export enum ExtensionEnvironment { + Local = 1, + Dogfood, + MPAC, + Prod, + Fairfax, + USSec, + USNat, +} + +export interface FeatureStateOverride { + cloudEnvironment: ExtensionEnvironment; + explicitFeatureState: ExplicitFeatureState; +} + +interface FeatureState { + defaultValue?: boolean; + featureStateOverrides?: FeatureStateOverride[]; +} + +export interface FeatureConfig extends FeatureState { + feature: string /* The string to be used in the URL in order to control the flag */; + defaultValue?: boolean /* Default: false */; + featureStateOverrides?: FeatureStateOverride[]; +} + +export enum ConnectivityCriteriaType { + IsConnectedQuery = "IsConnectedQuery", + OmsSolutions = "OmsSolutions", + SentinelKinds = "SentinelKinds", + AzureActiveDirectory = "AzureActiveDirectory", + SecurityEvents = "SecurityEvents", + AzureGraph = "AzureGraph", +} + +export interface ConnectivityCriteriaModel { + type: ConnectivityCriteriaType; + value?: T; +} + +export interface ConnectorAvailability { + status: ConnectorAvailabilityStatus; + isPreview?: boolean; + featureFlag?: FeatureConfig; + previewRegistrationLink?: string; + isComingSoon?: boolean; +} + +export enum InstructionType { + SentinelResourceProvider = "SentinelResourceProvider", + ThreatIntelligenceTaxii = "ThreatIntelligenceTaxii", + CopyableLabel = "CopyableLabel", + OmsSolution = "OmsSolutions", + InstallAgent = "InstallAgent", + InstructionStepsGroup = "InstructionStepsGroup", + InfoMessage = "InfoMessage", + + // Internal + Office365 = "Office365", + OfficeATP = "OfficeATP", + AzureSecurityCenterSubscriptions = "AzureSecurityCenterSubscriptions", + AWS = "AWS", + AzureActiveDirectory = "AzureActiveDirectory", + SecurityEvents = "SecurityEvents", + FilterAlert = "FilterAlert", + MicrosoftDefenderATP = "MicrosoftDefenderATP", + MicrosoftDefenderATPEvents = "MicrosoftDefenderATPEvents", + MicrosoftThreatProtection = "MicrosoftThreatProtection", + MicrosoftThreatIntelligence = "MicrosoftThreatIntelligence", + IoT = "IoT", + OfficeDataTypes = "OfficeDataTypes", + MCasDataTypes = "MCasDataTypes", + AADDataTypes = "AADDataTypes", + OAuth = "OAuth", +} + +export abstract class ConnectorInstructionModelBase { + public abstract type: InstructionType; + + public parameters?: T; + + constructor(parameters: T) { + this.parameters = parameters; + } +} + +export interface InstructionStep { + title?: string; + description?: string; + instructions?: ConnectorInstructionModelBase[]; + innerSteps?: InstructionStep[]; + featureFlag?: FeatureConfig; + bottomBorder?: boolean; + isComingSoon?: boolean; +} \ No newline at end of file diff --git a/.script/utils/schemas/DataConnectorSchema.json b/.script/utils/schemas/DataConnectorSchema.json new file mode 100644 index 0000000000..9d0ff8f57f --- /dev/null +++ b/.script/utils/schemas/DataConnectorSchema.json @@ -0,0 +1,442 @@ +{ + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "title": { + "type": "string" + }, + "publisher": { + "type": "string" + }, + "descriptionMarkdown": { + "type": "string" + }, + "additionalRequirementBanner": { + "type": "string" + }, + "graphQueries": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "metricName": { + "type": "string" + }, + "legend": { + "type": "string" + }, + "baseQuery": { + "type": "string" + } + }, + "required": [ + "metricName", + "legend", + "baseQuery" + ] + } + ] + }, + "sampleQueries": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "query": { + "type": "string" + } + }, + "required": [ + "description", + "query" + ] + }, + { + "type": "object", + "properties": { + "description": { + "type": "string" + }, + "query": { + "type": "string" + } + }, + "required": [ + "description", + "query" + ] + } + ] + }, + "dataTypes": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "lastDataReceivedQuery": { + "type": "string" + } + }, + "required": [ + "name", + "lastDataReceivedQuery" + ] + } + ] + }, + "connectivityCriterias": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "value": { + "type": "array", + "items": [ + { + "type": "string" + } + ] + } + }, + "required": [ + "type", + "value" + ] + } + ] + }, + "availability": { + "type": "object", + "properties": { + "status": { + "type": "integer" + } + }, + "required": [ + "status" + ] + }, + "permissions": { + "type": "object", + "properties": { + "resourceProvider": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "provider": { + "type": "string" + }, + "permissionsDisplayText": { + "type": "string" + }, + "providerDisplayName": { + "type": "string" + }, + "scope": { + "type": "string" + }, + "requiredPermissions": { + "type": "object", + "properties": { + "write": { + "type": "boolean" + }, + "delete": { + "type": "boolean" + } + }, + "required": [ + "write", + "delete" + ] + } + }, + "required": [ + "provider", + "permissionsDisplayText", + "providerDisplayName", + "scope", + "requiredPermissions" + ] + } + ] + }, + "customs": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "name", + "description" + ] + } + ] + } + }, + "required": [ + "resourceProvider" + ] + }, + "instructionSteps": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": {} + } + }, + "required": [ + "title", + "description", + "instructions" + ] + }, + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "instructionSteps": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "properties": { + "linkType": { + "type": "string" + } + }, + "required": [ + "linkType" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "parameters", + "type" + ] + } + ] + } + }, + "required": [ + "title", + "description", + "instructions" + ] + }, + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "properties": { + "linkType": { + "type": "string" + } + }, + "required": [ + "linkType" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "parameters", + "type" + ] + } + ] + } + }, + "required": [ + "title", + "description", + "instructions" + ] + } + ] + } + }, + "required": [ + "title", + "instructionSteps" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "parameters", + "type" + ] + } + ] + } + }, + "required": [ + "title", + "description", + "instructions" + ] + }, + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + }, + "instructions": { + "type": "array", + "items": [ + { + "type": "object", + "properties": { + "parameters": { + "type": "object", + "properties": { + "linkType": { + "type": "string" + } + }, + "required": [ + "linkType" + ] + }, + "type": { + "type": "string" + } + }, + "required": [ + "parameters", + "type" + ] + } + ] + } + }, + "required": [ + "title", + "description", + "instructions" + ] + }, + { + "type": "object", + "properties": { + "title": { + "type": "string" + }, + "description": { + "type": "string" + } + }, + "required": [ + "title", + "description" + ] + } + ] + } + }, + "required": [ + "id", + "title", + "publisher", + "descriptionMarkdown", + "graphQueries", + "sampleQueries", + "dataTypes", + "connectivityCriterias", + "availability", + "permissions", + "instructionSteps" + ] +} \ No newline at end of file diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 63ab429469..5be454a61c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -16,3 +16,4 @@ jobs: - template: .azure-pipelines/yamlFileValidator.yaml - template: .azure-pipelines/jsonFileValidator.yaml - template: .azure-pipelines/documentsLinkValidator.yaml +- template: .azure-pipelines/dataConnectorValidations.yaml