Logic to validate schema of DataConnector json

This commit is contained in:
v-rucdu 2020-12-31 18:02:44 +05:30
Родитель ca15137b32
Коммит 9ffbc33efd
8 изменённых файлов: 879 добавлений и 0 удалений

Просмотреть файл

@ -0,0 +1,12 @@
jobs:
- job: "DataConnectorValidations"
pool:
vmImage: 'Ubuntu 16.04'
steps:
- task: Npm@1
displayName: 'npm install'
inputs:
verbose: false
command: 'install'
- script: 'npm run tsc && node .script/dataConnectorValidator.js'
displayName: 'Data Connector Validations'

Просмотреть файл

@ -0,0 +1,31 @@
import fs from "fs";
import { runCheckOverChangedFiles } from "./utils/changedFilesValidator";
import { ExitCode } from "./utils/exitCode";
import { isValidSchema } from "./utils/jsonSchemaChecker";
import * as logger from "./utils/logger";
export async function IsValidDataConnectorSchema(filePath: string): Promise<ExitCode> {
let dataConnector = JSON.parse(fs.readFileSync(filePath, "utf8"));
let schema = JSON.parse(fs.readFileSync(".script/utils/schemas/DataConnectorSchema.json", "utf8"));
isValidSchema(dataConnector, schema);
return ExitCode.SUCCESS;
}
let fileTypeSuffixes = ["*.json"];
let filePathFolderPrefixes = ["DataConnectors"];
let fileKinds = ["Added", "Modified"];
let CheckOptions = {
onCheckFile: (filePath: string) => {
return IsValidDataConnectorSchema(filePath);
},
onExecError: async (e: any, filePath: string) => {
console.log(`Data Connector Validation Failed. File path: ${filePath}. Error message: ${e.message}`);
},
onFinalFailed: async () => {
logger.logError("An error occurred, please open an issue");
},
};
runCheckOverChangedFiles(CheckOptions, fileKinds, fileTypeSuffixes, filePathFolderPrefixes);

Просмотреть файл

@ -0,0 +1,25 @@
import chai, { expect } from "chai";
import chaiAsPromised from "chai-as-promised";
import { ExitCode } from "../../utils/exitCode";
import { IsValidDataConnectorSchema } from "../../dataConnectorValidator";
chai.use(chaiAsPromised);
describe("dataConnectorValidator", () => {
it("should pass when dataConnectorSchema.json is valid", async () => {
await checkValid(".script/tests/dataConnectorValidatorTest/testFiles/validDataConnectorSchema.json");
});
it("should throw an exception when dataConnectorSchema.json is missing a required property", async () => {
await checkInvalid(".script/tests/dataConnectorValidatorTest/testFiles/missingRequiredPropertyDataConnectorSchema.json", "SchemaError");
});
async function checkValid(filePath: string): Promise<Chai.PromisedAssertion> {
let result = await IsValidDataConnectorSchema(filePath);
expect(result).to.equal(ExitCode.SUCCESS);
}
async function checkInvalid(filePath: string, expectedError: string): Promise<Chai.PromisedAssertion> {
expect(IsValidDataConnectorSchema(filePath)).eventually.rejectedWith(Error).and.have.property("name", expectedError);
}
});

Просмотреть файл

@ -0,0 +1,89 @@
{
"id": "NXLogDnsLogs",
"title": "NXLog DNS Logs",
"descriptionMarkdown": "The NXLog DNS Logs data connector uses Event Tracing for Windows ([ETW](https://docs.microsoft.com/windows/apps/trace-processing/overview)) for collecting both Audit and Analytical DNS Server events. The [NXLog *im_etw* module](https://nxlog.co/documentation/nxlog-user-guide/im_etw.html) reads event tracing data directly for maximum efficiency, without the need to capture the event trace into an .etl file. This REST API connector can forward DNS Server events to Azure Sentinel in real-time.",
"graphQueries": [
{
"metricName": "Total data received",
"legend": "DNS_Logs_CL",
"baseQuery": "DNS_Logs_CL"
}
],
"sampleQueries": [
{
"description" : "EventID frequency",
"query": "DNS_Logs_CL| summarize EventCount = count() by EventID = EventID_d| order by EventID| render barchart"
},
{
"description" : "Event Type counts",
"query": "DNS_Logs_CL| where strlen(DNS_LogType_s) > 1| summarize EventCount = count() by EventType = DNS_LogType_s| order by EventType asc | render barchart"
}
],
"dataTypes": [
{
"name": "DNS_Logs_CL",
"lastDataReceivedQuery": "DNS_Logs_CL | summarize Time = max(TimeGenerated) | where isnotempty(Time)"
}
],
"connectivityCriterias": [
{
"type": "IsConnectedQuery",
"value": [
"DNS_Logs_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)"
]
}
],
"availability": {
"status": 1
},
"permissions": {
"resourceProvider": [
{
"provider": "Microsoft.OperationalInsights/workspaces",
"permissionsDisplayText": "read and write permissions are required.",
"providerDisplayName": "Workspace",
"scope": "Workspace",
"requiredPermissions": {
"write": true,
"read": true,
"delete": true
}
},
{
"provider": "Microsoft.OperationalInsights/workspaces/sharedKeys",
"permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key)",
"providerDisplayName": "Keys",
"scope": "Workspace",
"requiredPermissions": {
"action": true
}
}
]
},
"instructionSteps": [
{
"title": "",
"description": "Follow the step-by-step instructions in the *NXLog User Guide* Integration Topic [Microsoft Azure Sentinel](https://nxlog.co/documentation/nxlog-user-guide/sentinel.html) to configure this connector.",
"instructions": [
{
"parameters": {
"fillWith": [
"WorkspaceId"
],
"label": "Workspace ID"
},
"type": "CopyableLabel"
},
{
"parameters": {
"fillWith": [
"PrimaryKey"
],
"label": "Primary Key"
},
"type": "CopyableLabel"
}
]
}
]
}

Просмотреть файл

@ -0,0 +1,90 @@
{
"id": "NXLogDnsLogs",
"title": "NXLog DNS Logs",
"publisher": "NXLog",
"descriptionMarkdown": "The NXLog DNS Logs data connector uses Event Tracing for Windows ([ETW](https://docs.microsoft.com/windows/apps/trace-processing/overview)) for collecting both Audit and Analytical DNS Server events. The [NXLog *im_etw* module](https://nxlog.co/documentation/nxlog-user-guide/im_etw.html) reads event tracing data directly for maximum efficiency, without the need to capture the event trace into an .etl file. This REST API connector can forward DNS Server events to Azure Sentinel in real-time.",
"graphQueries": [
{
"metricName": "Total data received",
"legend": "DNS_Logs_CL",
"baseQuery": "DNS_Logs_CL"
}
],
"sampleQueries": [
{
"description" : "EventID frequency",
"query": "DNS_Logs_CL| summarize EventCount = count() by EventID = EventID_d| order by EventID| render barchart"
},
{
"description" : "Event Type counts",
"query": "DNS_Logs_CL| where strlen(DNS_LogType_s) > 1| summarize EventCount = count() by EventType = DNS_LogType_s| order by EventType asc | render barchart"
}
],
"dataTypes": [
{
"name": "DNS_Logs_CL",
"lastDataReceivedQuery": "DNS_Logs_CL | summarize Time = max(TimeGenerated) | where isnotempty(Time)"
}
],
"connectivityCriterias": [
{
"type": "IsConnectedQuery",
"value": [
"DNS_Logs_CL | summarize LastLogReceived = max(TimeGenerated) | project IsConnected = LastLogReceived > ago(30d)"
]
}
],
"availability": {
"status": 1
},
"permissions": {
"resourceProvider": [
{
"provider": "Microsoft.OperationalInsights/workspaces",
"permissionsDisplayText": "read and write permissions are required.",
"providerDisplayName": "Workspace",
"scope": "Workspace",
"requiredPermissions": {
"write": true,
"read": true,
"delete": true
}
},
{
"provider": "Microsoft.OperationalInsights/workspaces/sharedKeys",
"permissionsDisplayText": "read permissions to shared keys for the workspace are required. [See the documentation to learn more about workspace keys](https://docs.microsoft.com/azure/azure-monitor/platform/agent-windows#obtain-workspace-id-and-key)",
"providerDisplayName": "Keys",
"scope": "Workspace",
"requiredPermissions": {
"action": true
}
}
]
},
"instructionSteps": [
{
"title": "",
"description": "Follow the step-by-step instructions in the *NXLog User Guide* Integration Topic [Microsoft Azure Sentinel](https://nxlog.co/documentation/nxlog-user-guide/sentinel.html) to configure this connector.",
"instructions": [
{
"parameters": {
"fillWith": [
"WorkspaceId"
],
"label": "Workspace ID"
},
"type": "CopyableLabel"
},
{
"parameters": {
"fillWith": [
"PrimaryKey"
],
"label": "Primary Key"
},
"type": "CopyableLabel"
}
]
}
]
}

Просмотреть файл

@ -0,0 +1,189 @@
export interface DataConnectorModel {
id: string;
title: string;
publisher: string;
descriptionMarkdown: string;
graphQueries: GraphQuery[];
sampleQueries: SampleQuery[];
dataTypes: DataType[];
availability: ConnectorAvailability;
instructionSteps: InstructionStep[];
permissions: RequiredConnectorPermissions;
isConnectivityCriteriasMatchSome?: boolean;
connectivityCriterias: ConnectivityCriteriaModel<any>[];
additionalRequirementBanner?: string;
logo?: string ;
}
export interface GraphQuery {
metricName: string;
legend: string;
baseQuery: string;
}
export interface SampleQuery {
description?: string;
query: string;
}
export interface DataType {
name: string;
lastDataReceivedQuery: string;
}
export enum TenantPermissions {
GlobalAdmin = "GlobalAdmin",
SecurityAdmin = "SecurityAdmin",
SecurityReader = "SecurityReader",
InformationProtection = "InformationProtection",
}
export enum RequiredLicense {
OfficeATP = "OfficeATP",
Office365 = "Office365",
AadP1P2 = "AadP1P2",
Mcas = "Mcas",
Aatp = "Aatp",
Asc = "Asc",
Mdatp = "Mdatp",
Mtp = "Mtp",
IoT = "IoT",
}
export interface CustomPreCondition {
name: string;
description: string;
}
export interface RequiredPermissionSet {
read?: boolean;
write?: boolean;
delete?: boolean;
action?: boolean;
}
export interface ResourceProviderPermissions {
provider: string;
providerDisplayName: string;
permissionsDisplayText: string;
requiredPermissions: RequiredPermissionSet;
scope: string;
}
export interface RequiredConnectorPermissions {
tenant?: TenantPermissions[];
licenses?: RequiredLicense[];
customs?: CustomPreCondition[];
resourceProvider?: ResourceProviderPermissions[];
}
export enum ConnectorAvailabilityStatus {
Available = 1,
FeatureFlag,
ComingSoon,
Internal,
}
export enum ExplicitFeatureState {
PrivatePreview,
Disabled,
}
export enum ExtensionEnvironment {
Local = 1,
Dogfood,
MPAC,
Prod,
Fairfax,
USSec,
USNat,
}
export interface FeatureStateOverride {
cloudEnvironment: ExtensionEnvironment;
explicitFeatureState: ExplicitFeatureState;
}
interface FeatureState {
defaultValue?: boolean;
featureStateOverrides?: FeatureStateOverride[];
}
export interface FeatureConfig extends FeatureState {
feature: string /* The string to be used in the URL in order to control the flag */;
defaultValue?: boolean /* Default: false */;
featureStateOverrides?: FeatureStateOverride[];
}
export enum ConnectivityCriteriaType {
IsConnectedQuery = "IsConnectedQuery",
OmsSolutions = "OmsSolutions",
SentinelKinds = "SentinelKinds",
AzureActiveDirectory = "AzureActiveDirectory",
SecurityEvents = "SecurityEvents",
AzureGraph = "AzureGraph",
}
export interface ConnectivityCriteriaModel<T> {
type: ConnectivityCriteriaType;
value?: T;
}
export interface ConnectorAvailability {
status: ConnectorAvailabilityStatus;
isPreview?: boolean;
featureFlag?: FeatureConfig;
previewRegistrationLink?: string;
isComingSoon?: boolean;
}
export enum InstructionType {
SentinelResourceProvider = "SentinelResourceProvider",
ThreatIntelligenceTaxii = "ThreatIntelligenceTaxii",
CopyableLabel = "CopyableLabel",
OmsSolution = "OmsSolutions",
InstallAgent = "InstallAgent",
InstructionStepsGroup = "InstructionStepsGroup",
InfoMessage = "InfoMessage",
// Internal
Office365 = "Office365",
OfficeATP = "OfficeATP",
AzureSecurityCenterSubscriptions = "AzureSecurityCenterSubscriptions",
AWS = "AWS",
AzureActiveDirectory = "AzureActiveDirectory",
SecurityEvents = "SecurityEvents",
FilterAlert = "FilterAlert",
MicrosoftDefenderATP = "MicrosoftDefenderATP",
MicrosoftDefenderATPEvents = "MicrosoftDefenderATPEvents",
MicrosoftThreatProtection = "MicrosoftThreatProtection",
MicrosoftThreatIntelligence = "MicrosoftThreatIntelligence",
IoT = "IoT",
OfficeDataTypes = "OfficeDataTypes",
MCasDataTypes = "MCasDataTypes",
AADDataTypes = "AADDataTypes",
OAuth = "OAuth",
}
export abstract class ConnectorInstructionModelBase<T extends {}> {
public abstract type: InstructionType;
public parameters?: T;
constructor(parameters: T) {
this.parameters = parameters;
}
}
export interface InstructionStep {
title?: string;
description?: string;
instructions?: ConnectorInstructionModelBase<any>[];
innerSteps?: InstructionStep[];
featureFlag?: FeatureConfig;
bottomBorder?: boolean;
isComingSoon?: boolean;
}

Просмотреть файл

@ -0,0 +1,442 @@
{
"type": "object",
"properties": {
"id": {
"type": "string"
},
"title": {
"type": "string"
},
"publisher": {
"type": "string"
},
"descriptionMarkdown": {
"type": "string"
},
"additionalRequirementBanner": {
"type": "string"
},
"graphQueries": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"metricName": {
"type": "string"
},
"legend": {
"type": "string"
},
"baseQuery": {
"type": "string"
}
},
"required": [
"metricName",
"legend",
"baseQuery"
]
}
]
},
"sampleQueries": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"description": {
"type": "string"
},
"query": {
"type": "string"
}
},
"required": [
"description",
"query"
]
},
{
"type": "object",
"properties": {
"description": {
"type": "string"
},
"query": {
"type": "string"
}
},
"required": [
"description",
"query"
]
}
]
},
"dataTypes": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"name": {
"type": "string"
},
"lastDataReceivedQuery": {
"type": "string"
}
},
"required": [
"name",
"lastDataReceivedQuery"
]
}
]
},
"connectivityCriterias": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"type": {
"type": "string"
},
"value": {
"type": "array",
"items": [
{
"type": "string"
}
]
}
},
"required": [
"type",
"value"
]
}
]
},
"availability": {
"type": "object",
"properties": {
"status": {
"type": "integer"
}
},
"required": [
"status"
]
},
"permissions": {
"type": "object",
"properties": {
"resourceProvider": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"provider": {
"type": "string"
},
"permissionsDisplayText": {
"type": "string"
},
"providerDisplayName": {
"type": "string"
},
"scope": {
"type": "string"
},
"requiredPermissions": {
"type": "object",
"properties": {
"write": {
"type": "boolean"
},
"delete": {
"type": "boolean"
}
},
"required": [
"write",
"delete"
]
}
},
"required": [
"provider",
"permissionsDisplayText",
"providerDisplayName",
"scope",
"requiredPermissions"
]
}
]
},
"customs": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"name": {
"type": "string"
},
"description": {
"type": "string"
}
},
"required": [
"name",
"description"
]
}
]
}
},
"required": [
"resourceProvider"
]
},
"instructionSteps": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "array",
"items": {}
}
},
"required": [
"title",
"description",
"instructions"
]
},
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"parameters": {
"type": "object",
"properties": {
"title": {
"type": "string"
},
"instructionSteps": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"parameters": {
"type": "object",
"properties": {
"linkType": {
"type": "string"
}
},
"required": [
"linkType"
]
},
"type": {
"type": "string"
}
},
"required": [
"parameters",
"type"
]
}
]
}
},
"required": [
"title",
"description",
"instructions"
]
},
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"parameters": {
"type": "object",
"properties": {
"linkType": {
"type": "string"
}
},
"required": [
"linkType"
]
},
"type": {
"type": "string"
}
},
"required": [
"parameters",
"type"
]
}
]
}
},
"required": [
"title",
"description",
"instructions"
]
}
]
}
},
"required": [
"title",
"instructionSteps"
]
},
"type": {
"type": "string"
}
},
"required": [
"parameters",
"type"
]
}
]
}
},
"required": [
"title",
"description",
"instructions"
]
},
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
},
"instructions": {
"type": "array",
"items": [
{
"type": "object",
"properties": {
"parameters": {
"type": "object",
"properties": {
"linkType": {
"type": "string"
}
},
"required": [
"linkType"
]
},
"type": {
"type": "string"
}
},
"required": [
"parameters",
"type"
]
}
]
}
},
"required": [
"title",
"description",
"instructions"
]
},
{
"type": "object",
"properties": {
"title": {
"type": "string"
},
"description": {
"type": "string"
}
},
"required": [
"title",
"description"
]
}
]
}
},
"required": [
"id",
"title",
"publisher",
"descriptionMarkdown",
"graphQueries",
"sampleQueries",
"dataTypes",
"connectivityCriterias",
"availability",
"permissions",
"instructionSteps"
]
}

Просмотреть файл

@ -16,3 +16,4 @@ jobs:
- template: .azure-pipelines/yamlFileValidator.yaml
- template: .azure-pipelines/jsonFileValidator.yaml
- template: .azure-pipelines/documentsLinkValidator.yaml
- template: .azure-pipelines/dataConnectorValidations.yaml