Azure
/
Azure-Sentinel
зеркало из
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #2402 from Azure/v-rucdu/PRValidationFixesForSolutionsFolder 

Extension of PR Validations to solutions folder
This commit is contained in:
v-jayakal 2021-06-13 22:18:59 -07:00 коммит произвёл GitHub
Родитель 66014a39b2 188e1d2398
Коммит 4c98a3c186
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
11 изменённых файлов: 158 добавлений и 137 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

25
.script/dataConnectorValidator.ts
Просмотреть файл

@ -13,20 +13,27 @@ export async function IsValidDataConnectorSchema(filePath: string): Promise<Exit
if(!filePath.includes('Templates'))
{
let jsonFile = JSON.parse(fs.readFileSync(filePath, "utf8"));
if(isPotentialConnectorJson(jsonFile))
{
let connectorCategory = getConnectorCategory(jsonFile.dataTypes, jsonFile.instructionSteps);
let schema = JSON.parse(fs.readFileSync(".script/utils/schemas/"+ connectorCategory +"_ConnectorSchema.json", "utf8"));
isValidSchema(jsonFile, schema);
isValidId(jsonFile.id);
isValidDataType(jsonFile.dataTypes);
if(jsonFile.dataTypes[0].name.includes("Events"))
{
let connectorCategory = getConnectorCategory(jsonFile.dataTypes, jsonFile.instructionSteps);
let schema = JSON.parse(fs.readFileSync(".script/utils/schemas/"+ connectorCategory +"_ConnectorSchema.json", "utf8"));
isValidSchema(jsonFile, schema);
isValidId(jsonFile.id);
isValidDataType(jsonFile.dataTypes);
/* Disabling temporarily till we get confirmation from PM*/
// isValidFileName(filePath
isValidPermissions(jsonFile.permissions, connectorCategory);
/* Disabling temporarily till we get confirmation from PM*/
// isValidFileName(filePath
isValidPermissions(jsonFile.permissions, connectorCategory);
}
else{
console.warn(`Could not identify json file as a connector. Skipping File path: ${filePath}`)
}
}
else{
console.warn(`Could not identify json file as a connector. Skipping File path: ${filePath}`)
console.warn(`Skipping File as it is of type Events : ${filePath}`)
}
}
else{

34
.script/idChangeValidator.ts
Просмотреть файл

@ -10,27 +10,31 @@ const templateIdRegex:string = `(id: ${guidRegex}(.|\n)*){2}`;
const git: SimpleGit = gitP(workingDir);
export async function IsIdHasChanged(filePath: string): Promise<ExitCode> {
filePath = workingDir + '/' + filePath;
const pr = await GetPRDetails();
console.log(filePath);
if (typeof pr === "undefined") {
console.log("Azure DevOps CI for a Pull Request wasn't found. If issue persists - please open an issue");
return ExitCode.ERROR;
}
let options = [pr.targetBranch, pr.sourceBranch, filePath];
let diffSummary = await git.diff(options);
let idHasChanged = diffSummary.search(templateIdRegex) > 0;
if (idHasChanged){
throw new Error();
if(filePath.includes("Detections") || filePath.includes("Analytic Rules"))
{
filePath = workingDir + '/' + filePath;
const pr = await GetPRDetails();
console.log(filePath);
if (typeof pr === "undefined") {
console.log("Azure DevOps CI for a Pull Request wasn't found. If issue persists - please open an issue");
return ExitCode.ERROR;
}
let options = [pr.targetBranch, pr.sourceBranch, filePath];
let diffSummary = await git.diff(options);
let idHasChanged = diffSummary.search(templateIdRegex) > 0;
if (idHasChanged){
throw new Error();
}
}
return ExitCode.SUCCESS;
}
let fileKinds = ["Modified"];
let fileTypeSuffixes = ["yaml", "yml", "json"];
let filePathFolderPrefixes = ["Detections","Solutions"];
let filePathFolderPrefixes = ["Detections", "Solutions"];
let CheckOptions = {
onCheckFile: (filePath: string) => {
return IsIdHasChanged(filePath);

20
.script/logoValidator.ts
Просмотреть файл

@ -6,17 +6,21 @@ import { isValidLogoImage } from "./utils/LogoChecker/logoImageChecker";
import { isValidLogoImageSVGContent } from "./utils/LogoChecker/logoImageSVGChecker";
export async function IsValidLogo(FileName: string): Promise<ExitCode> {
isValidLogoImage(FileName);
const svgContent: string = fs.readFileSync(FileName, { encoding: "utf8", flag: "r" });
if(svgContent != "undefined")
{
isValidLogoImageSVGContent(svgContent)
}
return ExitCode.SUCCESS;
if(FileName.includes("Logos") || FileName.includes("Data Connectors/Logo"))
{
isValidLogoImage(FileName);
const svgContent: string = fs.readFileSync(FileName, { encoding: "utf8", flag: "r" });
if(svgContent != "undefined")
{
isValidLogoImageSVGContent(svgContent)
}
}
return ExitCode.SUCCESS;
}
let fileTypeSuffixes;
let filePathFolderPrefixes = ["Logos"];
let filePathFolderPrefixes = ["Logos","Solutions"];
let fileKinds = ["Added","Modified"];
let CheckOptions = {
onCheckFile: (filePath: string) => {

2
.script/playbooksValidator.ts
Просмотреть файл

@ -35,7 +35,7 @@ function validateARMTemplateWithPlaybookResource(filePath: string, playbookARMTe
}
let fileTypeSuffixes = ["azuredeploy.json"];
let filePathFolderPrefixes = ["Playbooks"];
let filePathFolderPrefixes = ["Playbooks","Solutions"];
let fileKinds = ["Modified"];
let CheckOptions = {
onCheckFile: (filePath: string) => {

10
.script/tests/NonAsciiValidationsTests/DetectionsYamlFilesTestData.cs
Просмотреть файл

@ -7,9 +7,9 @@ using System.Text;
namespace NonAsciiValidations.Tests
{
public class DetectionsYamlFilesTestData : FilesTestData
{
protected override string FolderName => "Detections";
protected override string FileExtension => "*.yaml";
}
public class DetectionsYamlFilesTestData : FilesTestData
{
protected override string[] FolderName => new string[2] { "Detections", "Solutions" };
protected override string FileExtension => "*.yaml";
}
}

10
.script/tests/NonAsciiValidationsTests/ExplorationQueriesYamlFilesTestData.cs
Просмотреть файл

@ -7,9 +7,9 @@ using System.Text;
namespace NonAsciiValidations.Tests
{
public class ExplorationQueriesYamlFilesTestData : FilesTestData
{
protected override string FolderName => "Exploration Queries";
protected override string FileExtension => "*.yaml";
}
public class ExplorationQueriesYamlFilesTestData : FilesTestData
{
protected override string[] FolderName => new string[2] { "Exploration Queries", "Solutions" };
protected override string FileExtension => "*.yaml";
}
}

103
.script/tests/NonAsciiValidationsTests/FilesTestData.cs
Просмотреть файл

@ -7,59 +7,62 @@ using System.Text;
namespace NonAsciiValidations.Tests
{
public abstract class FilesTestData : TheoryData<string,string>
{
public FilesTestData()
{
string rootPath = GetFilesRootPath();
var files = Directory.GetFiles(rootPath, FileExtension, SearchOption.AllDirectories).ToList();
files.ForEach(f => Add(Path.GetFileName(f),f));
}
public abstract class FilesTestData : TheoryData<string[], string>
{
public FilesTestData()
{
foreach (string folder in FolderName)
{
string rootPath = GetFilesRootPath(folder);
var files = Directory.GetFiles(rootPath, FileExtension, SearchOption.AllDirectories).ToList();
files.ForEach(f => Add(Path.GetFileName(f), f));
}
}
private string GetFilesRootPath()
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 6;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
var detectionPath = Path.Combine(rootDir.FullName, FolderName);
return detectionPath;
}
private string GetFilesRootPath(string folderName)
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 6;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
var detectionPath = Path.Combine(rootDir.FullName, folderName);
return detectionPath;
}
public string GetCustomTablesPath()
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 3;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
var rootPath = Path.Combine(rootDir.FullName, FolderName);
return rootPath;
}
public string GetCustomTablesPath(string folderName)
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 3;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
var rootPath = Path.Combine(rootDir.FullName, folderName);
return rootPath;
}
protected abstract string FolderName {get;}
protected abstract string FileExtension { get; }
protected abstract string[] FolderName { get; }
protected abstract string FileExtension { get; }
public static string GetSkipTemplatesPath()
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 3;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
return rootDir.FullName;
}
public static string GetSkipTemplatesPath()
{
var rootDir = Directory.CreateDirectory(GetAssemblyDirectory());
var testFolderDepth = 3;
for (int i = 0; i < testFolderDepth; i++)
{
rootDir = rootDir.Parent;
}
return rootDir.FullName;
}
private static string GetAssemblyDirectory()
{
string codeBase = Assembly.GetExecutingAssembly().CodeBase;
UriBuilder uri = new UriBuilder(codeBase);
string path = Uri.UnescapeDataString(uri.Path);
return Path.GetDirectoryName(path);
}
}
private static string GetAssemblyDirectory()
{
string codeBase = Assembly.GetExecutingAssembly().CodeBase;
UriBuilder uri = new UriBuilder(codeBase);
string path = Uri.UnescapeDataString(uri.Path);
return Path.GetDirectoryName(path);
}
}
}

10
.script/tests/NonAsciiValidationsTests/HuntingQueriesYamlFilesTestData.cs
Просмотреть файл

@ -7,9 +7,9 @@ using System.Text;
namespace NonAsciiValidations.Tests
{
public class HuntingQueriesYamlFilesTestData : FilesTestData
{
protected override string FolderName => "Hunting Queries";
protected override string FileExtension => "*.yaml";
}
public class HuntingQueriesYamlFilesTestData : FilesTestData
{
protected override string[] FolderName => new string[2] { "Hunting Queries", "Solutions" };
protected override string FileExtension => "*.yaml";
}
}

69
.script/tests/NonAsciiValidationsTests/NonAsciiValidationsTests.cs
Просмотреть файл

@ -8,43 +8,46 @@ using Xunit.Abstractions;
namespace NonAsciiValidations.Tests
{
public class NonAsciiValidationsTests
{
[Theory]
[ClassData(typeof(HuntingQueriesYamlFilesTestData))]
public void Validate_Hunting_HaveOnlyAsciiChars(string huntingYamlFileName,string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, huntingYamlFileName);
}
public class NonAsciiValidationsTests
{
[Theory]
[ClassData(typeof(HuntingQueriesYamlFilesTestData))]
public void Validate_Hunting_HaveOnlyAsciiChars(string huntingYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, huntingYamlFileName);
}
[Theory]
[ClassData(typeof(ParsersYamlFilesTestData))]
public void Validate_Parsers_HaveOnlyAsciiChars(string parserYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, parserYamlFileName);
}
[Theory]
[ClassData(typeof(ParsersYamlFilesTestData))]
public void Validate_Parsers_HaveOnlyAsciiChars(string parserYamlFileName, string yamlFullPath)
{
if (yamlFullPath.Contains("Parsers"))
{
ValidateOnlyAscii(yamlFullPath, parserYamlFileName);
}
}
[Theory]
[ClassData(typeof(ExplorationQueriesYamlFilesTestData))]
public void Validate_ExplorationQueries_HaveOnlyAsciiChars(string explorationQueryYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, explorationQueryYamlFileName);
}
[Theory]
[ClassData(typeof(ExplorationQueriesYamlFilesTestData))]
public void Validate_ExplorationQueries_HaveOnlyAsciiChars(string explorationQueryYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, explorationQueryYamlFileName);
}
[Theory]
[ClassData(typeof(DetectionsYamlFilesTestData))]
public void Validate_DetectionFile_HaveOnlyAsciiChars(string detectionsYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, detectionsYamlFileName);
}
[Theory]
[ClassData(typeof(DetectionsYamlFilesTestData))]
public void Validate_DetectionFile_HaveOnlyAsciiChars(string detectionsYamlFileName, string yamlFullPath)
{
ValidateOnlyAscii(yamlFullPath, detectionsYamlFileName);
}
private void ValidateOnlyAscii(string yamlFilePath, string yamlFileName)
{
var yaml = File.ReadAllText(yamlFilePath);
var nonAsciiCharMatch = Regex.Match(yaml, @"[^\u0000-\u007F]+");
Assert.False(nonAsciiCharMatch.Success, $"${yamlFileName} includes the non ascii char:{nonAsciiCharMatch.Value} string index:{nonAsciiCharMatch.Index}");
}
}
private void ValidateOnlyAscii(string yamlFilePath, string yamlFileName)
{
var yaml = File.ReadAllText(yamlFilePath);
var nonAsciiCharMatch = Regex.Match(yaml, @"[^\u0000-\u007F]+");
Assert.False(nonAsciiCharMatch.Success, $"${yamlFileName} includes the non ascii char:{nonAsciiCharMatch.Value} string index:{nonAsciiCharMatch.Index}");
}
}
}

10
.script/tests/NonAsciiValidationsTests/ParsersYamlFilesTestData.cs
Просмотреть файл

@ -7,9 +7,9 @@ using System.Text;
namespace NonAsciiValidations.Tests
{
public class ParsersYamlFilesTestData : FilesTestData
{
protected override string FolderName => "Parsers";
protected override string FileExtension => "*.*";
}
public class ParsersYamlFilesTestData : FilesTestData
{
protected override string[] FolderName => new string[2] { "Parsers", "Solutions" };
protected override string FileExtension => "*.*";
}
}

2
Tools/Sample-Data-Ingest-Tool/SampleDataIngestTool/config.txt
Просмотреть файл

@ -4,4 +4,4 @@
"clientId": "enter_your_clientId_here",
"clientSecret": "enter_your_client_secret_here",
"domain": "enter_your_app_domain_or_tenantId_here"
}
}