updates to workflow and scripts
This commit is contained in:
vakohl
Родитель
22d1ae8289
Коммит
8b33548d3f
|
|
@ -60,11 +60,21 @@ jobs:
|
|||
git merge --abort
|
||||
exit 1
|
||||
fi
|
||||
- name: Run ASIM testers
|
||||
- name: Run ASIM Schema and Data tests PowerShell script
|
||||
uses: azure/powershell@v2
|
||||
with:
|
||||
inlineScript: |
|
||||
& ".script/tests/asimParsersTest/runAsimTesters.ps1"
|
||||
$filePath = ".script/tests/asimParsersTest/runAsimTesters.ps1"
|
||||
$url = "https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/runAsimTesters.ps1"
|
||||
# Check if file exists and delete if it does
|
||||
if (Test-Path $filePath) {
|
||||
Remove-Item $filePath -Force
|
||||
}
|
||||
# Download the file
|
||||
Write-Host "Downloading script from the master: $url"
|
||||
Invoke-WebRequest -Uri $url -OutFile $filePath
|
||||
# Execute the script
|
||||
& $filePath
|
||||
azPSVersion: "latest"
|
||||
errorActionPreference: continue
|
||||
failOnStandardError: false
|
||||
|
|
@ -102,9 +112,19 @@ jobs:
|
|||
pip install requests
|
||||
pip install PyYAML
|
||||
pip install tabulate
|
||||
- name: Run Python script
|
||||
- name: Run ASim parsers template validations python script
|
||||
run: |
|
||||
python .script/tests/asimParsersTest/VerifyASimParserTemplate.py
|
||||
filePath=".script/tests/asimParsersTest/VerifyASimParserTemplate.py"
|
||||
url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/VerifyASimParserTemplate.py"
|
||||
# Check if file exists and delete if it does
|
||||
if [ -f "$filePath" ]; then
|
||||
rm -f "$filePath"
|
||||
fi
|
||||
# Download the file
|
||||
echo "Downloading script from the master: $url"
|
||||
curl -o "$filePath" "$url"
|
||||
# Execute the script
|
||||
python "$filePath"
|
||||
Run-ASim-Parser-Filtering-Tests:
|
||||
name: Run ASim Parser Filtering tests
|
||||
runs-on: ubuntu-latest
|
||||
|
|
@ -142,9 +162,19 @@ jobs:
|
|||
- name: Login to Azure Public Cloud
|
||||
uses: azure/login@v2
|
||||
with:
|
||||
client-id: ${{ secrets.AZURE_CLIENT_ID }}
|
||||
client-id: ${{ secrets.AZURE_ASIM_CLIENT_ID }}
|
||||
tenant-id: ${{ secrets.AZURE_TENANT_ID }}
|
||||
allow-no-subscriptions: true
|
||||
- name: Run Python script
|
||||
- name: Run ASim parsers filtering tests python script
|
||||
run: |
|
||||
python .script/tests/asimParsersTest/ASimFilteringTest.py
|
||||
filePath=".script/tests/asimParsersTest/ASimFilteringTest.py"
|
||||
url="https://raw.githubusercontent.com/Azure/Azure-Sentinel/master/.script/tests/asimParsersTest/ASimFilteringTest.py"
|
||||
# Check if file exists and delete if it does
|
||||
if [ -f "$filePath" ]; then
|
||||
rm -f "$filePath"
|
||||
fi
|
||||
# Download the file
|
||||
echo "Downloading script from the master: $url"
|
||||
curl -o "$filePath" "$url"
|
||||
# Execute the script
|
||||
python "$filePath"
|
||||
|
|
@ -23,6 +23,9 @@ TIME_SPAN_IN_DAYS = 7
|
|||
# exclusion_file_path refers to the CSV file path containing a list of parsers. Despite failing tests, these parsers will not cause the overall workflow to fail
|
||||
exclusion_file_path = '.script/tests/asimParsersTest/ExclusionListForASimTests.csv'
|
||||
|
||||
# Sentinel Repo URL
|
||||
SentinelRepoUrl = f"https://github.com/Azure/Azure-Sentinel.git"
|
||||
|
||||
# Negative value as it is cannot be a port number and less likely to be an ID of some event. Also, the absolute value is greater than the maximal possible port number.
|
||||
INT_DUMMY_VALUE = -967799
|
||||
# The index of the column with the value from a query response.
|
||||
|
|
@ -230,7 +233,18 @@ def read_exclusion_list_from_csv():
|
|||
def main():
|
||||
# Get modified ASIM Parser files along with their status
|
||||
current_directory = os.path.dirname(os.path.abspath(__file__))
|
||||
GetModifiedFiles = f"git diff --name-only origin/master {current_directory}/../../../Parsers/"
|
||||
|
||||
# Add upstream remote if not already present
|
||||
git_remote_command = "git remote"
|
||||
remote_result = subprocess.run(git_remote_command, shell=True, text=True, capture_output=True, check=True)
|
||||
if 'upstream' not in remote_result.stdout.split():
|
||||
git_add_upstream_command = f"git remote add upstream '{SentinelRepoUrl}'"
|
||||
subprocess.run(git_add_upstream_command, shell=True, text=True, capture_output=True, check=True)
|
||||
# Fetch from upstream
|
||||
git_fetch_upstream_command = "git fetch upstream"
|
||||
subprocess.run(git_fetch_upstream_command, shell=True, text=True, capture_output=True, check=True)
|
||||
|
||||
GetModifiedFiles = f"git diff --name-only upstream/master {current_directory}/../../../Parsers/"
|
||||
try:
|
||||
modified_files = subprocess.run(GetModifiedFiles, shell=True, text=True, capture_output=True, check=True)
|
||||
except subprocess.CalledProcessError as e:
|
||||
|
|
|
|||
|
|
@ -9,9 +9,11 @@ from urllib.parse import urlparse
|
|||
from tabulate import tabulate
|
||||
|
||||
# Constants
|
||||
SENTINEL_REPO_URL = f'https://raw.githubusercontent.com/Azure/Azure-Sentinel'
|
||||
SENTINEL_REPO_RAW_URL = f'https://raw.githubusercontent.com/Azure/Azure-Sentinel'
|
||||
SAMPLE_DATA_PATH = '/Sample%20Data/ASIM/'
|
||||
parser_exclusion_file_path = '.script/tests/asimParsersTest/ExclusionListForASimTests.csv'
|
||||
# Sentinel Repo URL
|
||||
SentinelRepoUrl = f"https://github.com/Azure/Azure-Sentinel.git"
|
||||
SCHEMA_INFO = [
|
||||
{"SchemaName": "AuditEvent", "SchemaVersion": "0.1", "SchemaTitle":"ASIM Audit Event Schema", "SchemaLink": "https://aka.ms/ASimAuditEventDoc"},
|
||||
{"SchemaName": "Authentication", "SchemaVersion": "0.1.3","SchemaTitle":"ASIM Authentication Schema","SchemaLink": "https://aka.ms/ASimAuthenticationDoc"},
|
||||
|
|
@ -42,7 +44,7 @@ def run():
|
|||
current_directory = os.path.dirname(os.path.abspath(__file__))
|
||||
modified_files = get_modified_files(current_directory)
|
||||
commit_number = get_current_commit_number()
|
||||
sample_data_url = f'{SENTINEL_REPO_URL}/{commit_number}/{SAMPLE_DATA_PATH}'
|
||||
sample_data_url = f'{SENTINEL_REPO_RAW_URL}/{commit_number}/{SAMPLE_DATA_PATH}'
|
||||
parser_yaml_files = filter_yaml_files(modified_files)
|
||||
print(f"{GREEN}Following files were found to be modified:{RESET}")
|
||||
for file in parser_yaml_files:
|
||||
|
|
@ -64,9 +66,9 @@ def run():
|
|||
else :
|
||||
# Skip the vim parser file as the corresponding ASim parser file is present and vim files will be tested with ASim files in upcoming steps.
|
||||
continue
|
||||
asim_parser_url = f'{SENTINEL_REPO_URL}/{commit_number}/{parser}'
|
||||
asim_parser_url = f'{SENTINEL_REPO_RAW_URL}/{commit_number}/{parser}'
|
||||
print(f'{YELLOW}Constructed parser raw url: {asim_parser_url}{RESET}') # uncomment for debugging
|
||||
asim_union_parser_url = f'{SENTINEL_REPO_URL}/{commit_number}/Parsers/ASim{schema_name}/Parsers/ASim{schema_name}.yaml'
|
||||
asim_union_parser_url = f'{SENTINEL_REPO_RAW_URL}/{commit_number}/Parsers/ASim{schema_name}/Parsers/ASim{schema_name}.yaml'
|
||||
print(f'{YELLOW}Constructed union parser raw url: {asim_union_parser_url}{RESET}') # uncomment for debugging
|
||||
asim_parser = read_github_yaml(asim_parser_url)
|
||||
asim_union_parser = read_github_yaml(asim_union_parser_url)
|
||||
|
|
@ -280,7 +282,17 @@ def filter_yaml_files(modified_files):
|
|||
return [line for line in modified_files if line.endswith('.yaml')]
|
||||
|
||||
def get_modified_files(current_directory):
|
||||
cmd = f"git diff --name-only origin/master {current_directory}/../../../Parsers/"
|
||||
|
||||
# Add upstream remote if not already present
|
||||
git_remote_command = "git remote"
|
||||
remote_result = subprocess.run(git_remote_command, shell=True, text=True, capture_output=True, check=True)
|
||||
if 'upstream' not in remote_result.stdout.split():
|
||||
git_add_upstream_command = f"git remote add upstream '{SentinelRepoUrl}'"
|
||||
subprocess.run(git_add_upstream_command, shell=True, text=True, capture_output=True, check=True)
|
||||
# Fetch from upstream
|
||||
git_fetch_upstream_command = "git fetch upstream"
|
||||
subprocess.run(git_fetch_upstream_command, shell=True, text=True, capture_output=True, check=True)
|
||||
cmd = f"git diff --name-only upstream/master {current_directory}/../../../Parsers/"
|
||||
try:
|
||||
return subprocess.check_output(cmd, shell=True).decode().split("\n")
|
||||
except subprocess.CalledProcessError as e:
|
||||
|
|
|
|||
|
|
@ -10,6 +10,8 @@ $reset = "`e[0m"
|
|||
|
||||
# Parser exclusion file path
|
||||
$ParserExclusionsFilePath ="$($PSScriptRoot)/ExclusionListForASimTests.csv"
|
||||
# Sentinel repository URL
|
||||
$SentinelRepoUrl = "https://github.com/Azure/Azure-Sentinel.git"
|
||||
|
||||
Class Parser {
|
||||
[string] $Name
|
||||
|
|
@ -26,8 +28,21 @@ Class Parser {
|
|||
}
|
||||
|
||||
function run {
|
||||
Write-Host "This is the script from PR."
|
||||
# Check if upstream remote already exists
|
||||
$remoteExists = Invoke-Expression "git remote" | Select-String -Pattern "upstream"
|
||||
|
||||
if (-not $remoteExists) {
|
||||
Write-Host "Adding upstream remote..."
|
||||
Invoke-Expression "git remote add upstream $SentinelRepoUrl"
|
||||
}
|
||||
|
||||
# Fetch the latest changes from upstream repositories
|
||||
Write-Host "Fetching latest changes from upstream..."
|
||||
Invoke-Expression "git fetch upstream" *> $null
|
||||
|
||||
# Get modified ASIM Parser files along with their status
|
||||
$modifiedFilesStatus = Invoke-Expression "git diff --name-status origin/master -- $($PSScriptRoot)/../../../Parsers/"
|
||||
$modifiedFilesStatus = Invoke-Expression "git diff --name-status upstream/master -- $($PSScriptRoot)/../../../Parsers/"
|
||||
# Split the output into lines
|
||||
$modifiedFilesStatusLines = $modifiedFilesStatus -split "`n"
|
||||
# Initialize an empty array to store the file names and their status
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче