Adding a Regression Job (#1012)
* Add new regression pipeline. This workload runs on four pipelines, Two target versions of oav, both validate-spec and validate-example on each. Within each pipeline, we clone the complete azure-rest-api-specs repository and invoke the chosen oav version + check against each spec present within the repo * Finally, we diff the outputs, to catch any OAV behavior regressions Co-authored-by: semick-dev <sbeddall@gmail.com>
This commit is contained in:
Scott Beddall
GitHub
Родитель
a0291e3249
Коммит
760a3352a0
|
|
@ -0,0 +1,52 @@
|
|||
parameters:
|
||||
- name: TargetVersion
|
||||
type: string
|
||||
default: 'LOCAL' #specialcase value that will install the local version of oav
|
||||
- name: Type
|
||||
type: string
|
||||
|
||||
|
||||
# assume presence of variable $(OutputFolder)
|
||||
# $(RestSpecsRepo)
|
||||
|
||||
steps:
|
||||
- bash: |
|
||||
mkdir -p $(OutputFolder)
|
||||
mkdir -p $(RestSpecsRepo)
|
||||
displayName: Create Folders
|
||||
|
||||
- task: UseNode@1
|
||||
inputs:
|
||||
versionSpec: '16.x'
|
||||
checkLatest: true
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
displayName: 'Use Python 3.11'
|
||||
inputs:
|
||||
versionSpec: "3.11"
|
||||
|
||||
- bash: |
|
||||
if [[ "${{ parameters.TargetVersion }}" == "LOCAL" ]]; then
|
||||
npm ci
|
||||
npm run build
|
||||
npm link
|
||||
else
|
||||
npm install -g oav@${{ parameters.TargetVersion }}
|
||||
fi
|
||||
displayName: Install OAV
|
||||
workingDirectory: $(Build.SourcesDirectory)
|
||||
|
||||
- bash: |
|
||||
git clone https://github.com/azure/azure-rest-api-specs.git --depth=1 $(RestSpecsRepo)
|
||||
displayName: Clone the Git Repo
|
||||
|
||||
- bash: |
|
||||
python $(Build.SourcesDirectory)/.ci/scripts/run_oav_regression.py --oav oav --target $(RestSpecsRepo) --output $(OutputFolder) --type "${{ parameters.Type }}"
|
||||
displayName: Run the Regression Script
|
||||
timeoutInMinutes: 350
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
condition: always()
|
||||
inputs:
|
||||
pathtoPublish: $(OutputFolder)
|
||||
artifactName: "${{ parameters.TargetVersion }}-${{ parameters.Type }}"
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
# TODO: refactor the hilariously common jobs
|
||||
|
||||
parameters:
|
||||
- name: BeforeVersion
|
||||
type: string
|
||||
default: ''
|
||||
- name: AfterVersion
|
||||
type: string
|
||||
default: 'LOCAL' #specialcase value that will install the local version of oav
|
||||
|
||||
pool:
|
||||
name: azsdk-pool-mms-ubuntu-2004-general
|
||||
vmImage: MMSUbuntu20.04
|
||||
|
||||
jobs:
|
||||
- job: RegressionASpecs
|
||||
variables:
|
||||
- name: OutputFolder
|
||||
value: $(Build.ArtifactStagingDirectory)/${{ parameters.BeforeVersion }}
|
||||
- name: RestSpecsRepo
|
||||
value: $(Build.ArtifactStagingDirectory)/azure-rest-api-specs
|
||||
timeoutInMinutes: 360
|
||||
displayName: Run ${{ parameters.BeforeVersion }} Specs
|
||||
steps:
|
||||
- template: ./regression-steps.yml
|
||||
parameters:
|
||||
TargetVersion: ${{ parameters.BeforeVersion }}
|
||||
Type: "validate-spec"
|
||||
|
||||
- job: RegressionAExamples
|
||||
variables:
|
||||
- name: OutputFolder
|
||||
value: $(Build.ArtifactStagingDirectory)/${{ parameters.BeforeVersion }}
|
||||
- name: RestSpecsRepo
|
||||
value: $(Build.ArtifactStagingDirectory)/azure-rest-api-specs
|
||||
timeoutInMinutes: 360
|
||||
displayName: Run ${{ parameters.BeforeVersion }} Examples
|
||||
steps:
|
||||
- template: ./regression-steps.yml
|
||||
parameters:
|
||||
TargetVersion: ${{ parameters.BeforeVersion }}
|
||||
Type: "validate-example"
|
||||
|
||||
- job: RegressionBSpecs
|
||||
variables:
|
||||
- name: OutputFolder
|
||||
value: $(Build.ArtifactStagingDirectory)/${{ parameters.AfterVersion }}
|
||||
- name: RestSpecsRepo
|
||||
value: $(Build.ArtifactStagingDirectory)/azure-rest-api-specs
|
||||
timeoutInMinutes: 360
|
||||
displayName: Run ${{ parameters.AfterVersion }} Specs
|
||||
steps:
|
||||
- template: ./regression-steps.yml
|
||||
parameters:
|
||||
TargetVersion: ${{ parameters.AfterVersion }}
|
||||
Type: "validate-spec"
|
||||
|
||||
- job: RegressionBExamples
|
||||
variables:
|
||||
- name: OutputFolder
|
||||
value: $(Build.ArtifactStagingDirectory)/${{ parameters.AfterVersion }}
|
||||
- name: RestSpecsRepo
|
||||
value: $(Build.ArtifactStagingDirectory)/azure-rest-api-specs
|
||||
timeoutInMinutes: 360
|
||||
displayName: Run ${{ parameters.AfterVersion }} Examples
|
||||
steps:
|
||||
- template: ./regression-steps.yml
|
||||
parameters:
|
||||
TargetVersion: ${{ parameters.AfterVersion }}
|
||||
Type: "validate-example"
|
||||
|
||||
- job: Summarize
|
||||
timeoutInMinutes: 180
|
||||
dependsOn:
|
||||
- RegressionASpecs
|
||||
- RegressionBSpecs
|
||||
- RegressionAExamples
|
||||
- RegressionBExamples
|
||||
displayName: Run Diff
|
||||
steps:
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
artifactName: "${{ parameters.BeforeVersion }}-validate-spec"
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/before
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
artifactName: "${{ parameters.BeforeVersion }}-validate-example"
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/before
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
artifactName: "${{ parameters.AfterVersion }}-validate-spec"
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/after
|
||||
- task: DownloadPipelineArtifact@2
|
||||
inputs:
|
||||
artifactName: "${{ parameters.AfterVersion }}-validate-example"
|
||||
targetPath: $(Build.ArtifactStagingDirectory)/after
|
||||
- bash: |
|
||||
echo "CACHE BEFORE"
|
||||
cat $(Build.ArtifactStagingDirectory)/before/.spec_cache && rm $(Build.ArtifactStagingDirectory)/before/.spec_cache
|
||||
|
||||
echo "CACHE AFTER"
|
||||
cat $(Build.ArtifactStagingDirectory)/after/.spec_cache && rm $(Build.ArtifactStagingDirectory)/after/.spec_cache
|
||||
displayName: Dump Spec Caches
|
||||
- bash: |
|
||||
diff -qr $(Build.ArtifactStagingDirectory)/before $(Build.ArtifactStagingDirectory)/after
|
||||
displayName: Diff outputs
|
||||
|
|
@ -0,0 +1,211 @@
|
|||
# This script is used to invoke oav (examples AND specification) against every specification
|
||||
# discovered within a target repo.
|
||||
#
|
||||
# It is intended to be used twice, with different invoking versions of oav.
|
||||
#
|
||||
# FOR COMPATIBILITY The script expects that:
|
||||
# - node 16+ is on the PATH
|
||||
|
||||
|
||||
import glob
|
||||
import argparse
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
import difflib
|
||||
from dataclasses import dataclass
|
||||
|
||||
import subprocess
|
||||
from typing import List, Dict, Tuple
|
||||
|
||||
CACHE_FILE_NAME: str = ".spec_cache"
|
||||
|
||||
|
||||
@dataclass
|
||||
class OAVScanResult:
|
||||
"""Used to track the results of an oav invocation"""
|
||||
|
||||
target_folder: str
|
||||
stdout: str
|
||||
stderr: str
|
||||
success: int
|
||||
oav_version: str
|
||||
|
||||
@property
|
||||
def stdout_length_in_bytes(self) -> int:
|
||||
return os.path.getsize(self.stdout)
|
||||
|
||||
@property
|
||||
def stderr_length_in_bytes(self) -> int:
|
||||
return os.path.getsize(self.stderr)
|
||||
|
||||
|
||||
def get_oav_output(
|
||||
oav_exe: str,
|
||||
target_folder: str,
|
||||
collection_std_out: str,
|
||||
collection_std_err: str,
|
||||
oav_command: str,
|
||||
oav_version: str,
|
||||
) -> OAVScanResult:
|
||||
try:
|
||||
with open(collection_std_out, "w", encoding="utf-8") as out, open(
|
||||
collection_std_err, "w", encoding="utf-8"
|
||||
) as err:
|
||||
print([oav_exe, oav_command, target_folder])
|
||||
result = subprocess.run(
|
||||
[oav_exe, oav_command, target_folder], capture_output=True, check=True, text=True
|
||||
)
|
||||
|
||||
out.write(result.stdout)
|
||||
err.write(result.stderr)
|
||||
|
||||
return OAVScanResult(target_folder, collection_std_out, collection_std_err, result.returncode, oav_version)
|
||||
except subprocess.CalledProcessError as e:
|
||||
with open(collection_std_err, "a", encoding="utf-8") as err:
|
||||
err.write(str(e))
|
||||
|
||||
return OAVScanResult(target_folder, collection_std_out, collection_std_err, -1, oav_version)
|
||||
|
||||
|
||||
def is_word_present_in_file(file_path, word):
|
||||
try:
|
||||
with open(file_path, "rb") as file:
|
||||
first_bytes = file.read(20)
|
||||
return word in first_bytes
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
|
||||
def get_specification_files(target_folder: str, output_folder: str) -> List[str]:
|
||||
target = os.path.join(target_folder, "specification", "**", "*.json")
|
||||
jsons = glob.glob(target, recursive=True)
|
||||
search_word = b"swagger"
|
||||
specs = []
|
||||
num = len(jsons)
|
||||
|
||||
output_cache = os.path.join(output_folder, CACHE_FILE_NAME)
|
||||
|
||||
if os.path.exists(output_cache):
|
||||
with open(output_cache, "r", encoding="utf-8") as c:
|
||||
specs = c.readlines()
|
||||
return [spec.strip() for spec in specs]
|
||||
|
||||
print(f"Scanned directory, found {len(jsons)} json files.")
|
||||
|
||||
for index, json_file in enumerate(jsons):
|
||||
if is_word_present_in_file(json_file, search_word):
|
||||
specs.append(json_file)
|
||||
|
||||
print(f"Filtered to {len(specs)} swagger files.")
|
||||
with open(output_cache, "w", encoding="utf-8") as c:
|
||||
c.write("\n".join(specs))
|
||||
|
||||
return specs
|
||||
|
||||
|
||||
def verify_oav_version(oav: str) -> str:
|
||||
try:
|
||||
result = subprocess.run([oav, "--version"], capture_output=True, shell=True)
|
||||
return result.stdout.decode("utf-8").strip()
|
||||
except Exception as f:
|
||||
return "-1"
|
||||
|
||||
|
||||
def get_output_files(root_target_folder: str, choice: str, target_folder: str) -> Tuple[str, str]:
|
||||
"""Given the root of the azure-rest-api-specs repo AND a folder that is some deeper child of that,
|
||||
come up with the output file names"""
|
||||
|
||||
relpath = os.path.relpath(target_folder, root_target_folder)
|
||||
flattened_path = relpath.replace("\\", "_").replace("/", "_").replace(".json", "")
|
||||
|
||||
return (f"{flattened_path}_{choice}_out.log", f"{flattened_path}_{choice}_err.log")
|
||||
|
||||
|
||||
def prepare_output_folder(target_folder: str) -> str:
|
||||
must_repopulate_cache = False
|
||||
|
||||
if os.path.exists(target_folder):
|
||||
cache_file = os.path.join(target_folder, CACHE_FILE_NAME)
|
||||
if os.path.exists(cache_file):
|
||||
tmp_dir = tempfile.gettempdir()
|
||||
cache_location = os.path.join(tmp_dir, CACHE_FILE_NAME)
|
||||
shutil.move(cache_file, cache_location)
|
||||
must_repopulate_cache = True
|
||||
|
||||
shutil.rmtree(target_folder)
|
||||
|
||||
os.makedirs(target_folder)
|
||||
|
||||
if must_repopulate_cache:
|
||||
shutil.move(cache_location, cache_file)
|
||||
|
||||
return target_folder
|
||||
|
||||
|
||||
def dump_summary(summary: Dict[str, OAVScanResult]) -> None:
|
||||
print(f"Scanned {len(summary.keys())} files successfully.")
|
||||
|
||||
|
||||
def run(oav_exe: str, spec: str, output_folder: str, choice: str, oav_version: str):
|
||||
collection_stdout_file, collection_stderr_file = get_output_files(args.target, choice, spec)
|
||||
resolved_out = os.path.join(output_folder, collection_stdout_file)
|
||||
resolved_err = os.path.join(output_folder, collection_stderr_file)
|
||||
summary[f"{spec}-{choice}"] = get_oav_output(oav_exe, spec, resolved_out, resolved_err, choice, oav_version)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description="Scan azure-rest-api-specs repository, invoke oav")
|
||||
|
||||
parser.add_argument(
|
||||
"--target",
|
||||
dest="target",
|
||||
help="The azure-rest-api-specs repo root.",
|
||||
required=True,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
dest="output",
|
||||
help="The folder which will contain the oav output.",
|
||||
required=True,
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--type",
|
||||
dest="type",
|
||||
required=True,
|
||||
help="Are we running specs or examples?",
|
||||
choices=["validate-spec","validate-example"]
|
||||
)
|
||||
|
||||
parser.add_argument(
|
||||
"--oav",
|
||||
dest="oav",
|
||||
help="The oav exe this script will be using! If OAV is on the PATH just pass nothing!",
|
||||
required=False
|
||||
)
|
||||
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.oav:
|
||||
oav_exe = args.oav
|
||||
else:
|
||||
oav_exe = "oav"
|
||||
|
||||
oav_version = verify_oav_version(oav_exe)
|
||||
|
||||
if oav_version == "-1":
|
||||
print("OAV is not available on the PATH. Resolve this ane reinvoke.")
|
||||
sys.exit(1)
|
||||
|
||||
output_folder: str = prepare_output_folder(args.output)
|
||||
specs: List[str] = get_specification_files(args.target, output_folder)
|
||||
summary: Dict[str, OAVScanResult] = {}
|
||||
|
||||
for spec in specs:
|
||||
run(oav_exe, spec, output_folder, args.type, oav_version)
|
||||
|
||||
dump_summary(summary)
|
||||
Загрузка…
Ссылка в новой задаче