Convert the analyze-change.ps1 to typescript which allows to reuse a
common config for which area belong to who as well as some other
helpers.

The testing also is then all built-in the same system
This commit is contained in:
Timothee Guerin 2024-07-26 11:38:09 -07:00 коммит произвёл GitHub
Родитель eb245109c0
Коммит 868891845f
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
18 изменённых файлов: 296 добавлений и 306 удалений

56
eng/common/config/area.ts Normal file
Просмотреть файл

@ -0,0 +1,56 @@
import type { AreaLabels } from "./labels.js";
/**
* Set the paths that each area applies to.
*/
export const AreaPaths: Record<keyof typeof AreaLabels, string[]> = {
"compiler:core": ["packages/compiler/"],
"compiler:emitter-framework": [],
ide: ["packages/typespec-vscode/", "packages/typespec-vs/"],
"lib:http": ["packages/http/"],
"lib:openapi": ["packages/openapi/"],
"lib:rest": ["packages/rest/"],
"lib:versioning": ["packages/versioning/"],
"meta:blog": ["blog/"],
"meta:website": ["website/"],
tspd: ["packages/tspd/"],
"emitter:client:csharp": ["packages/http-client-csharp/"],
"emitter:client:java": ["packages/http-client-java/"],
"emitter:json-schema": ["packages/json-schema/"],
"emitter:protobuf": ["packages/protobuf/"],
"emitter:openapi3": ["packages/openapi3/"],
"openapi3:converter": ["packages/openapi3/src/cli/actions/convert/"],
"emitter:service:csharp": [],
"emitter:service:js": [],
eng: ["eng/", ".github/"],
"ui:playground": ["packages/playground/"],
"ui:type-graph-viewer": ["packages/html-program-viewer/"],
};
/**
* Path that should trigger every CI build.
*/
const all = ["eng/common/", "vitest.config.ts"];
/**
* Path that should trigger all isolated emitter builds
*/
const isolatedEmitters = ["eng/emitters/"];
export const CIRules = {
CSharp: [...all, ...isolatedEmitters, ...AreaPaths["emitter:client:csharp"], ".editorconfig"],
Core: [
"**/*",
"!.prettierignore", // Prettier is already run as its dedicated CI(via github action)
"!.prettierrc.json",
"!cspell.yaml", // CSpell is already run as its dedicated CI(via github action)
"!esling.config.json", // Eslint is already run as its dedicated CI(via github action)
...ignore(isolatedEmitters),
...ignore(AreaPaths["emitter:client:csharp"]),
],
};
function ignore(paths: string[]) {
return paths.map((x) => `!${x}`);
}

Просмотреть файл

@ -1,6 +1,7 @@
// cspell:ignore bfff
import { repo } from "../scripts/common.js";
import { defineConfig, defineLabels } from "../scripts/labels/config.js";
import { repo } from "../scripts/utils/common.js";
import { AreaPaths } from "./area.js";
/**
* Labels that are used to categorize issue for which area they belong to.
@ -166,33 +167,6 @@ export const CommonLabels = {
},
};
/**
* Set the paths that each area applies to.
*/
export const AreaPaths: Record<keyof typeof AreaLabels, string[]> = {
"compiler:core": ["packages/compiler/"],
"compiler:emitter-framework": [],
ide: ["packages/typespec-vscode/", "packages/typespec-vs/"],
"lib:http": ["packages/http/"],
"lib:openapi": ["packages/openapi/"],
"lib:rest": ["packages/rest/"],
"lib:versioning": ["packages/versioning/"],
"meta:blog": ["blog/"],
"meta:website": ["website/"],
tspd: ["packages/tspd/"],
"emitter:client:csharp": ["packages/http-client-csharp/"],
"emitter:client:java": ["packages/http-client-java/"],
"emitter:json-schema": ["packages/json-schema/"],
"emitter:protobuf": ["packages/protobuf/"],
"emitter:openapi3": ["packages/openapi3/"],
"openapi3:converter": ["packages/openapi3/src/cli/actions/convert/"],
"emitter:service:csharp": [],
"emitter:service:js": [],
eng: ["eng/", ".github/"],
"ui:playground": ["packages/playground/"],
"ui:type-graph-viewer": ["packages/html-program-viewer/"],
};
export default defineConfig({
repo,
labels: {

Просмотреть файл

@ -22,18 +22,20 @@ extends:
- job: InitJob
displayName: Initialize
steps:
- script: |
corepack enable
corepack prepare pnpm --activate
displayName: Install pnpm
- script: pnpm install
displayName: Install JavaScript Dependencies
- script: node $(Build.SourcesDirectory)/eng/common/scripts/resolve-target-branch.js
displayName: Resolve target branch
- task: PowerShell@2
- script: pnpm tsx ./eng/common/scripts/dispatch-area-triggers.ts --target-branch $(TARGET_BRANCH)
displayName: "Analyze PR changes"
name: InitStep
inputs:
pwsh: true
filePath: $(Build.SourcesDirectory)/eng/common/scripts/Analyze-Changes.ps1
arguments: >
-TargetBranch $(TARGET_BRANCH)
workingDirectory: $(Build.SourcesDirectory)
# Run csharp stages if RunCSharp == true
- template: /packages/http-client-csharp/eng/pipeline/templates/ci-stages.yml

Просмотреть файл

@ -1,76 +0,0 @@
BeforeAll {
. $PSScriptRoot/Analyze-Changes.ps1 *>&1 | Out-Null
}
Describe 'Analyze-Changes' {
AfterEach {
foreach($package in $isolatedPackages.Values) {
$package.RunValue = $false;
}
}
It 'Should return package variables if package specific changes are detected' {
$actual = Get-ActiveVariables @(
"packages/http-client-csharp/src/constants.ts"
)
$expected = @('RunCSharp')
$actual | ForEach-Object {
$_ | Should -BeIn $expected
}
}
It 'Should return RunCore if common files are changed' {
$actual = Get-ActiveVariables @(
"packages/compiler/package.json"
)
$expected = @('RunCore')
$actual | ForEach-Object {
$_ | Should -BeIn $expected
}
}
It 'Should return a combination of core and isolated packages' {
$actual = Get-ActiveVariables @(
"packages/http-client-csharp/src/constants.ts",
"packages/compiler/package.json"
)
$expected = @('RunCore', 'RunCSharp')
$actual | ForEach-Object {
$_ | Should -BeIn $expected
}
}
It 'Should return RunCSharp and RunCore if .editorconfig is changed' {
$actual = Get-ActiveVariables @(
".editorconfig"
)
$expected = @('RunCore', 'RunCSharp')
$actual | ForEach-Object {
$_ | Should -BeIn $expected
}
}
It 'Should not return runCore for .prettierignore, .prettierrc.json, cspell.yaml, esling.config.json' {
$actual = Get-ActiveVariables @(
".prettierignore",
".prettierrc.json",
"cspell.yaml",
"esling.config.json"
"packages/http-client-csharp/emitter/src/constants.ts"
)
$expected = @('RunCore', 'RunCSharp')
$actual | ForEach-Object {
$_ | Should -BeIn $expected
}
}
}

Просмотреть файл

@ -1,181 +0,0 @@
#Requires -Version 7.0
param(
[string] $TargetBranch
)
# Represents an isolated package which has its own stages in typespec - ci pipeline
class IsolatedPackage {
[string[]] $Paths
[string] $RunVariable
[bool] $RunValue
IsolatedPackage([string[]]$paths, [string]$runVariable, [bool]$runValue) {
$this.Paths = $paths
$this.RunVariable = $runVariable
$this.RunValue = $runValue
}
}
# Emitter packages in the repo
$isolatedPackages = @{
"http-client-csharp" = [IsolatedPackage]::new(@("packages/http-client-csharp", ".editorconfig"), "RunCSharp", $false)
"http-client-java" = [IsolatedPackage]::new(@("packages/http-client-java"), "RunJava", $false)
"http-client-typescript" = [IsolatedPackage]::new(@("packages/http-client-typescript"), "RunTypeScript", $false)
"http-client-python" = [IsolatedPackage]::new(@("packages/http-client-python"), "RunPython", $false)
}
# A tree representation of a set of files
# Each node represents a directory and contains a list of child nodes.
class TreeNode {
[string] $Name
[System.Collections.Generic.List[TreeNode]] $Children
TreeNode([string]$name) {
$this.Name = $name
$this.Children = @()
}
# Add a file to the tree
[void] Add([string]$filePath) {
$parts = $filePath -split '/'
$currentNode = $this
foreach ($part in $parts) {
$childNode = $currentNode.Children | Where-Object { $_.Name -eq $part }
if (-not $childNode) {
$childNode = [TreeNode]::new($part)
$currentNode.Children.Add($childNode)
}
$currentNode = $childNode
}
}
# Check if a file exists in the tree
[bool] PathExists([string]$filePath) {
$parts = $filePath -split '/'
$currentNode = $this
foreach ($part in $parts) {
$childNode = $currentNode.Children | Where-Object { $_.Name -eq $part }
if (-not $childNode) {
return $false
}
$currentNode = $childNode
}
return $true
}
# Check if anything outside of emitter packages exists
[bool] AnythingOutsideIsolatedPackagesExists($isolatedPackages) {
if ($this.Children.Count -eq 0) {
return $false
}
# if anything in first level is not 'packages', return true
foreach ($child in $this.Children) {
# skip .prettierignore, .prettierrc.json, cspell.yaml, esling.config.json since these are all covered by github actions globally
if ($child.Name -in @('.prettierignore', '.prettierrc.json', 'cspell.yaml', 'esling.config.json')) {
continue
}
if ($child.Name -ne 'packages') {
return $true
}
}
$packagesNode = $this.Children | Where-Object { $_.Name -eq "packages" }
if (-not $packagesNode) {
return $false
}
# if anything in second level is not an emitter package, return true
foreach ($child in $packagesNode.Children) {
if ($child.Name -notin $isolatedPackages.Keys) {
return $true
}
}
return $false
}
[string] ToString() {
return $this.Name
}
}
function Get-ActiveVariables($changes) {
# initialize tree
$root = [TreeNode]::new('Root')
$variables = @()
$changes | ForEach-Object {
$root.Add($_)
}
# exit early if no changes detected
if ($root.Children.Count -eq 0) {
Write-Host "##[error] No changes detected"
exit 1
}
# set global flag to run all if common files are changed
$runAll = $root.PathExists('eng/common') -or $root.PathExists('vitest.config.ts')
# set global isolated package flag to run if any eng/emiters files changed
$runIsolated = $root.PathExists('eng/emitters')
# no need to check individual packages if runAll is true
if (-not $runAll) {
if (-not $runIsolated) {
# set each isolated package flag
foreach ($package in $isolatedPackages.Values) {
foreach ($path in $package.Paths) {
$package.RunValue = $package.RunValue -or $root.PathExists($path)
if ($package.RunValue) {
break
}
}
}
}
# set runCore to true if none of the
$runCore = $root.AnythingOutsideIsolatedPackagesExists($isolatedPackages)
}
# set log commands
if ($runAll -or $runCore) {
$variables += "RunCore"
}
# foreach isolated package, set log commands if the RunValue is true
foreach ($package in $isolatedPackages.Values) {
if ($runAll -or $runIsolated -or $package.RunValue) {
$variables += $package.RunVariable
}
}
return $variables
}
# add all changed files to the tree
Write-Host "Checking for changes in current branch compared to $TargetBranch"
$changes = git diff --name-only origin/$TargetBranch...
Write-Host "##[group]Files changed in this pr"
$changes | ForEach-Object {
Write-Host " - $_"
}
Write-Host "##[endgroup]"
if ($LASTEXITCODE -ne 0) {
Write-Host "##[error] 'git diff --name-only origin/$TargetBranch...' failed, exiting..."
exit 1 # Exit with a non-zero exit code to indicate failure
}
$variables = Get-ActiveVariables $changes
foreach ($variable in $variables) {
Write-Host "Setting $variable to true"
Write-Host "##vso[task.setvariable variable=$variable;isOutput=true]true"
}

Просмотреть файл

@ -0,0 +1,33 @@
import { parseArgs } from "util";
import { setOutputVariable } from "./utils/ado.js";
import { repoRoot } from "./utils/common.js";
import { findAreasChanged } from "./utils/find-area-changed.js";
import { listChangedFilesSince } from "./utils/git.js";
const args = parseArgs({
args: process.argv.slice(2),
options: {
"target-branch": { type: "string" },
},
});
const targetBranch = args.values["target-branch"];
if (!targetBranch) {
console.error("--target-branch is required");
process.exit(1);
}
console.log("Checking for changes in current branch compared to $TargetBranch");
const files = await listChangedFilesSince(`origin/${targetBranch}`, { repositoryPath: repoRoot });
console.log("##[group]Files changed in this pr");
console.log(files.map((x) => ` - ${x}`).join("\n"));
console.log("##[endgroup]");
const areaChanged = findAreasChanged(files);
for (const area of areaChanged) {
console.log(`Setting output variable Run${area} to true`);
setOutputVariable(`Run${area}`, "true");
}

Просмотреть файл

@ -1,6 +1,6 @@
import { resolve } from "path";
import { stringify } from "yaml";
import { CheckOptions, syncFile } from "../common.js";
import { CheckOptions, syncFile } from "../utils/common.js";
import {
PolicyServiceConfig,
and,

Просмотреть файл

@ -0,0 +1,3 @@
export function setOutputVariable(name: string, value: string) {
process.stdout.write(`##vso[task.setvariable variable=${name};isOutput=true]${value}\n`);
}

Просмотреть файл

@ -2,12 +2,13 @@ import { readFile, writeFile } from "fs/promises";
import { dirname, resolve } from "path";
import pc from "picocolors";
import { fileURLToPath } from "url";
export const repo = {
owner: "microsoft",
repo: "typespec",
};
export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../..");
export const repoRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../../..");
export interface CheckOptions {
readonly check?: boolean;

Просмотреть файл

@ -0,0 +1,42 @@
import { spawn, type SpawnOptions } from "child_process";
export interface ExecResult {
readonly code: number | null;
readonly stdall: Buffer;
readonly stdout: Buffer;
readonly stderr: Buffer;
}
export function execAsync(
cmd: string,
args: string[],
opts: SpawnOptions = {}
): Promise<ExecResult> {
return new Promise((resolve, reject) => {
const child = spawn(cmd, args, opts);
let stdall = Buffer.from("");
let stdout = Buffer.from("");
let stderr = Buffer.from("");
if (child.stdout) {
child.stdout.on("data", (data) => {
stdout = Buffer.concat([stdout, data]);
stdall = Buffer.concat([stdall, data]);
});
}
if (child.stderr) {
child.stderr.on("data", (data) => {
stderr = Buffer.concat([stderr, data]);
stdall = Buffer.concat([stdall, data]);
});
}
child.on("error", (err) => {
reject(err);
});
child.on("close", (code) => {
resolve({ code, stdout, stderr, stdall });
});
});
}

Просмотреть файл

@ -0,0 +1,64 @@
import { describe, expect, it } from "vitest";
import { findAreasChanged } from "./find-area-changed.js";
describe("paths that should trigger CSharp CI", () => {
it.each([
["packages/http-client-csharp/src/constants.ts"],
[
"eng/emitters/pipelines/templates/jobs/test-job.yml",
"packages/http-client-csharp/eng/scripts/Test-CadlRanch.ps1",
"packages/http-client-csharp/generator/TestProjects/CadlRanch.Tests/Infrastructure/AssemblyCleanFixture.cs",
"packages/http-client-csharp/generator/TestProjects/CadlRanch.Tests/Infrastructure/CadlRanchServer.cs",
],
])("%s", (...paths) => {
const areas = findAreasChanged(paths);
expect(areas).toEqual(["CSharp"]);
});
});
describe("paths that should trigger Core CI", () => {
it.each([
"packages/compiler/package.json",
"packages/http/package.json",
"packages/openapi3/package.json",
])("%s", (path) => {
const areas = findAreasChanged([path]);
expect(areas).toEqual(["Core"]);
});
});
describe("paths that should trigger all isolated packages", () => {
it.each(["eng/emitters/pipelines/templates/jobs/detect-api-changes.yml"])("%s", (path) => {
const areas = findAreasChanged([path]);
expect(areas).toEqual(["CSharp"]);
});
});
it("Should return a combination of core and isolated packages", () => {
const areas = findAreasChanged([
"packages/http-client-csharp/src/constants.ts",
"packages/compiler/package.json",
]);
expect(areas).toEqual(["CSharp", "Core"]);
});
it("Should return CSharp and Core if .editorconfig is changed", () => {
const areas = findAreasChanged([".editorconfig"]);
expect(areas).toEqual(["CSharp", "Core"]);
});
it("Should not return Core for .prettierignore, .prettierrc.json, cspell.yaml, esling.config.json", () => {
const areas = findAreasChanged([
".prettierignore",
".prettierrc.json",
"cspell.yaml",
"esling.config.json",
"packages/http-client-csharp/emitter/src/constants.ts",
]);
expect(areas).toEqual(["CSharp"]);
});
it("should return Core for random files at the root", () => {
const areas = findAreasChanged(["some.file", "file/in/deep/directory"]);
expect(areas).toEqual(["Core"]);
});

Просмотреть файл

@ -0,0 +1,28 @@
import micromatch from "micromatch";
import pc from "picocolors";
import { CIRules } from "../../config/area.js";
export function findAreasChanged(files: string[]): (keyof typeof CIRules)[] {
const result: (keyof typeof CIRules)[] = [];
for (const [name, patterns] of Object.entries(CIRules)) {
const expandedPatterns = patterns.map(expandFolder);
console.log(`Checking trigger ${name}, with patterns:`, expandedPatterns);
const match = micromatch(files, expandedPatterns, { dot: true });
if (match.length > 0) {
result.push(name as any);
console.log(`Changes matched for trigger ${pc.cyan(name)}`, files);
} else {
console.log(`No changes matched for trigger ${pc.cyan(name)}`);
}
}
return result;
}
function expandFolder(maybeFolder: string) {
if (maybeFolder.endsWith("/")) {
return `${maybeFolder}**/*`;
}
return maybeFolder;
}

Просмотреть файл

@ -0,0 +1,36 @@
import { execAsync, type ExecResult } from "./exec-async.js";
export async function listChangedFilesSince(
ref: string,
{ repositoryPath }: { repositoryPath: string }
) {
return splitStdoutLines(await execGit(["diff", "--name-only", `${ref}...`], { repositoryPath }));
}
async function execGit(
args: string[],
{ repositoryPath }: { repositoryPath: string }
): Promise<ExecResult> {
const result = await execAsync("git", args, { cwd: repositoryPath });
if (result.code !== 0) {
throw new GitError(args, result.stderr.toString());
}
return result;
}
export class GitError extends Error {
args: string[];
constructor(args: string[], stderr: string) {
super(`GitError running: git ${args.join(" ")}\n${stderr}`);
this.args = args;
}
}
function splitStdoutLines(result: ExecResult): string[] {
return result.stdout
.toString()
.split("\n")
.filter((a) => a);
}

4
eng/vitest.config.ts Normal file
Просмотреть файл

@ -0,0 +1,4 @@
import { defineConfig, mergeConfig } from "vitest/config";
import { defaultTypeSpecVitestConfig } from "../vitest.workspace.js";
export default mergeConfig(defaultTypeSpecVitestConfig, defineConfig({}));

Просмотреть файл

@ -43,6 +43,7 @@
"@octokit/plugin-paginate-graphql": "^5.2.2",
"@octokit/plugin-rest-endpoint-methods": "^13.2.4",
"@pnpm/find-workspace-packages": "^6.0.9",
"@types/micromatch": "^4.0.9",
"@types/node": "~18.11.19",
"@typescript-eslint/parser": "^7.17.0",
"@typescript-eslint/utils": "^7.17.0",
@ -55,6 +56,7 @@
"eslint-plugin-react-hooks": "^4.6.2",
"eslint-plugin-unicorn": "^54.0.0",
"eslint-plugin-vitest": "^0.5.4",
"micromatch": "^4.0.7",
"picocolors": "~1.0.1",
"prettier": "~3.3.3",
"prettier-plugin-organize-imports": "~4.0.0",

16
pnpm-lock.yaml сгенерированный
Просмотреть файл

@ -32,6 +32,9 @@ importers:
'@pnpm/find-workspace-packages':
specifier: ^6.0.9
version: 6.0.9(@pnpm/logger@5.0.0)
'@types/micromatch':
specifier: ^4.0.9
version: 4.0.9
'@types/node':
specifier: ~18.11.19
version: 18.11.19
@ -68,6 +71,9 @@ importers:
eslint-plugin-vitest:
specifier: ^0.5.4
version: 0.5.4(eslint@8.57.0)(typescript@5.5.4)(vitest@2.0.4(@types/node@18.11.19)(@vitest/ui@2.0.4)(happy-dom@14.12.3)(jsdom@19.0.0)(terser@5.30.0))
micromatch:
specifier: ^4.0.7
version: 4.0.7
picocolors:
specifier: ~1.0.1
version: 1.0.1
@ -7050,10 +7056,6 @@ packages:
resolution: {integrity: sha512-pjmC+bkIF8XI7fWaH8KxHcZL3DPybs1roSKP4rKDvy20tAWwIObE4+JIseG2byfGKhud5ZnM4YSGKBz7Sh0ndQ==}
engines: {node: '>= 0.4.0'}
fill-range@7.0.1:
resolution: {integrity: sha512-qOo9F+dMUmC2Lcb4BbVvnKJxTPjCm+RRpe4gDuGrzkL7mEVl/djYSu2OdQ2Pa302N4oqkSg9ir6jaLWJ2USVpQ==}
engines: {node: '>=8'}
fill-range@7.1.1:
resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==}
engines: {node: '>=8'}
@ -17337,7 +17339,7 @@ snapshots:
braces@3.0.2:
dependencies:
fill-range: 7.0.1
fill-range: 7.1.1
braces@3.0.3:
dependencies:
@ -19127,10 +19129,6 @@ snapshots:
filesize@8.0.7: {}
fill-range@7.0.1:
dependencies:
to-regex-range: 5.0.1
fill-range@7.1.1:
dependencies:
to-regex-range: 5.0.1

Просмотреть файл

@ -1,4 +1,5 @@
{
"extends": "./tsconfig.base.json",
"include": ["eng"]
"include": ["eng"],
"exclude": ["eng/vitest.config.ts"]
}

Просмотреть файл

@ -1,7 +1,10 @@
import { defineConfig } from "vitest/config";
export default ["packages/*/vitest.config.ts", "packages/*/vitest.config.mts"];
export default [
"packages/*/vitest.config.ts",
"packages/*/vitest.config.mts",
"eng/vitest.config.ts",
];
/**
* Default Config For all TypeSpec projects using vitest.
*/