diff --git a/.github/workflows/generate-single.yml b/.github/workflows/generate-single.yml index 91923e563..86ee938e2 100644 --- a/.github/workflows/generate-single.yml +++ b/.github/workflows/generate-single.yml @@ -8,7 +8,7 @@ on: required: true default: 'main' single_path: - description: 'The path to generate types for (e.g. "compute", or "keyvault").' + description: 'The path to generate types for (e.g. "compute/resource-manager", or "keyvault/resource-manager").' required: true jobs: @@ -40,7 +40,7 @@ jobs: run: | npm run generate-single -- \ --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \ - --base-path '${{ github.event.inputs.single_path }}/resource-manager' + --base-path '${{ github.event.inputs.single_path }}' working-directory: generator - name: Create Pull Request diff --git a/.vscode/launch.json b/.vscode/launch.json index 37fef624a..f09e08496 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -26,7 +26,7 @@ "args": [ "${workspaceFolder}/generator/cmd/generatesingle.ts", "--base-path", - "${input:resourceProvider}/resource-manager" + "${input:resourceProvider}" ] }, { @@ -53,10 +53,10 @@ ], "inputs": [ { - "id": "resourceProvider", + "id": "basePath", "type": "promptString", - "default": "alertsmanagement", - "description": "The ResourceProvider name, e.g., compute, network" + "default": "alertsmanagement/resource-manager", + "description": "The path to generate types for (e.g. \"compute/resource-manager\", or \"keyvault/resource-manager\")." } ] } diff --git a/generator/autogenlist.ts b/generator/autogenlist.ts index 36f87e75c..ef2420238 100644 --- a/generator/autogenlist.ts +++ b/generator/autogenlist.ts @@ -17,6 +17,7 @@ import { postProcessor as azureStackHciPostProcessor } from './processors/Micros import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources'; import { postProcessor as serviceFabricPostProcessor } from './processors/Microsoft.ServiceFabric'; import { lowerCaseEquals } from './utils'; +import { detectProviderNamespaces } from './generate'; // New providers are onboarded by default. The providers listed here are the only ones **not** onboarded. const disabledProviders: AutoGenConfig[] = [ @@ -110,10 +111,16 @@ const disabledProviders: AutoGenConfig[] = [ disabledForAutogen: true }, { - // Disabled temporally due to unsupported directory structure - basePath: 'containerservice/resource-manager', + basePath: 'containerservice/resource-manager/Microsoft.ContainerService/aks', namespace: 'Microsoft.ContainerService', - disabledForAutogen: true, + useNamespaceFromConfig: true, + suffix: 'Aks' + }, + { + basePath: 'containerservice/resource-manager/Microsoft.ContainerService/fleet', + namespace: 'Microsoft.ContainerService', + useNamespaceFromConfig: true, + suffix: 'Fleet' }, ]; @@ -1160,10 +1167,16 @@ export function findAutogenEntries(basePath: string): AutoGenConfig[] { return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath)); } -export function findOrGenerateAutogenEntries(basePath: string, namespaces: string[]): AutoGenConfig[] { - const entries = findAutogenEntries(basePath).filter(e => namespaces.some(ns => lowerCaseEquals(e.namespace, ns))); +export async function findOrGenerateAutogenEntries(basePath: string, readme: string): Promise { + let entries = findAutogenEntries(basePath); + if (entries.some(e => e.useNamespaceFromConfig)) { + return entries; + } - for (const namespace of namespaces) { + const detectedNamespaces = await detectProviderNamespaces(readme); + entries = entries.filter(e => detectedNamespaces.some(ns => lowerCaseEquals(e.namespace, ns))); + + for (const namespace of detectedNamespaces) { if (!entries.some(e => lowerCaseEquals(e.namespace, namespace))) { // Generate configuration for any RPs not explicitly declared in the autogen list entries.push({ diff --git a/generator/cmd/generateall.ts b/generator/cmd/generateall.ts index 4de94556e..b4ba76022 100644 --- a/generator/cmd/generateall.ts +++ b/generator/cmd/generateall.ts @@ -2,10 +2,10 @@ // Licensed under the MIT License. import * as constants from '../constants'; import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs'; -import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate'; +import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs } from '../generate'; import { findOrGenerateAutogenEntries } from '../autogenlist'; import colors from 'colors'; -import { flatten, keys } from 'lodash'; +import { flatten } from 'lodash'; import { executeSynchronous, chunker, writeJsonFile } from '../utils'; import { Package } from '../models'; import yargs from 'yargs'; @@ -72,8 +72,7 @@ executeSynchronous(async () => { for (const basePath of basePaths) { try { const readme = validateAndReturnReadmePath(localPath, basePath); - const namespaces = keys(await getApiVersionsByNamespace(readme)); - let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces) + let filteredAutoGenList = (await findOrGenerateAutogenEntries(basePath, readme)) .filter(x => x.disabledForAutogen !== true); if (args['readme-files']) { diff --git a/generator/cmd/generateonboardedreport.ts b/generator/cmd/generateonboardedreport.ts index f5cdf53c7..80b9fe33d 100644 --- a/generator/cmd/generateonboardedreport.ts +++ b/generator/cmd/generateonboardedreport.ts @@ -4,8 +4,7 @@ import * as constants from '../constants'; import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs'; import { findOrGenerateAutogenEntries } from '../autogenlist'; import { executeSynchronous, writeJsonFile, safeMkdir } from '../utils'; -import { getApiVersionsByNamespace } from '../generate'; -import { keys, partition } from 'lodash'; +import { partition } from 'lodash'; import path from 'path'; executeSynchronous(async () => { @@ -15,8 +14,7 @@ executeSynchronous(async () => { for (const basePath of basePaths) { const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath); - const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces); + const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme); const [unautogened, autogened] = partition( autogenlistEntries, diff --git a/generator/cmd/generatesingle.ts b/generator/cmd/generatesingle.ts index cd4dae9a9..498cd3412 100644 --- a/generator/cmd/generatesingle.ts +++ b/generator/cmd/generatesingle.ts @@ -2,10 +2,9 @@ // Licensed under the MIT License. import * as constants from '../constants'; import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs'; -import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate'; +import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate'; import { findOrGenerateAutogenEntries } from '../autogenlist'; import colors from 'colors'; -import { keys } from 'lodash'; import { executeSynchronous } from '../utils'; import yargs from 'yargs'; @@ -34,8 +33,7 @@ executeSynchronous(async () => { } const schemaConfigs = []; - const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autoGenEntries = findOrGenerateAutogenEntries(basePath, namespaces); + const autoGenEntries = await findOrGenerateAutogenEntries(basePath, readme); for (const autoGenConfig of autoGenEntries) { if (autoGenConfig.disabledForAutogen === true) { diff --git a/generator/cmd/listbasepaths.ts b/generator/cmd/listbasepaths.ts index 5049acff7..027222611 100644 --- a/generator/cmd/listbasepaths.ts +++ b/generator/cmd/listbasepaths.ts @@ -5,16 +5,14 @@ import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs import colors from 'colors'; import { findOrGenerateAutogenEntries } from '../autogenlist'; import { executeSynchronous } from '../utils'; -import { getApiVersionsByNamespace } from '../generate'; -import { keys, partition } from 'lodash'; +import { partition } from 'lodash'; executeSynchronous(async () => { const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash); for (const basePath of basePaths) { const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath); - const namespaces = keys(await getApiVersionsByNamespace(readme)); - const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces); + const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme); const [unautogened, autogened] = partition( autogenlistEntries, diff --git a/generator/constants.ts b/generator/constants.ts index dddb8f29c..89733e9d9 100644 --- a/generator/constants.ts +++ b/generator/constants.ts @@ -8,8 +8,6 @@ export const generatorRoot = path.resolve(__dirname, '../'); export const specsRepoPath = path.join(os.tmpdir(), 'schm_azspc'); export const specsRepoUri = 'https://github.com/azure/azure-rest-api-specs'; export const specsRepoCommitHash = 'origin/main'; -// eslint-disable-next-line no-useless-escape -export const pathRegex = /(microsoft\.\w+|NGINX.NGINXPLUS|DYNATRACE.OBSERVABILITY)[\\\/]\S*[\\\/](\d{4}-\d{2}-\d{2}(|-preview))[\\\/]/i; export const autoRestVerboseOutput = false; diff --git a/generator/generate.ts b/generator/generate.ts index 830d5c644..38f94ecbf 100644 --- a/generator/generate.ts +++ b/generator/generate.ts @@ -2,15 +2,15 @@ // Licensed under the MIT License. import path from 'path'; import os from 'os'; -import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals, lowerCaseContains } from './utils'; +import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals } from './utils'; import * as constants from './constants'; import { prepareReadme } from './specs'; import colors from 'colors'; import { ScopeType, AutoGenConfig } from './models'; -import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy } from 'lodash'; +import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference } from 'lodash'; const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest'; -const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/; +export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/; export interface SchemaConfiguration { references: SchemaReference[]; @@ -35,27 +35,17 @@ const RootSchemaConfigs: Map = new Map([ [ScopeType.ManagementGroup, constants.managementGroupRootSchema] ]); -export async function getApiVersionsByNamespace(readme: string): Promise> { +export async function detectProviderNamespaces(readme: string) { const searchPath = path.resolve(`${readme}/..`); + + // To try and detect possible provider namespaces, assume a folder structure of /preview|stable//..., based on convention const apiVersionPaths = await findDirRecursive(searchPath, p => path.basename(p).match(apiVersionRegex) !== null); - - const output: Dictionary = {}; - for (const [namespace, , apiVersion] of apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep))) { - output[namespace] = [...(output[namespace] ?? []), apiVersion]; - } - - return output; + return uniq(apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep)[0])); } -export async function generateSchemas(readme: string, autoGenConfig?: AutoGenConfig): Promise { +export async function generateSchemas(readme: string, autoGenConfig: AutoGenConfig): Promise { await prepareReadme(readme, autoGenConfig); - const apiVersionsByNamespace = pickBy( - await getApiVersionsByNamespace(readme), - (_, key) => !autoGenConfig || lowerCaseEquals(key, autoGenConfig.namespace)); - - const namespaces = keys(apiVersionsByNamespace); - const schemaConfigs: SchemaConfiguration[] = []; const tmpFolder = path.join(os.tmpdir(), Math.random().toString(36).substr(2)); @@ -64,7 +54,7 @@ export async function generateSchemas(readme: string, autoGenConfig?: AutoGenCon for (const schemaPath of generatedSchemas) { const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath)))); - if (!lowerCaseContains(namespaces, namespace)) { + if (!lowerCaseEquals(autoGenConfig.namespace, namespace)) { continue; } diff --git a/generator/models.ts b/generator/models.ts index b75a03ee9..ef57931da 100644 --- a/generator/models.ts +++ b/generator/models.ts @@ -14,6 +14,7 @@ export interface AutoGenConfig { disabledForAutogen?: true, basePath: string, namespace: string, + useNamespaceFromConfig?: boolean, readmeFile?: string, readmeTag?: ReadmeTag, suffix?: string, diff --git a/generator/specs.ts b/generator/specs.ts index 2b219bd7c..124f95094 100644 --- a/generator/specs.ts +++ b/generator/specs.ts @@ -2,13 +2,14 @@ // Licensed under the MIT License. import path from 'path'; import { cloneGitRepo } from './git'; -import { findRecursive, lowerCaseEquals } from './utils'; +import { findRecursive, lowerCaseContains } from './utils'; import { ReadmeTag, AutoGenConfig, CodeBlock } from './models'; import * as constants from './constants' import * as cm from '@ts-common/commonmark-to-markdown' import * as yaml from 'js-yaml' import { existsSync } from 'fs'; import { readFile, writeFile } from 'fs/promises'; +import { apiVersionRegex } from './generate'; export async function resolveAbsolutePath(localPath: string) { if (path.isAbsolute(localPath)) { @@ -76,7 +77,7 @@ function isExcludedBasePath(basePath: string) { .some(prefix => basePath.toLowerCase().startsWith(prefix)); } -export async function prepareReadme(readme: string, autoGenConfig?: AutoGenConfig) { +export async function prepareReadme(readme: string, autoGenConfig: AutoGenConfig) { const content = (await readFile(readme)).toString(); const markdownEx = cm.parse(content); const fileSet = new Set(); @@ -101,21 +102,24 @@ export async function prepareReadme(readme: string, autoGenConfig?: AutoGenConfi } let readmeTag = {} as ReadmeTag; - fileSet.forEach(inputFile => { - const match = constants.pathRegex.exec(inputFile); - if (match) { - const mNamespace = match[1]; - const mApiVersion = match[2]; - if (!autoGenConfig || lowerCaseEquals(mNamespace, autoGenConfig.namespace)) { - if (!readmeTag[mApiVersion]) { - readmeTag[mApiVersion] = []; - } - readmeTag[mApiVersion].push(inputFile); - } - } - }); + for (const inputFile of fileSet) { + const pathComponents = inputFile.split(path.sep); - if (autoGenConfig?.readmeTag) { + if (!autoGenConfig.useNamespaceFromConfig && + !lowerCaseContains(pathComponents, autoGenConfig.namespace)) { + continue; + } + + const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0]; + if (!apiVersion) { + continue; + } + + readmeTag[apiVersion] ??= readmeTag[apiVersion] || []; + readmeTag[apiVersion].push(inputFile); + } + + if (autoGenConfig.readmeTag) { readmeTag = {...readmeTag, ...autoGenConfig.readmeTag }; }