Fix AKS autogeneration

.github/workflows/generate-single.yml

@@ -8,7 +8,7 @@ on:
         required: true
         default: 'main'
       single_path:
-        description: 'The path to generate types for (e.g. "compute", or "keyvault").'
+        description: 'The path to generate types for (e.g. "compute/resource-manager", or "keyvault/resource-manager").'
         required: true
 
 jobs:
@@ -40,7 +40,7 @@ jobs:
         run: |
           npm run generate-single -- \
             --local-path "$GITHUB_WORKSPACE/workflow-temp/azure-rest-api-specs" \
-            --base-path '${{ github.event.inputs.single_path }}/resource-manager'
+            --base-path '${{ github.event.inputs.single_path }}'
         working-directory: generator
 
       - name: Create Pull Request

.vscode/launch.json

@@ -26,7 +26,7 @@
       "args": [
         "${workspaceFolder}/generator/cmd/generatesingle.ts",
         "--base-path",
-        "${input:resourceProvider}/resource-manager"
+        "${input:resourceProvider}"
       ]
     },
     {
@@ -53,10 +53,10 @@
   ],
   "inputs": [
     {
-      "id": "resourceProvider",
+      "id": "basePath",
       "type": "promptString",
-      "default": "alertsmanagement",
-      "description": "The ResourceProvider name, e.g., compute, network"
+      "default": "alertsmanagement/resource-manager",
+      "description": "The path to generate types for (e.g. \"compute/resource-manager\", or \"keyvault/resource-manager\")."
     }
   ]
 }

generator/autogenlist.ts

@@ -17,6 +17,7 @@ import { postProcessor as azureStackHciPostProcessor } from './processors/Micros
 import { postProcessor as resourcesPostProcessor } from './processors/Microsoft.Resources';
 import { postProcessor as serviceFabricPostProcessor } from './processors/Microsoft.ServiceFabric';
 import { lowerCaseEquals } from './utils';
+import { detectProviderNamespaces } from './generate';
 
 // New providers are onboarded by default. The providers listed here are the only ones **not** onboarded.
 const disabledProviders: AutoGenConfig[] = [
@@ -110,10 +111,16 @@ const disabledProviders: AutoGenConfig[] = [
         disabledForAutogen: true
     },
     {
-        // Disabled temporally due to unsupported directory structure
-        basePath: 'containerservice/resource-manager',
+        basePath: 'containerservice/resource-manager/Microsoft.ContainerService/aks',
         namespace: 'Microsoft.ContainerService',
-        disabledForAutogen: true,
+        useNamespaceFromConfig: true,
+        suffix: 'Aks'
+    },
+    {
+        basePath: 'containerservice/resource-manager/Microsoft.ContainerService/fleet',
+        namespace: 'Microsoft.ContainerService',
+        useNamespaceFromConfig: true,
+        suffix: 'Fleet'
     },
 ];
 
@@ -1160,10 +1167,16 @@ export function findAutogenEntries(basePath: string): AutoGenConfig[] {
     return autoGenList.filter(w => lowerCaseEquals(w.basePath, basePath));
 }
 
-export function findOrGenerateAutogenEntries(basePath: string, namespaces: string[]): AutoGenConfig[] {
-    const entries = findAutogenEntries(basePath).filter(e => namespaces.some(ns => lowerCaseEquals(e.namespace, ns)));
+export async function findOrGenerateAutogenEntries(basePath: string, readme: string): Promise<AutoGenConfig[]> {
+    let entries = findAutogenEntries(basePath);
+    if (entries.some(e => e.useNamespaceFromConfig)) {
+        return entries;
+    }
 
-    for (const namespace of namespaces) {
+    const detectedNamespaces = await detectProviderNamespaces(readme);
+    entries = entries.filter(e => detectedNamespaces.some(ns => lowerCaseEquals(e.namespace, ns)));
+
+    for (const namespace of detectedNamespaces) {
         if (!entries.some(e => lowerCaseEquals(e.namespace, namespace))) {
             // Generate configuration for any RPs not explicitly declared in the autogen list
             entries.push({

generator/cmd/generateall.ts

@@ -2,10 +2,10 @@
 // Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, generateBasePaths, getPackageString, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
-import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
+import { SchemaConfiguration, generateSchemas, clearAutoGeneratedSchemaRefs, saveAutoGeneratedSchemaRefs } from '../generate';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import colors from 'colors';
-import { flatten, keys } from 'lodash';
+import { flatten } from 'lodash';
 import { executeSynchronous, chunker, writeJsonFile } from '../utils';
 import { Package } from '../models';
 import yargs from 'yargs';
@@ -72,8 +72,7 @@ executeSynchronous(async () => {
     for (const basePath of basePaths) {
         try {
             const readme = validateAndReturnReadmePath(localPath, basePath);
-            const namespaces = keys(await getApiVersionsByNamespace(readme));
-            let filteredAutoGenList = findOrGenerateAutogenEntries(basePath, namespaces)
+            let filteredAutoGenList = (await findOrGenerateAutogenEntries(basePath, readme))
                 .filter(x => x.disabledForAutogen !== true);
 
             if (args['readme-files']) {

generator/cmd/generateonboardedreport.ts

@@ -4,8 +4,7 @@ import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous, writeJsonFile, safeMkdir } from '../utils';
-import { getApiVersionsByNamespace } from '../generate';
-import { keys, partition } from 'lodash';
+import { partition } from 'lodash';
 import path from 'path';
 
 executeSynchronous(async () => {
@@ -15,8 +14,7 @@ executeSynchronous(async () => {
 
     for (const basePath of basePaths) {
         const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath);
-        const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+        const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
         const [unautogened, autogened] = partition(
             autogenlistEntries,

generator/cmd/generatesingle.ts

@@ -2,10 +2,9 @@
 // Licensed under the MIT License.
 import * as constants from '../constants';
 import { cloneAndGenerateBasePaths, resolveAbsolutePath, validateAndReturnReadmePath } from '../specs';
-import { generateSchemas, saveAutoGeneratedSchemaRefs, getApiVersionsByNamespace } from '../generate';
+import { generateSchemas, saveAutoGeneratedSchemaRefs } from '../generate';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import colors from 'colors';
-import { keys } from 'lodash';
 import { executeSynchronous } from '../utils';
 import yargs from 'yargs';
 
@@ -34,8 +33,7 @@ executeSynchronous(async () => {
     }
 
     const schemaConfigs = [];
-    const namespaces = keys(await getApiVersionsByNamespace(readme));
-    const autoGenEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+    const autoGenEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
     for (const autoGenConfig of autoGenEntries) {
         if (autoGenConfig.disabledForAutogen === true) {

generator/cmd/listbasepaths.ts

@@ -5,16 +5,14 @@ import { cloneAndGenerateBasePaths, validateAndReturnReadmePath } from '../specs
 import colors from 'colors';
 import { findOrGenerateAutogenEntries } from '../autogenlist';
 import { executeSynchronous } from '../utils';
-import { getApiVersionsByNamespace } from '../generate';
-import { keys, partition } from 'lodash';
+import { partition } from 'lodash';
 
 executeSynchronous(async () => {
     const basePaths = await cloneAndGenerateBasePaths(constants.specsRepoPath, constants.specsRepoUri, constants.specsRepoCommitHash);
 
     for (const basePath of basePaths) {
         const readme = validateAndReturnReadmePath(constants.specsRepoPath, basePath);
-        const namespaces = keys(await getApiVersionsByNamespace(readme));
-        const autogenlistEntries = findOrGenerateAutogenEntries(basePath, namespaces);
+        const autogenlistEntries = await findOrGenerateAutogenEntries(basePath, readme);
 
         const [unautogened, autogened] = partition(
             autogenlistEntries,

generator/constants.ts

@@ -8,8 +8,6 @@ export const generatorRoot = path.resolve(__dirname, '../');
 export const specsRepoPath = path.join(os.tmpdir(), 'schm_azspc');
 export const specsRepoUri = 'https://github.com/azure/azure-rest-api-specs';
 export const specsRepoCommitHash = 'origin/main';
-// eslint-disable-next-line no-useless-escape
-export const pathRegex = /(microsoft\.\w+|NGINX.NGINXPLUS|DYNATRACE.OBSERVABILITY)[\\\/]\S*[\\\/](\d{4}-\d{2}-\d{2}(|-preview))[\\\/]/i;
 
 export const autoRestVerboseOutput = false;
 

generator/generate.ts

@@ -2,15 +2,15 @@
 // Licensed under the MIT License.
 import path from 'path';
 import os from 'os';
-import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals, lowerCaseContains } from './utils';
+import { findRecursive, findDirRecursive, executeCmd, rmdirRecursive, lowerCaseCompare, lowerCaseCompareLists, lowerCaseStartsWith, readJsonFile, writeJsonFile, safeMkdir, safeUnlink, fileExists, lowerCaseEquals } from './utils';
 import * as constants from './constants';
 import { prepareReadme } from './specs';
 import colors from 'colors';
 import { ScopeType, AutoGenConfig } from './models';
-import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference, pickBy } from 'lodash';
+import { get, set, flatten, uniq, concat, Dictionary, groupBy, keys, difference } from 'lodash';
 
 const autorestBinary = os.platform() === 'win32' ? 'autorest.cmd' : 'autorest';
-const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
+export const apiVersionRegex = /^\d{4}-\d{2}-\d{2}(|-preview)$/;
 
 export interface SchemaConfiguration {
     references: SchemaReference[];
@@ -35,27 +35,17 @@ const RootSchemaConfigs: Map<ScopeType, RootSchemaConfiguration> = new Map([
     [ScopeType.ManagementGroup, constants.managementGroupRootSchema]
 ]);
 
-export async function getApiVersionsByNamespace(readme: string): Promise<Dictionary<string[]>> {
+export async function detectProviderNamespaces(readme: string) {
     const searchPath = path.resolve(`${readme}/..`);
+
+    // To try and detect possible provider namespaces, assume a folder structure of <provider>/preview|stable/<api-version>/..., based on convention
     const apiVersionPaths = await findDirRecursive(searchPath, p => path.basename(p).match(apiVersionRegex) !== null);
-
-    const output: Dictionary<string[]> = {};
-    for (const [namespace, , apiVersion] of apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep))) {
-        output[namespace] = [...(output[namespace] ?? []), apiVersion];
-    }
-
-    return output;
+    return uniq(apiVersionPaths.map(p => path.relative(searchPath, p).split(path.sep)[0]));
 }
 
-export async function generateSchemas(readme: string, autoGenConfig?: AutoGenConfig): Promise<SchemaConfiguration[]> {
+export async function generateSchemas(readme: string, autoGenConfig: AutoGenConfig): Promise<SchemaConfiguration[]> {
     await prepareReadme(readme, autoGenConfig);
 
-    const apiVersionsByNamespace = pickBy(
-        await getApiVersionsByNamespace(readme),
-        (_, key) => !autoGenConfig || lowerCaseEquals(key, autoGenConfig.namespace));
-
-    const namespaces = keys(apiVersionsByNamespace);
-
     const schemaConfigs: SchemaConfiguration[] = [];
     const tmpFolder = path.join(os.tmpdir(), Math.random().toString(36).substr(2));
 
@@ -64,7 +54,7 @@ export async function generateSchemas(readme: string, autoGenConfig?: AutoGenCon
 
         for (const schemaPath of generatedSchemas) {
             const namespace = path.basename(schemaPath.substring(0, schemaPath.lastIndexOf(path.extname(schemaPath))));
-            if (!lowerCaseContains(namespaces, namespace)) {
+            if (!lowerCaseEquals(autoGenConfig.namespace, namespace)) {
                 continue;
             }
 

generator/models.ts

@@ -14,6 +14,7 @@ export interface AutoGenConfig {
   disabledForAutogen?: true,
   basePath: string,
   namespace: string,
+  useNamespaceFromConfig?: boolean,
   readmeFile?: string,
   readmeTag?: ReadmeTag,
   suffix?: string,

generator/specs.ts

@@ -2,13 +2,14 @@
 // Licensed under the MIT License.
 import path from 'path';
 import { cloneGitRepo } from './git';
-import { findRecursive, lowerCaseEquals } from './utils';
+import { findRecursive, lowerCaseContains } from './utils';
 import { ReadmeTag, AutoGenConfig, CodeBlock } from './models';
 import * as constants from './constants'
 import * as cm from '@ts-common/commonmark-to-markdown'
 import * as yaml from 'js-yaml'
 import { existsSync } from 'fs';
 import { readFile, writeFile } from 'fs/promises';
+import { apiVersionRegex } from './generate';
 
 export async function resolveAbsolutePath(localPath: string) {
     if (path.isAbsolute(localPath)) {
@@ -76,7 +77,7 @@ function isExcludedBasePath(basePath: string) {
         .some(prefix => basePath.toLowerCase().startsWith(prefix));
 }
 
-export async function prepareReadme(readme: string, autoGenConfig?: AutoGenConfig) {
+export async function prepareReadme(readme: string, autoGenConfig: AutoGenConfig) {
     const content = (await readFile(readme)).toString();
     const markdownEx = cm.parse(content);
     const fileSet = new Set<string>();
@@ -101,21 +102,24 @@ export async function prepareReadme(readme: string, autoGenConfig?: AutoGenConfi
     }
 
     let readmeTag = {} as ReadmeTag;
-    fileSet.forEach(inputFile => {
-        const match = constants.pathRegex.exec(inputFile);
-        if (match) {
-            const mNamespace = match[1];
-            const mApiVersion = match[2];
-            if (!autoGenConfig || lowerCaseEquals(mNamespace, autoGenConfig.namespace)) {
-                if (!readmeTag[mApiVersion]) {
-                    readmeTag[mApiVersion] = [];
-                }
-                readmeTag[mApiVersion].push(inputFile);
-            }
-        }
-    });
+    for (const inputFile of fileSet) {
+        const pathComponents = inputFile.split(path.sep);
 
-    if (autoGenConfig?.readmeTag) {
+        if (!autoGenConfig.useNamespaceFromConfig &&
+            !lowerCaseContains(pathComponents, autoGenConfig.namespace)) {
+            continue;
+        }
+
+        const apiVersion = pathComponents.filter(p => p.match(apiVersionRegex) !== null)[0];
+        if (!apiVersion) {
+            continue;
+        }
+
+        readmeTag[apiVersion] ??= readmeTag[apiVersion] || [];
+        readmeTag[apiVersion].push(inputFile);
+    }
+
+    if (autoGenConfig.readmeTag) {
         readmeTag = {...readmeTag, ...autoGenConfig.readmeTag };
     }