Next gen stuff (WIP) (#2639)

* require with test

* declare directive docs and set

* declare directive support

* set as a declaration

* more directive

* operation removal

* gist URIs and some cleanup

* regen

* fix

* test fix

* stuffs

Samples/3b-custom-transformations/generated/configuration.yaml

@@ -46,6 +46,7 @@ output-artifact:
   - configuration.yaml
   - source-file-csharp
   - source-file-jsonrpcclient
+require: []
 use: []
 clear-output-folder: {}
 use-extension:
@@ -233,6 +234,24 @@ scope-cm/emitter:
   output-uri-expr: |
     "code-model-v1"
 client-side-validation: true
+declare-directive:
+  set: '{ transform: `return ${JSON.stringify($)}` }'
+  where-operation: |-
+    (() => {
+      switch ($context.from) {
+        case "code-model-v1":
+          return { from: "code-model-v1", where: `$.operations[*].methods[?(@.serializedName == ${JSON.stringify($)})]` };
+        case "swagger-document":
+        default:
+          return { from: "swagger-document", where: `$.paths.*[?(@.operationId == ${JSON.stringify($)})]` };
+      }
+    })()
+  remove-operation: |-
+    {
+      from: 'swagger-document',
+      "where-operation": $,
+      transform: 'return undefined'
+    }
 scope-csharp/emitter:
   input-artifact: source-file-csharp
   output-uri-expr: $key

Samples/3g-require-config/.gitignore

@@ -0,0 +1,2 @@
+Client/*
+!Client/SwaggerPetstore.cs

Samples/3g-require-config/readme.md

@@ -0,0 +1,20 @@
+# Scenario: Include further configuration files
+
+> see https://aka.ms/autorest
+
+
+One can include other configuration files into the current one using `require`:
+
+``` yaml
+require: # can be a single relative/absolute path/URL or an array of such
+  - ../1a-code-generation-minimal/readme.md
+```
+
+However, note that paths within included configuration files will be resolved relative to *this* file instead of the included one.
+This becomes relevant for `input-file`s or further `require`s in referenced files.
+Use `require` together with `batch` mode and `base-folder` if the intent is to invoke AutoRest on other, self-contained configuration files.
+
+Common use cases for external configuration files include extracting and reusing:
+- verbose code generation settings
+- large numbers of directives
+- declarations

Samples/3g-require-config/shell/code.txt

@@ -0,0 +1 @@
+0

Samples/3g-require-config/shell/stderr.txt

@@ -0,0 +1 @@
+

Samples/3g-require-config/shell/stdout.txt

@@ -0,0 +1,4 @@
+(C) 2017 **Microsoft Corporation.**  
+# AutoRest code generation utility [version: 2.0.0]
+https://aka.ms/autorest
+Including configuration file '/Samples/1a-code-generation-minimal/readme.md'

Samples/test/error-behavior/config-bad-syntax/shell/code.txt

@@ -1 +1 @@
-2
+1

Samples/test/error-behavior/config-bad-syntax/shell/stderr.txt

@@ -1,4 +1,2 @@
     - /Samples/test/error-behavior/config-bad-syntax/readme.md:6:0
-   at ...
-{ Error: [OperationAbortedException] Error occurred. Exiting.
 ERROR: Syntax error: Invalid YAML object.

Samples/test/error-behavior/config-bad-syntax/shell/stdout.txt

@@ -1,3 +1,4 @@
 (C) 2017 **Microsoft Corporation.**  
+[OperationAbortedException] Error occurred. Exiting.
 # AutoRest code generation utility [version: 2.0.0]
 https://aka.ms/autorest

Samples/test/error-behavior/openapi-json-bad-syntax/shell/code.txt

@@ -1 +1 @@
-4
+1

Samples/test/error-behavior/openapi-md-bad-syntax/shell/code.txt

@@ -1 +1 @@
-7
+1

Samples/test/error-behavior/openapi-md-bad-syntax/shell/stderr.txt

@@ -2,8 +2,6 @@
     - /Samples/test/error-behavior/openapi-md-bad-syntax/tiny.md:59:15
     - /Samples/test/error-behavior/openapi-md-bad-syntax/tiny.md:65:18
     - /Samples/test/error-behavior/openapi-md-bad-syntax/tiny.md:65:4
-   at ...
-{ Error: [OperationAbortedException] Error occurred. Exiting.
 ERROR: Syntax Error Encountered: Syntax error: bad indentation of a mapping entry
 ERROR: Syntax Error Encountered: Syntax error: bad indentation of a mapping entry
 ERROR: Syntax Error Encountered: Syntax error: bad indentation of a mapping entry

Samples/test/error-behavior/openapi-md-bad-syntax/shell/stdout.txt

@@ -1,4 +1,3 @@
 (C) 2017 **Microsoft Corporation.**  
 # AutoRest code generation utility [version: 2.0.0]
 https://aka.ms/autorest
-Process() Cancelled due to exception : [OperationAbortedException] Error occurred. Exiting.

Samples/test/error-behavior/openapi-not-swagger/shell/code.txt

@@ -1 +1 @@
-3
+1

Samples/test/error-behavior/openapi-yaml-bad-file-reference/shell/code.txt

@@ -1 +1 @@
-4
+1

Samples/test/error-behavior/openapi-yaml-bad-file-reference/shell/stderr.txt

@@ -1,6 +1,4 @@
     - /Samples/test/error-behavior/openapi-yaml-bad-file-reference/tiny.yaml ($)
-   at ...
-{ Error: [OperationAbortedException] Error occurred. Exiting.
 ERROR: Referenced file '/Samples/test/error-behavior/openapi-yaml-bad-file-reference/i-do-not-exist.json' not found
 FATAL: Error: [OperationAbortedException] Error occurred. Exiting.
 FATAL: swagger-document/loader - FAILED

Samples/test/error-behavior/openapi-yaml-bad-file-reference/shell/stdout.txt

@@ -1,5 +1,4 @@
 (C) 2017 **Microsoft Corporation.**  
 # AutoRest code generation utility [version: 2.0.0]
 https://aka.ms/autorest
-Process() Cancelled due to exception : [OperationAbortedException] Error occurred. Exiting.
 WARNING: Failed to blame {"path":[]} in '"/Samples/test/error-behavior/openapi-yaml-bad-file-reference/tiny.yaml"' (Error: Object '/Samples/test/error-behavior/openapi-yaml-bad-file-reference/tiny.yaml' does not exist.)

Samples/test/error-behavior/openapi-yaml-bad-reference/shell/code.txt

@@ -1 +1 @@
-4
+1

Samples/test/error-behavior/openapi-yaml-bad-syntax/shell/code.txt

@@ -1 +1 @@
-7
+1

Samples/test/hidden-methods/readme.md

@@ -21,7 +21,7 @@ csharp:
   - output-folder: ClientFancy
     directive:
       from: code-model-v1
-      where: $.operations[*].methods[?(@.serializedName == "Cowbell_Add")]
+      where-operation: Cowbell_Add
       transform: $.hidden = true
 ```
 

Samples/test/remove-methods/readme.md

@@ -17,9 +17,7 @@ batch:
     output-folder: ClientFancy
     # add the following directive to your configuration to hide operation `Cowbell_Add`
     directive:
-      - from: swagger-document
-        where: $.paths.*[?(@.operationId == "Cowbell_Add")]
-        transform: return undefined
+      - remove-operation: Cowbell_Add
     # for demonstration purposes, also emit a Swagger file that represents the original Swagger minus the removed operation
     output-artifact: swagger-document.yaml
 ```
@@ -30,7 +28,5 @@ The directive finds method `Cowbell_Add` and removes the corresponding object gr
 
 Add a `directive` as follows in order to remove an operation with operation ID `<OPERATION_ID>`:
 ``` yaml false
-from: swagger-document
-where: $.paths.*[?(@.operationId == "<OPERATION_ID>")]
-transform: return undefined
+remove-operation: <OPERATION_ID>
 ```

src/autorest-core/app.ts

@@ -257,43 +257,34 @@ async function currentMain(autorestArgs: string[]): Promise<number> {
   api.GeneratedFile.Subscribe((_, artifact) => artifacts.push(artifact));
   api.ClearFolder.Subscribe((_, folder) => clearFolders.push(folder));
 
-  try {
-    const config = (await api.view);
+  const config = (await api.view);
 
-    // maybe a resource schema batch process
-    if (config["resource-schema-batch"]) {
-      return await resourceSchemaBatch(api);
-    }
+  // maybe a resource schema batch process
+  if (config["resource-schema-batch"]) {
+    return await resourceSchemaBatch(api);
+  }
 
-    // maybe a merge process
-    if (config["merge"]) {
-      return await merge(api);
-    }
+  // maybe a merge process
+  if (config["merge"]) {
+    return await merge(api);
+  }
 
-    if (config["batch"]) {
-      await batch(api);
-    }
-    else {
-      const result = await api.Process().finish;
-      if (result !== true) {
-        throw result;
-      }
-    }
-
-    // perform file system operations.
-    for (const folder of clearFolders) {
-      try { await ClearFolder(folder); } catch (e) { }
-    }
-    for (const artifact of artifacts) {
-      await WriteString(artifact.uri, artifact.content);
+  if (config["batch"]) {
+    await batch(api);
+  }
+  else {
+    const result = await api.Process().finish;
+    if (result !== true) {
+      throw result;
     }
   }
-  catch (e) {
-    api.Message.Dispatch({
-      Channel: Channel.Fatal,
-      Text: e
-    });
-    exitcode = exitcode || 1;
+
+  // perform file system operations.
+  for (const folder of clearFolders) {
+    try { await ClearFolder(folder); } catch (e) { }
+  }
+  for (const artifact of artifacts) {
+    await WriteString(artifact.uri, artifact.content);
   }
 
   // return the exit code to the caller.
@@ -469,26 +460,22 @@ async function main() {
       exitcode = await currentMain(autorestArgs);
     }
 
-    // for relaxed profiling (assuming that no one calls `main` from electron... use AAAL!)
-    if (require("process").versions.electron) await new Promise(_ => { });
-
     process.exit(exitcode);
   } catch (e) {
+    // be very careful about the following check:
+    // - doing the inversion (instanceof Error) doesn't reliably work since that seems to return false on Errors marshalled from safeEval
     if (e instanceof Exception) {
-      console.log(e.message);
-
-      if (autorestArgs.indexOf("--debug")) {
+      if (autorestArgs.indexOf("--debug") !== -1) {
         console.log(e);
+      } else {
+        console.log(e.message);
       }
       process.exit(e.exitCode);
     }
 
-    if (e instanceof Error) {
+    if (e !== false) {
       console.error(e);
-      process.exit(1);
     }
-
-    console.error(e);
     process.exit(1);
   }
 }

src/autorest-core/lib/autorest-core.ts

@@ -42,16 +42,16 @@ export class AutoRest extends EventEmitter {
 
   public static async LiterateToJson(content: string): Promise<string> {
     try {
-    let autorest = new AutoRest({
-      EnumerateFileUris: async function (folderUri: string): Promise<Array<string>> { return []; },
+      let autorest = new AutoRest({
+        EnumerateFileUris: async function (folderUri: string): Promise<Array<string>> { return []; },
         ReadFile: async (f: string): Promise<string> => f == "none:///empty-file.md" ? content || "# empty file" : "# empty file"
-    });
-    let result = "";
+      });
+      let result = "";
       autorest.AddConfiguration({ "input-file": "none:///empty-file.md", "output-artifact": ["swagger-document"] });
-    autorest.GeneratedFile.Subscribe((source, artifact) => {
-      result = artifact.content;
-    });
-    // run autorest and wait.
+      autorest.GeneratedFile.Subscribe((source, artifact) => {
+        result = artifact.content;
+      });
+      // run autorest and wait.
 
       await (await autorest.Process()).finish;
       return result;
@@ -212,25 +212,22 @@ export class AutoRest extends EventEmitter {
         view.messageEmitter.removeAllListeners();
         return true;
       } catch (e) {
-        if (e instanceof Error) {
-          /* if (!(e instanceof OperationCanceledException)) {
-            console.error(e.message);
-          } */
-          this.Message.Dispatch({ Channel: Channel.Debug, Text: `Process() Cancelled due to exception : ${e.message}` });
-          this.Finished.Dispatch(e);
-
+        if (e instanceof Exception) {
+          // console.error(e);
+          this.Finished.Dispatch(false);
           if (view) {
             view.messageEmitter.removeAllListeners();
           }
-          return e;
+          return false;
         }
 
-        // console.error(e);
-        this.Finished.Dispatch(false);
+        this.Message.Dispatch({ Channel: Channel.Debug, Text: `Process() Cancelled due to exception : ${e.message}` });
+        this.Finished.Dispatch(e);
+
         if (view) {
           view.messageEmitter.removeAllListeners();
         }
-        return false;
+        return e;
       }
     };
     return {

src/autorest-core/lib/configuration.ts

@@ -22,10 +22,11 @@ import { exists } from './ref/async';
 import { CancellationToken, CancellationTokenSource } from './ref/cancellation';
 import { stringify } from './ref/jsonpath';
 import { From } from './ref/linq';
-import { CreateFileUri, CreateFolderUri, EnsureIsFolderUri, ResolveUri } from './ref/uri';
+import { CreateFileUri, CreateFolderUri, EnsureIsFolderUri, ExistsUri, ResolveUri } from './ref/uri';
 import { BlameTree } from './source-map/blaming';
 import { MergeOverwriteOrAppend, resolveRValue } from './source-map/merging';
 import { TryDecodeEnhancedPositionFromName } from './source-map/source-map';
+import { safeEval } from './ref/safe-eval';
 
 const untildify: (path: string) => string = require("untildify");
 
@@ -37,9 +38,11 @@ export interface AutoRestConfigurationImpl {
   "input-file"?: string[] | string;
   "base-folder"?: string;
   "directive"?: Directive[] | Directive;
+  "declare-directive"?: { [name: string]: string };
   "output-artifact"?: string[] | string;
   "message-format"?: "json";
   "use-extension"?: { [extensionName: string]: string };
+  "require"?: string[] | string;
   "vscode"?: any; // activates VS Code specific behavior and does *NOT* influence the core's behavior (only consumed by VS Code extension)
 
   "override-info"?: any; // make sure source maps are pulling it! (see "composite swagger" method)
@@ -198,7 +201,6 @@ export class ConfigurationView {
     /* @internal */public configFileFolderUri: string,
     ...configs: Array<AutoRestConfigurationImpl> // decreasing priority
   ) {
-
     // TODO: fix configuration loading, note that there was no point in passing that DataStore used
     // for loading in here as all connection to the sources is lost when passing `Array<AutoRestConfigurationImpl>` instead of `DataHandleRead`s...
     // theoretically the `ValuesOf` approach and such won't support blaming (who to blame if $.directives[3] sucks? which code block was it from)
@@ -207,6 +209,7 @@ export class ConfigurationView {
       "directive": [],
       "input-file": [],
       "output-artifact": [],
+      "require": [],
       "use": [],
     };
 
@@ -235,7 +238,6 @@ export class ConfigurationView {
     } else {
       this.config = this.rawConfig;
     }
-
     this.suppressor = new Suppressor(this);
     this.Message({ Channel: Channel.Debug, Text: `Creating ConfigurationView : ${configs.length} sections.` });
   }
@@ -244,7 +246,7 @@ export class ConfigurationView {
     return Object.getOwnPropertyNames(this.config);
   }
 
-  public Dump(title: string = "") {
+  public Dump(title: string = ""): void {
     console.log(`\n${title}\n===================================`)
     for (const each of Object.getOwnPropertyNames(this.config)) {
       console.log(`${each} : ${(<any>this.config)[each]}`);
@@ -294,9 +296,37 @@ export class ConfigurationView {
     });
   }
 
-  public get Directives(): Iterable<DirectiveView> {
-    return From(ValuesOf<Directive>(this.config["directive"]))
-      .Select(each => new DirectiveView(each));
+  public get IncludedConfigurationFiles(): string[] {
+    return From<string>(ValuesOf<string>(this.config["require"]))
+      .Select(each => this.ResolveAsPath(each))
+      .ToArray();
+  }
+
+  public get Directives(): DirectiveView[] {
+    const plainDirectives = ValuesOf<Directive>(this.config["directive"]);
+    const declarations = this.config["declare-directive"] || {};
+    const expandDirective = (dir: Directive): Iterable<Directive> => {
+      const makro = Object.keys(dir).filter(makro => declarations[makro])[0];
+      if (!makro) {
+        return [dir]; // nothing to expand
+      }
+      // prepare directive
+      let parameters = (dir as any)[makro];
+      if (!Array.isArray(parameters)) {
+        parameters = [parameters];
+      }
+      dir = { ...dir };
+      delete (dir as any)[makro];
+      // call makro
+      const makroResults: any = From(parameters).SelectMany(parameter => {
+        // console.log(new Error().stack);
+        const result = safeEval(declarations[makro], { $: parameter, $context: dir });
+        return Array.isArray(result) ? result : [result];
+      }).ToArray();
+      return From(makroResults).SelectMany((result: any) => expandDirective(Object.assign(result, dir)));
+    };
+    // makro expansion
+    return From(plainDirectives).SelectMany(expandDirective).Select(each => new DirectiveView(each)).ToArray();
   }
 
   public get InputFileUris(): string[] {
@@ -390,7 +420,7 @@ export class ConfigurationView {
             return [s];
           }
 
-          return blameTree.BlameLeafs().map(r => <SourceLocation>{ document: r.source, Position: Object.assign(TryDecodeEnhancedPositionFromName(r.name) || {}, { line: r.line, column: r.column }) });
+          return blameTree.BlameLeafs().map(r => <SourceLocation>{ document: r.source, Position: { ...TryDecodeEnhancedPositionFromName(r.name), line: r.line, column: r.column } });
         });
 
         //console.log("---");
@@ -484,7 +514,7 @@ export class ConfigurationView {
 
 export class Configuration {
   public constructor(
-    private fileSystem?: IFileSystem,
+    private fileSystem: IFileSystem = new RealFileSystem(),
     private configFileOrFolderUri?: string,
   ) { }
 
@@ -516,7 +546,7 @@ export class Configuration {
   private async DesugarRawConfig(configs: any): Promise<any> {
     // shallow copy
     configs = Object.assign({}, configs);
-    configs["use-extension"] = Object.assign({}, configs["use-extension"]);
+    configs["use-extension"] = { ...configs["use-extension"] };
 
     if (configs.hasOwnProperty('licence-header')) {
       configs['license-header'] = configs['licence-header'];
@@ -560,23 +590,54 @@ export class Configuration {
       : null;
     const configFileFolderUri = configFileUri ? ResolveUri(configFileUri, "./") : (this.configFileOrFolderUri || "file:///");
 
-    const createView = () => new ConfigurationView(messageEmitter, configFileFolderUri, ...configSegments);
-
     const configSegments: any[] = [];
+    const createView = () => new ConfigurationView(messageEmitter, configFileFolderUri, ...configSegments);
     const addSegments = async (configs: any[]): Promise<void> => { configSegments.push(...await this.DesugarRawConfigs(configs)); };
 
     // 1. overrides (CLI, ...)
     await addSegments(configs);
     // 2. file
     if (configFileUri !== null) {
-      const inputView = messageEmitter.DataStore.GetReadThroughScope(this.fileSystem as IFileSystem);
+      const inputView = messageEmitter.DataStore.GetReadThroughScope(this.fileSystem);
       const blocks = await this.ParseCodeBlocks(
         await inputView.ReadStrict(configFileUri),
         createView(),
         "config");
       await addSegments(blocks);
     }
-    // 3. default configuration
+    // 3. resolve 'require'd configuration
+    const addedConfigs = new Set<string>();
+    while (true) {
+      const tmpView = createView();
+      const additionalConfigs = tmpView.IncludedConfigurationFiles.filter(ext => !addedConfigs.has(ext));
+      if (additionalConfigs.length === 0) {
+        break;
+      }
+      // acquire additional configs
+      for (const additionalConfig of additionalConfigs) {
+        try {
+          messageEmitter.Message.Dispatch({
+            Channel: Channel.Verbose,
+            Text: `Including configuration file '${additionalConfig}'`
+          });
+          addedConfigs.add(additionalConfig);
+          // merge config
+          const inputView = messageEmitter.DataStore.GetReadThroughScope(this.fileSystem);
+          const blocks = await this.ParseCodeBlocks(
+            await inputView.ReadStrict(additionalConfig),
+            tmpView,
+            `require-config-${additionalConfig}`);
+          await addSegments(blocks);
+        } catch (e) {
+          messageEmitter.Message.Dispatch({
+            Channel: Channel.Fatal,
+            Text: `Failed to acquire 'require'd configuration '${additionalConfig}'`
+          });
+          throw e;
+        }
+      }
+    }
+    // 4. default configuration
     if (includeDefault) {
       const inputView = messageEmitter.DataStore.GetReadThroughScope(new RealFileSystem());
       const blocks = await this.ParseCodeBlocks(
@@ -585,7 +646,7 @@ export class Configuration {
         "default-config");
       await addSegments(blocks);
     }
-    // 4. resolve externals
+    // 5. resolve extensions
     const extMgr = await this.extensionManager;
     const addedExtensions = new Set<string>();
     while (true) {
@@ -680,10 +741,23 @@ export class Configuration {
   public static async DetectConfigurationFile(fileSystem: IFileSystem, configFileOrFolderUri: string | null, messageEmitter?: MessageEmitter, walkUpFolders: boolean = false): Promise<string | null> {
     const originalConfigFileOrFolderUri = configFileOrFolderUri;
 
+    //
     if (!configFileOrFolderUri || configFileOrFolderUri.endsWith(".md")) {
       return configFileOrFolderUri;
     }
 
+    // try querying the Uri directly
+    if (ExistsUri(configFileOrFolderUri)) {
+      try {
+        const content = await fileSystem.ReadFile(configFileOrFolderUri);
+        if (content.indexOf(Constants.MagicString) > -1) {
+          return configFileOrFolderUri;
+        }
+      } catch (e) {
+        // that didn't work... try next
+      }
+    }
+
     // search for a config file, walking up the folder tree
     while (configFileOrFolderUri !== null) {
       // scan the filesystem items for the configuration.

src/autorest-core/lib/data-store/data-store.ts

@@ -96,7 +96,7 @@ class ReadThroughDataSource extends DataSource {
   }
 
   public async Read(uri: string): Promise<DataHandle | null> {
-    uri = ToRawDataUrl(uri);
+    uri = ToRawDataUrl(uri); // makes sure logical paths (like for source maps) also reference the URLs of the actual data
 
     // sync cache (inner stuff is racey!)
     if (!this.cache[uri]) {

src/autorest-core/lib/exception.ts

@@ -6,7 +6,7 @@
 export class Exception extends Error {
   constructor(message: string, public exitCode: number = 1) {
 
-    super(message.indexOf('[') == -1 ? `[Exception] ${message}` : message);
+    super(message.includes('[') ? message : `[Exception] ${message}`);
 
     Object.setPrototypeOf(this, Exception.prototype);
   }

src/autorest-core/lib/file-system.ts

@@ -5,9 +5,8 @@
 import { MessageEmitter } from './configuration';
 import { ConfigurationView } from './autorest-core';
 import { Channel } from "./message";
-import { EnumerateFiles } from "./ref/uri";
+import { EnumerateFiles, ToRawDataUrl, ResolveUri, ReadUri, WriteString } from './ref/uri';
 import { From } from "./ref/linq";
-import { ResolveUri, ReadUri, WriteString } from "./ref/uri";
 import * as Constants from './constants';
 
 export interface IFileSystem {
@@ -70,6 +69,7 @@ export class RealFileSystem implements IFileSystem {
 }
 
 // handles:
+// - GitHub URI adjustment
 // - GitHub auth
 export class EnhancedFileSystem implements IFileSystem {
   public constructor(private githubAuthToken?: string) {
@@ -81,6 +81,8 @@ export class EnhancedFileSystem implements IFileSystem {
     ]);
   }
   async ReadFile(uri: string): Promise<string> {
+    uri = ToRawDataUrl(uri);
+
     const headers: { [key: string]: string } = {};
 
     // check for GitHub OAuth token

src/autorest-core/lib/parsing/literate-yaml.ts

@@ -112,11 +112,19 @@ export function EvaluateGuard(rawFenceGuard: string, contextObject: any): boolea
   let guardResult = false;
   let expressionFence: string = '';
   try {
+    if (!fence.includes("$(")) {
+      try {
+        return safeEval<boolean>(fence);
+      } catch (e) {
+        return false;
+      }
+    }
+
     expressionFence = `${resolveRValue(fence, "", contextObject, null, 2)}`;
     // is there unresolved values?  May be old-style. Or the values aren't defined. 
 
     // Let's run it only if there are no unresolved values for now. 
-    if (expressionFence.indexOf("$(") == -1) {
+    if (!expressionFence.includes("$(")) {
       return safeEval<boolean>(expressionFence);
     }
   } catch (E) {
@@ -126,7 +134,7 @@ export function EvaluateGuard(rawFenceGuard: string, contextObject: any): boolea
   // is this a single $( ... ) expression ?
   match = /^\$\((.*)\)$/.exec(fence.trim());
 
-  const guardExpression = match && match[1].indexOf("$(") == -1 && match[1];
+  const guardExpression = match && !match[1].includes("$(") && match[1];
   if (!guardExpression) {
     // Nope. this isn't an old style expression.
     // at best, it can be an expression that doesn't have all the values resolved.
@@ -135,7 +143,7 @@ export function EvaluateGuard(rawFenceGuard: string, contextObject: any): boolea
   }
 
   // fall back to original behavior, where the whole expression is in the $( ... )
-  const context = Object.assign({ $: contextObject }, contextObject);
+  const context = { $: contextObject, ...contextObject };
 
   try {
     guardResult = safeEval<boolean>(guardExpression, context);

src/autorest-core/lib/pipeline/manipulation.ts

@@ -17,7 +17,7 @@ export class Manipulator {
   private ctr = 0;
 
   public constructor(private config: ConfigurationView) {
-    this.transformations = From(config.Directives).ToArray();
+    this.transformations = config.Directives;
   }
 
   private MatchesSourceFilter(document: string, transform: DirectiveView, artifact: string | null): boolean {
@@ -57,22 +57,6 @@ export class Manipulator {
             }
             data = result.result;
           }
-          // set
-          for (const s of trans.set) {
-            const result = await ManipulateObject(data, sink, w, obj => s/*,
-              {
-                reason: trans.reason,
-                transformerSourceHandle: // TODO
-              }*/);
-            if (!result.anyHit) {
-              // this.config.Message({
-              //   Channel: Channel.Warning,
-              //   Details: trans,
-              //   Text: `Set directive with 'where' clause '${w}' was not used.`
-              // });
-            }
-            data = result.result;
-          }
           // test
           for (const t of trans.test) {
             const doc = data.ReadObject<any>();

src/autorest-core/lib/pipeline/pipeline.ts

@@ -256,7 +256,6 @@ function BuildPipeline(config: ConfigurationView): { pipeline: { [name: string]:
 }
 
 export async function RunPipeline(configView: ConfigurationView, fileSystem: IFileSystem): Promise<void> {
-
   // built-in plugins
   const plugins: { [name: string]: PipelinePlugin } = {
     "identity": CreatePluginIdentity(),
@@ -288,7 +287,8 @@ export async function RunPipeline(configView: ConfigurationView, fileSystem: IFi
   // __status scope
   const startTime = Date.now();
   (configView.Raw as any).__status = new Proxy<any>({}, {
-    async get(_, key) {
+    get(_, key) {
+      if (key === "__info") return false;
       const expr = new Buffer(key.toString(), "base64").toString("ascii");
       try {
         return FastStringify(safeEval(expr, {

src/autorest-core/lib/pipeline/suppression.ts

@@ -13,7 +13,7 @@ export class Suppressor {
   private suppressions: DirectiveView[];
 
   public constructor(private config: ConfigurationView) {
-    this.suppressions = From(config.Directives).Where(x => [...x.suppress].length > 0).ToArray();
+    this.suppressions = config.Directives.filter(x => [...x.suppress].length > 0);
   }
 
   private MatchesSourceFilter(document: string, path: JsonPath | undefined, supression: DirectiveView): boolean {

src/autorest-core/lib/ref/uri.ts

@@ -122,10 +122,17 @@ export function GetFilenameWithoutExtension(uri: string): string {
 }
 
 export function ToRawDataUrl(uri: string): string {
-  // special URI handlers                                                                                        
-  // - GitHub                                                                                                    
-  if (uri.startsWith("https://github")) {
-    uri = uri.replace(/^https?:\/\/(github.com)(\/[^\/]+\/[^\/]+\/)(blob|tree)\/(.*)/ig, "https://raw.githubusercontent.com$2$4");
+  // special URI handlers (the 'if's shouldn't be necessary but provide some additional isolation in case there is anything wrong with one of the regexes)
+  // - GitHub repo
+  if (uri.startsWith("https://github.com")) {
+    uri = uri.replace(/^https?:\/\/(github.com)(\/[^\/]+\/[^\/]+\/)(blob|tree)\/(.*)$/ig, "https://raw.githubusercontent.com$2$4");
+  }
+  // - GitHub gist
+  if (uri.startsWith("gist://")) {
+    uri = uri.replace(/^gist:\/\/([^\/]+\/[^\/]+)$/ig, "https://gist.githubusercontent.com/$1/raw/");
+  }
+  if (uri.startsWith("https://gist.github.com")) {
+    uri = uri.replace(/^https?:\/\/gist.github.com\/([^\/]+\/[^\/]+)$/ig, "https://gist.githubusercontent.com/$1/raw/");
   }
 
   return uri;
@@ -203,7 +210,7 @@ function isAccessibleFile(localPath: string) {
 function FileUriToLocalPath(fileUri: string): string {
   const uri = parse(fileUri);
   if (!fileUri.startsWith("file:///")) {
-    throw new Error((!fileUri.startsWith("file://")
+    throw new Error(`Cannot write data to '${fileUri}'. ` + (!fileUri.startsWith("file://")
       ? `Protocol '${uri.protocol}' not supported for writing.`
       : `UNC paths not supported for writing.`) + " Make sure to specify a local, absolute path as target file/folder.");
   }

src/autorest-core/lib/source-map/blaming.ts

@@ -17,10 +17,7 @@ export class BlameTree {
     const enhanced = TryDecodeEnhancedPositionFromName(position.name);
     if (enhanced !== undefined) {
       for (const blame of blames) {
-        blame.name = EncodeEnhancedPositionInName(blame.name, Object.assign(
-          {},
-          enhanced,
-          TryDecodeEnhancedPositionFromName(blame.name) || {}));
+        blame.name = EncodeEnhancedPositionInName(blame.name, { ...enhanced, ...TryDecodeEnhancedPositionFromName(blame.name) });
       }
     }
 

src/autorest-core/resources/default-configuration.md

@@ -194,3 +194,108 @@ On by default for backwards compatibility, but see https://github.com/Azure/auto
 ``` yaml
 client-side-validation: true
 ```
+
+# Directives
+
+The built-in `transform` directive with its filters `from` and `where` are very powerful, but can become verbose and thus hard to reuse for common patterns (e.g. rename an operation).
+Furthermore, they usually rely on precise data formats (e.g. where to find operations in the `code-model-v1`) and thus break once the data format changes.
+We propose the following mechanism of declaring directives similar to macros, which allows capturing commonly used directives in a more high-level way.
+Configuration files using these macros instead of "low-level" directives are robust against changes in the data format as the declaration in here will be adjusted accordingly.
+
+## How it works
+
+A declaration such as
+
+``` yaml false
+declare-directive:
+  my-directive: >-
+    [
+      {
+        transform: `some transformer, parameterized with '${JSON.stringify($)}'`
+      },
+      {
+        from: "code-model-v1"
+        transform: `some other transformer, parameterized with '${JSON.stringify($)}'`
+      }
+    ]
+```
+
+can be used by naming it in a `directive` section:
+
+``` yaml false
+directive:
+  - my-directive: # as a standalone, with an object as parameter
+      foo: bar
+      baz: 42
+  - from: a
+    where: b
+    my-directive: 42 # together with other stuff, with a number as parameter
+```
+
+Each `directive` entry that names `my-directive` will be expanded with the whatever the declaration evaluates to, where `$` is substituted with the value provided to the directive when used.
+If the declaration evaluates to an array, the directives are duplicated accordingly (this enables directive declarations that transform data on multiple stages).
+In the above example, `directive` gets expanded to:
+
+``` yaml false
+directive:
+  - transform: >-
+      some transformer, parameterized with '{ "foo": \"bar\", "baz": 42 }'
+  - from: code-model-v1
+    transform: >-
+      some other transformer, parameterized with '{ "foo": \"bar\", "baz": 42 }'
+  - from: a
+    where: b
+    transform: >-
+      some transformer, parameterized with '42'
+  - from: a
+    where: b
+    transform: >-
+      some other transformer, parameterized with '42'
+```
+
+As can be seen in the last `directive`, `from` as specified originally was not overridden by `code-model-v1`, i.e. what was specified by the user is given higher priority.
+
+
+## `set`
+
+Formerly implemented in the AutoRest core itself, `set` is now just syntactic sugar for `transform`.
+
+``` yaml
+declare-directive:
+  set: >-
+    { transform: `return ${JSON.stringify($)}` }
+```
+
+## Operations
+
+### Selection
+
+Select operations by ID at different stages of the pipeline.
+
+``` yaml
+declare-directive:
+  where-operation: >-
+    (() => {
+      switch ($context.from) {
+        case "code-model-v1":
+          return { from: "code-model-v1", where: `$.operations[*].methods[?(@.serializedName == ${JSON.stringify($)})]` };
+        case "swagger-document":
+        default:
+          return { from: "swagger-document", where: `$.paths.*[?(@.operationId == ${JSON.stringify($)})]` };
+      }
+    })()
+```
+
+## Removal
+
+Removes an operation by ID.
+
+``` yaml
+declare-directive:
+  remove-operation: >-
+    {
+      from: 'swagger-document',
+      "where-operation": $,
+      transform: 'return undefined'
+    }
+```

src/autorest-core/test/syntax-validation.ts

@@ -31,7 +31,7 @@ import { Parse } from "../lib/parsing/literate-yaml";
     return messages;
   }
 
-  @test async "syntax errors"() {
+  @test @timeout(10000) async "syntax errors"() {
     // good
     assert.strictEqual((await this.GetLoaderErrors("{ a: 3 }")).length, 0);
     assert.strictEqual((await this.GetLoaderErrors("a: 3")).length, 0);