ignore-nested-required a.k.a quirks mode (#2699)

* new C#

* fix build

* regen script

* fix install

* added quirksmode flag and transformer

* nested transform

* JSONPATH fix

* bump C#

* fix

* regen

Samples/3b-custom-transformations/generated/configuration.yaml

@@ -98,7 +98,7 @@ pipeline:
     suffixes:
       - ''
   csharp/commonmarker:
-    input: modeler
+    input: imodeler1
     output-artifact: code-model-v1
     suffixes:
       - ''
@@ -113,8 +113,8 @@ pipeline:
     plugin: csharp
     suffixes:
       - ''
-  csharp/modeler:
-    input: swagger-document/identity
+  csharp/imodeler1:
+    input: openapi-document/identity
     output-artifact: code-model-v1
     scope: csharp
     suffixes:
@@ -136,12 +136,12 @@ pipeline:
     scope: scope-jsonrpcclient/emitter
     suffixes: []
   jsonrpcclient/generate:
-    input: modeler
+    input: imodeler1
     output-artifact: source-file-jsonrpcclient
     plugin: jsonrpcclient
     suffixes: []
-  jsonrpcclient/modeler:
-    input: swagger-document/identity
+  jsonrpcclient/imodeler1:
+    input: openapi-document/identity
     output-artifact: code-model-v1
     scope: jsonrpcclient
     suffixes: []
@@ -156,7 +156,7 @@ pipeline:
     suffixes:
       - ''
   openapi-document/identity:
-    input: openapi-document-converter
+    input: transform
     output-artifact: openapi-document
     suffixes:
       - ''
@@ -165,6 +165,11 @@ pipeline:
     output-artifact: openapi-document
     suffixes:
       - ''
+  openapi-document/transform:
+    input: openapi-document-converter
+    output-artifact: openapi-document
+    suffixes:
+      - ''
   pipeline-emitter:
     scope: scope-pipeline-emitter
     suffixes:
@@ -289,9 +294,14 @@ scope-transform-string:
 skip-upgrade-check: {}
 use: []
 use-extension:
-  '@microsoft.azure/autorest.csharp': ~2.1.0
-  '@microsoft.azure/autorest.modeler': 2.1.22
+  '@microsoft.azure/autorest.csharp': ~2.2.0
+  '@microsoft.azure/autorest.modeler': 2.3.31
   '@microsoft.azure/classic-openapi-validator': ~1.0.3
   '@microsoft.azure/openapi-validator': ~1.0.0
+used-extension:
+  - '["@microsoft.azure/autorest.csharp","~2.2.0"]'
+  - '["@microsoft.azure/classic-openapi-validator","~1.0.3"]'
+  - '["@microsoft.azure/openapi-validator","~1.0.0"]'
+  - '["@microsoft.azure/autorest.modeler","2.3.31"]'
 verbose: false
 

Samples/3b-custom-transformations/generated/pipeline.yaml

@@ -22,7 +22,7 @@ csharp/commonmarker:
   configScope:
     - csharp
   inputs:
-    - csharp/modeler
+    - csharp/imodeler1
   outputArtifact: code-model-v1
   pluginName: commonmarker
 csharp/emitter:
@@ -40,13 +40,13 @@ csharp/generate:
     - csharp/cm/transform
   outputArtifact: source-file-csharp
   pluginName: csharp
-csharp/modeler:
+csharp/imodeler1:
   configScope:
     - csharp
   inputs:
-    - swagger-document/identity
+    - openapi-document/identity
   outputArtifact: code-model-v1
-  pluginName: modeler
+  pluginName: imodeler1
 csharp/simplifier:
   configScope:
     - csharp
@@ -71,7 +71,7 @@ openapi-document/emitter:
 openapi-document/identity:
   configScope: []
   inputs:
-    - openapi-document/openapi-document-converter
+    - openapi-document/transform
   outputArtifact: openapi-document
   pluginName: identity
 openapi-document/openapi-document-converter:
@@ -80,6 +80,12 @@ openapi-document/openapi-document-converter:
     - swagger-document/identity
   outputArtifact: openapi-document
   pluginName: openapi-document-converter
+openapi-document/transform:
+  configScope: []
+  inputs:
+    - openapi-document/openapi-document-converter
+  outputArtifact: openapi-document
+  pluginName: transform
 pipeline-emitter:
   configScope:
     - scope-pipeline-emitter

Samples/test/error-behavior/openapi-yaml-bad-reference/shell/stderr.txt

@@ -9,17 +9,7 @@
      openapi: 
    { patch: true,
    at ...
-  at ...
-  at ...
---- End of stack trace from previous location where exception was thrown ---
-(node) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.
-(node) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 10): Error: Could not resolve reference #/definitions/Cowball
-(node) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 8): Error: Could not resolve reference #/definitions/Cowball
-(node) UnhandledPromiseRejectionWarning: Unhandled promise rejection (rejection id: 9): Error: Could not resolve reference #/definitions/Cowball
 { Error: Could not resolve reference #/definitions/Cowball
-FATAL: csharp/modeler - FAILED
 FATAL: Error: Could not resolve reference #/definitions/Cowball
-FATAL: Error: Plugin modeler reported failure.
 FATAL: openapi-document/openapi-document-converter - FAILED
-FATAL: System.ArgumentException: Reference path 'Cowball' does not exist in the definition section of the Swagger document.
 options: 

Samples/test/error-behavior/openapi-yaml-bad-type/shell/stderr.txt

@@ -1 +1 @@
-ERROR: Error converting value "Cowbell" to type 'System.Nullable`1[AutoRest.Modeler.Model.DataType]'. Path 'type', line 2, position 19.
+ERROR: Error converting value "Cowbell" to type 'System.Nullable`1[AutoRest.Modeler.Model.DataType]'. Path 'requestBody.content.application/json.schema.type', line 14, position 27.

Samples/test/hidden-methods/Client/code-model-v1.yaml

@@ -68,6 +68,8 @@ operations:
           isNullable: true
         deprecated: false
         description: A good description.
+        extensions:
+          x-ms-requestBody-index: '0'
         group:
           fixed: false
           raw: Cowbell
@@ -84,6 +86,8 @@ operations:
               fixed: false
             documentation:
               fixed: false
+            extensions:
+              x-ms-requestBody-name: body
             isConstant: false
             isRequired: true
             location: body

Samples/test/hidden-methods/ClientFancy/code-model-v1.yaml

@@ -68,6 +68,8 @@ operations:
           isNullable: true
         deprecated: false
         description: A good description.
+        extensions:
+          x-ms-requestBody-index: '0'
         group:
           fixed: false
           raw: Cowbell
@@ -84,6 +86,8 @@ operations:
               fixed: false
             documentation:
               fixed: false
+            extensions:
+              x-ms-requestBody-name: body
             isConstant: false
             isRequired: true
             location: body

package.json

@@ -7,7 +7,8 @@
     "testci": "gulp testci",
     "build": "gulp build --threshold=1",
     "publish-preview": "gulp publish-preview",
-    "install": "gulp npm-install --verbose"
+    "install": "gulp npm-install --verbose",
+    "regenerate": "gulp regenerate"
   },
   "repository": {
     "type": "git",

src/autorest-core/lib/configuration.ts

@@ -619,8 +619,8 @@ export class Configuration {
     const configFileFolderUri = configFileUri ? ResolveUri(configFileUri, "./") : (this.configFileOrFolderUri || "file:///");
 
     const configSegments: any[] = [];
-    const createView = () => new ConfigurationView(messageEmitter, configFileFolderUri, ...configSegments);
-    const addSegments = async (configs: any[]): Promise<void> => { configSegments.push(...await this.DesugarRawConfigs(configs)); };
+    const createView = (segments: any[] = configSegments) => new ConfigurationView(messageEmitter, configFileFolderUri, ...segments);
+    const addSegments = async (configs: any[]): Promise<any[]> => { const segs = await this.DesugarRawConfigs(configs); configSegments.push(...segs); return segs; };
 
     // 1. overrides (CLI, ...)
     await addSegments(configs);
@@ -677,11 +677,13 @@ export class Configuration {
     // 5. resolve extensions
     const extMgr = await this.extensionManager;
     const addedExtensions = new Set<string>();
-    while (true) {
-      const tmpView = createView();
+    const viewsToHandle: ConfigurationView[] = [createView()];
+    while (viewsToHandle.length > 0) {
+      const tmpView = <ConfigurationView>viewsToHandle.pop();
       const additionalExtensions = tmpView.UseExtensions.filter(ext => !addedExtensions.has(ext.fullyQualified));
+      await addSegments([{ "used-extension": tmpView.UseExtensions.map(x => x.fullyQualified) }]);
       if (additionalExtensions.length === 0) {
-        break;
+        continue;
       }
       // acquire additional extensions
       for (const additionalExtension of additionalExtensions) {
@@ -750,9 +752,9 @@ export class Configuration {
           const inputView = messageEmitter.DataStore.GetReadThroughScope(new RealFileSystem());
           const blocks = await this.ParseCodeBlocks(
             await inputView.ReadStrict(CreateFileUri(await ext.extension.configurationPath)),
-            tmpView,
+            createView(),
             `extension-config-${additionalExtension.fullyQualified}`);
-          await addSegments(blocks);
+          viewsToHandle.push(createView(await addSegments(blocks)));
         } catch (e) {
           messageEmitter.Message.Dispatch({
             Channel: Channel.Fatal,

src/autorest-core/lib/pipeline/object-manipulator.ts

@@ -24,19 +24,11 @@ export async function ManipulateObject(
 
   // find paths matched by `whereJsonQuery`
   const doc = src.ReadObject<any>();
-  const allHits = nodes(doc, whereJsonQuery).sort((a, b) => a.path.length - b.path.length);
-  if (allHits.length === 0) {
+  const hits = nodes(doc, whereJsonQuery).sort((a, b) => a.path.length - b.path.length);
+  if (hits.length === 0) {
     return { anyHit: false, result: src };
   }
 
-  // filter out sub-hits (only consider highest hit)
-  const hits: { path: JsonPath, value: any }[] = [];
-  for (const hit of allHits) {
-    if (hits.every(existingHit => !IsPrefix(existingHit.path, hit.path))) {
-      hits.push(hit);
-    }
-  }
-
   // process
   let ast: YAMLNode = CloneAst(src.ReadYamlAst());
   const mapping = IdentitySourceMapping(src.key, ast).filter(m => !hits.some(hit => IsPrefix(hit.path, (m.generated as any).path)));

src/autorest-core/lib/pipeline/pipeline.ts

@@ -294,8 +294,8 @@ export async function RunPipeline(configView: ConfigurationView, fileSystem: IFi
 
   // dynamically loaded, auto-discovered plugins
   const __extensionExtension: { [pluginName: string]: AutoRestExtension } = {};
-  for (const useExtension of configView.UseExtensions) {
-    const extension = await GetExtension(useExtension.fullyQualified);
+  for (const useExtensionQualifiedName of configView.GetEntry("used-extension" as any) || []) {
+    const extension = await GetExtension(useExtensionQualifiedName);
     for (const plugin of await extension.GetPluginNames(configView.CancellationToken)) {
       plugins[plugin] = CreatePluginExternal(extension, plugin);
       __extensionExtension[plugin] = extension;

src/autorest-core/lib/ref/jsonpath.ts

@@ -8,10 +8,10 @@ import * as jsonpath from "jsonpath";
 
 // patch in smart filter expressions
 const handlers = (jsonpath as any).handlers;
-const filterExpressionHandler = function (component: any, partial: any, count: any) {
-  var src = component.expression.value.slice(1);
+handlers.register("subscript-descendant-filter_expression", function (component: any, partial: any, count: any) {
+  const src = component.expression.value.slice(1);
 
-  var passable = function (key: any, value: any) {
+  const passable = function (key: any, value: any) {
     try {
       return safeEval(src.replace(/\@/g, "$$$$"), { "$$": value });
     } catch (e) {
@@ -20,9 +20,20 @@ const filterExpressionHandler = function (component: any, partial: any, count: a
   }
 
   return eval("this").traverse(partial, null, passable, count);
-};
-handlers.register("subscript-descendant-filter_expression", filterExpressionHandler);
-handlers.register("subscript-child-filter_expression", filterExpressionHandler);
+});
+handlers.register("subscript-child-filter_expression", function (component: any, partial: any, count: any) {
+  const src = component.expression.value.slice(1);
+
+  const passable = function (key: any, value: any) {
+    try {
+      return safeEval(src.replace(/\@/g, "$$$$"), { "$$": value });
+    } catch (e) {
+      return false;
+    }
+  }
+
+  return eval("this").descend(partial, null, passable, count);
+});
 // patch end
 
 export type JsonPathComponent = jsonpath.PathComponent;
@@ -43,7 +54,11 @@ export function paths<T>(obj: T, jsonQuery: string): JsonPath[] {
 export function nodes<T>(obj: T, jsonQuery: string): { path: JsonPath, value: any }[] {
   // jsonpath only accepts objects
   if (obj instanceof Object) {
-    return jsonpath.nodes(obj, jsonQuery).map(x => { return { path: x.path.slice(1), value: x.value }; });
+    let result = jsonpath.nodes(obj, jsonQuery).map(x => { return { path: x.path.slice(1), value: x.value }; });
+    const comp = (a: string, b: string) => a < b ? -1 : (a > b ? 1 : 0);
+    result = result.sort((a, b) => comp(JSON.stringify(a.path), JSON.stringify(b.path)));
+    result = result.filter((x, i) => i === 0 || JSON.stringify(x.path) !== JSON.stringify(result[i - 1].path));
+    return result;
   } else {
     return matches(jsonQuery, []) ? [{ path: [], value: obj }] : [];
   }

src/autorest-core/package.json

@@ -43,7 +43,8 @@
     "mocha": "^4.0.1",
     "shx": "0.2.2",
     "mocha-typescript": "^1.1.7",
-    "static-link": "^0.2.2"
+    "static-link": "^0.2.2",
+    "vscode-jsonrpc": "^3.3.1"
   },
   "static-link": {
     "entrypoints": [],

src/autorest-core/post-install.js

@@ -1,4 +1,9 @@
 try {
+    for( const each in process.env) {
+      if( each.startsWith("npm_config") || each.startsWith("npm_lifecycle") || each.startsWith("npm_package") ) {
+        delete process.env[each];
+      }
+    }
     // if this installed for development
     // static-link should be installed. 
     // if we require it, it should build the static fs.

src/autorest-core/resources/default-configuration.md

@@ -21,12 +21,12 @@ use-extension:
 
 ``` yaml $(csharp)
 use-extension:
-  "@microsoft.azure/autorest.csharp": "~2.1.0"
+  "@microsoft.azure/autorest.csharp": "~2.2.0"
 ```
 
 ``` yaml $(jsonrpcclient)
 use-extension:
-  "@microsoft.azure/autorest.csharp": "~2.1.0"
+  "@microsoft.azure/autorest.csharp": "~2.2.0"
 ```
 
 ``` yaml $(go)
@@ -153,9 +153,12 @@ pipeline:
   openapi-document/openapi-document-converter:
     input: swagger-document/identity
     output-artifact: openapi-document
-  openapi-document/identity:
+  openapi-document/transform:
     input: openapi-document-converter
     output-artifact: openapi-document
+  openapi-document/identity:
+    input: transform
+    output-artifact: openapi-document
   openapi-document/emitter:
     input: identity
     scope: scope-openapi-document/emitter
@@ -185,7 +188,8 @@ scope-cm/emitter:
 
 #### Polyfills
 
-Support for `additionalProperties: true/false` in `definitions` section
+
+##### `additionalProperties: true/false` in definitions section
 
 ``` yaml
 directive:
@@ -198,6 +202,18 @@ directive:
   reason: polyfill
 ```
 
+##### Reproduce old buggy behavior of ignoring `required`ness of properties in nested schemas (anything outside `definitions` section)
+See https://github.com/Azure/autorest/issues/2688
+
+``` yaml $(ignore-nested-required)
+directive:
+- from: openapi-document
+  where: $..*[?(Array.isArray(@.required) && @.properties)]
+  transform: |
+    if ($path.length > 3) delete $.required;
+  reason: see issue https://github.com/Azure/autorest/issues/2688
+```
+
 #### Validation
 
 ``` yaml

src/autorest/post-install.js

@@ -1,6 +1,10 @@
 try {
+  for( const each in process.env) {
+    if( each.startsWith("npm_config") || each.startsWith("npm_lifecycle") || each.startsWith("npm_package") ) {
+      delete process.env[each];
+    }
+  }
   // if this installed for development
-
   // static-link should be installed. 
   // if we require it, it should build the static fs.
   require("./node_modules/static-link/dist/static-link");