Verify autorest in pipeline (#16667)

#13724
2021-04-26 15:53:22 -04:00 · 2021-04-26 15:53:22 -04:00 · ec58f5b82c
--- a/eng/.docsettings.yml
+++ b/eng/.docsettings.yml
@ -88,24 +88,15 @@ known_content_issues:
  - ['sdk/eventgrid/azure-eventgrid/README.md', '#4554']
  - ['sdk/graphrbac/azure-graphrbac/README.md', '#4554']
  - ['sdk/loganalytics/azure-loganalytics/README.md', '#4554']
-  - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
  - ['sdk/servicebus/azure-servicebus/README.md', '#4554']
-  - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
  - ['sdk/servicebus/azure-servicebus/tests/perf_tests/README.md', '#4554']
  - ['sdk/servicefabric/azure-servicefabric/README.md', '#4554']
  - ['sdk/storage/azure-storage-nspkg/README.md', '#4554']
-  - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']
-  - ['sdk/storage/azure-storage-file-datalake/swagger/README.md', '#4554']
-  - ['sdk/storage/azure-storage-file-share/swagger/README.md', '#4554']
  - ['sdk/storage/azure-storage/README.md', '#4554']
  - ['sdk/storage/azure-storage-nspkg/README.rst', '#4554']
-  - ['sdk/storage/azure-storage-queue/swagger/README.md', '#4554']
  - ['sdk/storage/README.md', '#4554']
  - ['sdk/textanalytics/azure-ai-textanalytics/samples/README.md', '#4554']
-  - ['sdk/monitor/azure-monitor-opentelemetry-exporter/swagger/README.md', '#4554']
  - ['sdk/monitor/azure-monitor-opentelemetry-exporter/README.md', '#4554']
-  - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
-  - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
  - ['sdk/media/azure-media-nspkg/README.md', '#4554']
  - ['sdk/containerregistry/azure-containerregistry/swagger/README.md', '#4554']
  - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
@ -156,6 +147,18 @@ known_content_issues:
  - ['doc/dev/mgmt/README.md', 'dev readme']
  - ['doc/dev/README.md', dev readme']
  - ['doc/README.md', 'dev readme']
+
+  # swagger files
+  - ['sdk/schemaregistry/azure-schemaregistry/swagger/README.md', '#4554']
+  - ['sdk/servicebus/azure-servicebus/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-blob/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-file-datalake/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-file-share/swagger/README.md', '#4554']
+  - ['sdk/storage/azure-storage-queue/swagger/README.md', '#4554']
+  - ['sdk/monitor/azure-monitor-opentelemetry-exporter/swagger/README.md', '#4554']
+  - ['sdk/digitaltwins/azure-digitaltwins-core/swagger/README.md', '#4554']
+  - ['sdk/textanalytics/azure-ai-textanalytics/swagger/README.md', '#4554']
+  - ['sdk/appconfiguration/azure-appconfiguration/swagger/README.md', '#4554']
 package_indexing_exclusion_list:
  - 'azure-sdk-tools'
  - 'azure-template'
--- a/eng/autorest_req.txt
+++ b/eng/autorest_req.txt
@ -4,5 +4,6 @@ pytest-cov==2.8.1
 pytest-asyncio==0.12.0; python_version >= '3.5'
 isodate==0.6.0
 msrest==0.6.14
-aiohttp==3.6.2
-wheel==0.34.2
+wheel==0.34.2
+GitPython==3.1.14
+aiohttp==3.6.2; python_version >= '3.6'
--- a/eng/pipelines/templates/jobs/ci.yml
+++ b/eng/pipelines/templates/jobs/ci.yml
@ -43,6 +43,9 @@ parameters:
  - name: MatrixReplace
    type: object
    default: []
+  - name: VerifyAutorest
+    type: boolean
+    default: false

 jobs:
  - job: 'Build'
@ -100,6 +103,7 @@ jobs:
        AdditionalTestArgs: '--wheel_dir="$(Build.ArtifactStagingDirectory)"'
        TestPipeline: ${{ parameters.TestPipeline }}
        Artifacts: ${{ parameters.Artifacts }}
+        VerifyAutorest: ${{ parameters.VerifyAutorest }}

  - template: /eng/common/pipelines/templates/jobs/archetype-sdk-tests-generate.yml
    parameters:
--- a/eng/pipelines/templates/stages/archetype-sdk-client.yml
+++ b/eng/pipelines/templates/stages/archetype-sdk-client.yml
@ -57,6 +57,9 @@ parameters:
 - name: MatrixReplace
  type: object
  default: []
+- name: VerifyAutorest
+  type: boolean
+  default: false

 stages:
  - stage: Build
@ -88,6 +91,7 @@ stages:
          - OsVmImage=macOS-10.15/MMSUbuntu18.04
          - ${{ each replacement in parameters.MatrixReplace }}:
            - ${{ replacement }}
+        VerifyAutorest: ${{ parameters.VerifyAutorest }}


  # The Prerelease and Release stages are conditioned on whether we are building a pull request and the branch.
@ -115,4 +119,4 @@ stages:
        ArtifactName: conda
        ${{ if eq(parameters.ServiceDirectory, 'template') }}:
          TestPipeline: true
-        
+
--- a/eng/pipelines/templates/steps/analyze.yml
+++ b/eng/pipelines/templates/steps/analyze.yml
@ -5,6 +5,7 @@ parameters:
  AdditionalTestArgs: ''
  Artifacts: []
  TestPipeline: false
+  VerifyAutorest: false

 steps:
  - template: /eng/pipelines/templates/steps/analyze_dependency.yml
@ -57,6 +58,11 @@ steps:
    parameters:
      SourceDirectory: $(Build.SourcesDirectory)

+  - template: ../steps/verify-autorest.yml
+    parameters:
+      ServiceDirectory: ${{ parameters.ServiceDirectory }}
+      VerifyAutorest: ${{ parameters.VerifyAutorest }}
+
  - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
    # ComponentGovernance is currently unable to run on pull requests of public projects. Running on non-PR
    # builds should be sufficient.
--- a/eng/pipelines/templates/steps/verify-autorest.yml
+++ b/eng/pipelines/templates/steps/verify-autorest.yml
@ -0,0 +1,66 @@
+parameters:
+  - name: ServiceDirectory
+    type: string
+  - name: NodeVersion
+    type: string
+    default: '12.x'
+  - name: PythonVersion
+    type: string
+    default: '$(PythonVersion)'
+  - name: auto_rest_clone_url
+    type: string
+    default: 'https://github.com/Azure/autorest.python.git'
+  - name: VerifyAutorest
+    type: boolean
+    default: false
+
+steps:
+  - ${{if eq(parameters.VerifyAutorest, 'true')}}:
+    - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+    - task: UsePythonVersion@0
+      displayName: Use
+      inputs:
+        versionSpec: ${{ parameters.PythonVersion }}
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: ${{ parameters.NodeVersion }}
+
+    - script: |
+        npm install -g autorest
+        autorest --help
+      displayName: "Install autorest"
+
+    - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+      parameters:
+        Paths:
+          - "/*"
+        Repositories:
+          - Name: "Azure/autorest.python"
+            Commitish: "master"
+            WorkingDirectory: "$(Build.SourcesDirectory)/autorest.python"
+        SkipDefaultCheckout: true
+
+    - script: |
+        python --version
+        pip install -r eng/autorest_req.txt
+        cd autorest.python
+        npm install
+      displayName: 'Prepare Environment'
+
+    - task: PythonScript@0
+      displayName: 'Verify autorest'
+      inputs:
+        scriptPath: 'scripts/devops_tasks/verify_autorest.py'
+        arguments: >-
+          --service_directory="${{ parameters.ServiceDirectory }}"
+
+    - ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}:
+      - template: /eng/common/pipelines/templates/steps/create-pull-request.yml
+    #   # displayName: Create PR for ${{ ServiceDirectory }}
+    #   condition: and(ne(variables['Build.Reason'],'PullRequest'), eq(variables['System.TeamProject'], 'internal'))
+        parameters:
+          CommitMsg: "Regenerated code from nightly builds"
+          PRTitle: "Automated autorest generation"
+          PRBranchName: 'autorest-${{ ServiceDirectory }}'
--- a/scripts/devops_tasks/verify_autorest.py
+++ b/scripts/devops_tasks/verify_autorest.py
@ -0,0 +1,108 @@
+#!/usr/bin/env python
+
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+import argparse
+import os
+import logging
+import sys
+
+from common_tasks import run_check_call
+
+logging.getLogger().setLevel(logging.INFO)
+
+root_dir = os.path.abspath(os.path.join(os.path.abspath(__file__), "..", "..", ".."))
+sdk_dir = os.path.join(root_dir, "sdk")
+
+SWAGGER_FOLDER = "swagger"
+
+
+def run_autorest(service_dir):
+    logging.info("Running autorest for {}".format(service_dir))
+
+    service_dir = os.path.join(sdk_dir, service_dir)
+
+    swagger_folders = find_swagger_folders(service_dir)
+
+    for working_dir in swagger_folders:
+        os.chdir(working_dir)
+        f = os.path.abspath(os.path.join(working_dir, "README.md"))
+        if os.path.exists(f):
+            reset_command = ["autorest", "--reset"]
+            run_check_call(reset_command, root_dir)
+
+            command = ["autorest", "--python", f, "--verbose"]
+            logging.info("Command: {}\nLocation: {}\n".format(command, working_dir))
+            run_check_call(command, working_dir)
+    return swagger_folders
+
+
+def find_swagger_folders(directory):
+    logging.info("Searching for swagger files in: {}".format(directory))
+
+    ret = []
+    for root, subdirs, files in os.walk(directory):
+        for d in subdirs:
+            if d == SWAGGER_FOLDER:
+                if os.path.exists(os.path.join(root, d, "README.md")):
+                    ret.append(os.path.join(root, d))
+
+    logging.info("Found swagger files at: {}".format(ret))
+    return ret
+
+
+def check_diff(folder):
+    # We don't care about changes to txt files (dev_requirements change)
+    run_check_call(["git", "status"], sdk_dir, always_exit=False)
+
+    command = [
+        "git",
+        "checkout",
+        "--",
+        "**/*.txt",
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+
+    # Remove the whl dirs
+    command = [
+        "rm",
+        "-r",
+        "**/.tmp_whl_dir/"
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+
+    # Next we need to move the autorest and _tox_logs directories and then replace them
+
+    dir_changed = folder.split("/")[:-2]
+    command = [
+        "git",
+        "diff",
+        "--exit-code",
+        "{}".format("/".join(dir_changed)),
+    ]
+    result = run_check_call(command, sdk_dir, always_exit=False)
+    if result:
+        command = ["git", "status"]
+        run_check_call(command, root_dir)
+        raise ValueError(
+            "Found difference between re-generated code and current commit. Please re-generate with the latest autorest."
+        )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Run autorest to verify generated code."
+    )
+    parser.add_argument(
+        "--service_directory", help="Directory of the package being tested"
+    )
+
+    args = parser.parse_args()
+    folders = run_autorest(args.service_directory)
+
+    if len(folders):
+        for folder in folders:
+            check_diff(folder)
--- a/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/_configuration.py
+++ b/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/_configuration.py
@ -51,7 +51,7 @@ class AzureAppConfigurationConfiguration(Configuration):
        self.endpoint = endpoint
        self.sync_token = sync_token
        self.api_version = "1.0"
-        self.credential_scopes = kwargs.pop('credential_scopes', [])
+        self.credential_scopes = kwargs.pop('credential_scopes', ['https://dev.azuresynapse.net/.default'])
        kwargs.setdefault('sdk_moniker', 'appconfiguration/{}'.format(VERSION))
        self._configure(**kwargs)

@ -69,7 +69,5 @@ class AzureAppConfigurationConfiguration(Configuration):
        self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs)
        self.redirect_policy = kwargs.get('redirect_policy') or policies.RedirectPolicy(**kwargs)
        self.authentication_policy = kwargs.get('authentication_policy')
-        if not self.credential_scopes and not self.authentication_policy:
-            raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs")
        if self.credential and not self.authentication_policy:
            self.authentication_policy = policies.BearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs)
--- a/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/aio/_configuration.py
+++ b/sdk/appconfiguration/azure-appconfiguration/azure/appconfiguration/_generated/aio/_configuration.py
@ -48,7 +48,7 @@ class AzureAppConfigurationConfiguration(Configuration):
        self.endpoint = endpoint
        self.sync_token = sync_token
        self.api_version = "1.0"
-        self.credential_scopes = kwargs.pop('credential_scopes', [])
+        self.credential_scopes = kwargs.pop('credential_scopes', ['https://dev.azuresynapse.net/.default'])
        kwargs.setdefault('sdk_moniker', 'appconfiguration/{}'.format(VERSION))
        self._configure(**kwargs)

@ -65,7 +65,5 @@ class AzureAppConfigurationConfiguration(Configuration):
        self.custom_hook_policy = kwargs.get('custom_hook_policy') or policies.CustomHookPolicy(**kwargs)
        self.redirect_policy = kwargs.get('redirect_policy') or policies.AsyncRedirectPolicy(**kwargs)
        self.authentication_policy = kwargs.get('authentication_policy')
-        if not self.credential_scopes and not self.authentication_policy:
-            raise ValueError("You must provide either credential_scopes or authentication_policy as kwargs")
        if self.credential and not self.authentication_policy:
            self.authentication_policy = policies.AsyncBearerTokenCredentialPolicy(self.credential, *self.credential_scopes, **kwargs)
--- a/sdk/appconfiguration/azure-appconfiguration/swagger/README.md
+++ b/sdk/appconfiguration/azure-appconfiguration/swagger/README.md
@ -15,4 +15,4 @@ enable-xml: true
 vanilla: true
 add-credentials: true
 credential-scopes: https://dev.azuresynapse.net/.default
-```
+```
--- a/sdk/appconfiguration/ci.yml
+++ b/sdk/appconfiguration/ci.yml
@ -31,8 +31,9 @@ extends:
  template: ../../eng/pipelines/templates/stages/archetype-sdk-client.yml
  parameters:
    ServiceDirectory: appconfiguration
+    VerifyAutorest: true
    Artifacts:
    - name: azure-appconfiguration
      safeName: azureappconfiguration
    - name: azure-mgmt-appconfiguration
-      safeName: azuremgmtappconfiguration
+      safeName: azuremgmtappconfiguration