Merged PR 5298: Added semantic versioning and nuget upload support

Added semantic versioning and nuget upload support

CI/AzureUpload.stage_template.yaml

@@ -1,104 +1,70 @@
 parameters:
   agent_pool: "" # vs2017-win2016 (Linux not support in some task)
-  project: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts)
+  azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts)
-  definition: "" # 717 (The definition ID of the build pipeline)
+  azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline)
   subscription: "" # AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)
   storage: "" # featurizersbuild (The name of storage account in Azure)
   containerName: "" # Archive (The name of the container to which the files will be copied)
 
 stages:
   - stage: AzureUpload_Stage
+    displayName: "Upload to Azure"
+    dependsOn: [] # No dependencies
 
     jobs:
       - job: AzureUpload_Job
+        timeoutInMinutes: 240
+
+        displayName: "Upload"
 
         pool:
           name: Azure Pipelines
           demands: azureps
           vmImage: "${{ parameters.agent_pool }}"
 
-        # This build task will try to download artifacts from the triggering build.
-        # If there is no triggering build from the specified pipeline, it will download
-        # artifacts from the build specified in the options below
         steps:
+          - template: PublishInitialize.steps_template.yaml
+            parameters:
+              azure_dev_ops_project_id: ${{ parameters.azure_dev_ops_project_id }}
+              azure_dev_ops_build_definition_id: ${{ parameters.azure_dev_ops_build_definition_id }}
+
+          - task: CopyFiles@2
+            displayName: Copying Documentation
+            inputs:
+              SourceFolder: "$(Pipeline.Workspace)/Documentation"
+              Contents: "**"
+              TargetFolder: "$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Documentation/"
+
+          - task: CopyFiles@2
+            displayName: Copying Packages
+            inputs:
+              SourceFolder: "$(Pipeline.Workspace)/Package"
+              Contents: "**"
+              TargetFolder: "$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Package/"
+
           - task: PythonScript@0
-            displayName: 'Check Variables'
+            displayName: "Trim Directory Name"
             inputs:
               scriptSource: inline
               script: |
-               import sys
+                import os
+                import sys
 
-               triggeredBuildNumber = sys.argv[1]
+                basedir = sys.argv[1]
-               buildId = sys.argv[2]
+                for fn in os.listdir(basedir):
-
+                  if not os.path.isdir(os.path.join(basedir, fn)):
-               # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of
+                    continue # Not a directory
-               # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value.
+                  if fn == "{}(Build.TriggeredBy.BuildNumber)".format("$"):
-               # Therefore, a non-standard comparison is necessary in this case
+                    os.rename(os.path.join(basedir, fn), os.path.join(basedir, 'Independent Publish_' + sys.argv[2]))
-               if triggeredBuildNumber == "{}(Build.TriggeredBy.BuildNumber)".format("$"):
+                  else:
-                 if buildId == "{}(buildId)".format("$"):
+                    os.rename(os.path.join(basedir, fn), os.path.join(basedir, fn[-13:]))
-                   raise ValueError('Please specify variable buildId since this is a publish with non-triggered build. e.g. set buildId = 104648')
-              arguments: '$(Build.TriggeredBy.BuildNumber) $(buildId)'
-
-          - task: DownloadPipelineArtifact@2
-            condition: eq(variables['Build.Reason'], 'Manual')
-            displayName: 'Download Pipeline Artifact for Mannual build'
-            inputs:
-              buildType: 'specific'
-              project: "${{ parameters.project }}"
-              definition: "${{ parameters.definition }}"
-              specificBuildWithTriggering: false
-              buildVersionToDownload: specific
-              pipelineId: '$(buildId)'
-              artifactName: ''
-              targetPath: '$(Pipeline.Workspace)'
-
-          - task: DownloadPipelineArtifact@2
-            condition: eq(variables['Build.Reason'], 'BuildCompletion')
-            displayName: 'Download Pipeline Artifact for Build triggered by BuildCompletion'
-            inputs:
-              buildType: 'specific'
-              project: "${{ parameters.project }}"
-              definition: "${{ parameters.definition }}"
-              specificBuildWithTriggering: true
-              buildVersionToDownload: latestFromBranch
-              artifactName: ''
-              targetPath: '$(Pipeline.Workspace)'
-
-          - task: CopyFiles@2
-            inputs:
-              SourceFolder: '$(Pipeline.Workspace)/Documentation'
-              Contents: '**'
-              TargetFolder: '$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Documentation/'
-
-          - task: CopyFiles@2
-            inputs:
-              SourceFolder: '$(Pipeline.Workspace)/Package'
-              Contents: '**'
-              TargetFolder: '$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Package/'
-
-          - task: PythonScript@0
-            displayName: 'Trim Directory Name'
-            inputs:
-              scriptSource: inline
-              script: |
-               import os
-               import sys
-
-               basedir = sys.argv[1]
-               for fn in os.listdir(basedir):
-                 if not os.path.isdir(os.path.join(basedir, fn)):
-                   continue # Not a directory
-                 if fn == "{}(Build.TriggeredBy.BuildNumber)".format("$"):
-                   os.rename(os.path.join(basedir, fn), os.path.join(basedir, 'Independent Publish_' + sys.argv[2]))
-                 else:
-                   os.rename(os.path.join(basedir, fn), os.path.join(basedir, fn[-13:]))
               arguments: '$(Pipeline.Workspace)\upload $(buildId)'
 
           - task: AzureFileCopy@2
-            displayName: 'AzureBlob File Copy'
+            displayName: "AzureBlob File Copy"
             inputs:
-              SourcePath: '$(Pipeline.Workspace)/upload'
+              SourcePath: "$(Pipeline.Workspace)/upload"
               azureSubscription: "${{ parameters.subscription }}"
-              Destination: 'AzureBlob'
+              Destination: "AzureBlob"
               storage: "${{ parameters.storage }}"
               ContainerName: "${{ parameters.containerName }}"

CI/BuildAndTest.stage_template.yaml

@@ -2,6 +2,7 @@ parameters:
   operating_system: "" # Windows|Linux|MacOS
   enable_code_coverage: "" # True|False
   esrp_connected_service_name: "None" # <Name>|None
+  dependencies: []
 
   configuration: ""
   agent_pool: ""
@@ -22,7 +23,7 @@ parameters:
 stages:
   - stage: BuildAndTest_${{ parameters.operating_system }}_${{ parameters.configuration }}_Stage
     displayName: "${{ parameters.operating_system }} - ${{ parameters.configuration }}: "
-    dependsOn: [] # No dependencies
+    dependsOn: ${{ parameters.dependencies }}
 
     jobs:
       - template: BuildAndTest.job_template.yaml
@@ -42,6 +43,7 @@ stages:
 
             - script: |-
                 echo "esrp_connected_service_name - ${{ parameters.esrp_connected_service_name }}"
+                echo "dependencies - ${{ join(', ', parameters.dependencies) }}"
                 echo "configuration - ${{ parameters.configuration }}"
                 echo "agent_pool - ${{ parameters.agent_pool }}"
                 echo "agent_pool_container - ${{ parameters.agent_pool_container }}"
@@ -62,8 +64,25 @@ stages:
 
             # TODO: Code formatting
 
+            # Dynamically create the builder args
+            - task: PythonScript@0
+              displayName: "[IMPL] Create Builder Args"
+              inputs:
+                scriptSource: inline
+                script: |-
+                  if "$(azure_is_release_build)" == "1":
+                      args = "/release_build"
+                  else:
+                      args = "/prerelease_build_name=$(azure_build_name)"
+
+                  args = '"/custom_build_args=src/FeaturizerPrep/SharedLibrary:{} /no_build_info"'.format(args)
+
+                  print("azure_builder_args = {}".format(args))
+                  print("##vso[task.setvariable variable=azure_builder_args]{}".format(args))
+              timeoutInMinutes: 180
+
             - script: |-
-                $(azure_activate_script) ${{ parameters.configuration }} && Builder$(azure_script_extension) Execute . "$(azure_artifacts_directory)/Builder" /verbose
+                $(azure_activate_script) ${{ parameters.configuration }} && Builder$(azure_script_extension) Execute . "$(azure_artifacts_directory)/Builder" /verbose $(azure_builder_args)
               displayName: "<Builder>"
               timeoutInMinutes: 180
 
@@ -119,6 +138,46 @@ stages:
                           }
                       ]
 
+            - task: PythonScript@0
+              displayName: "[IMPL] Persist Product Version Information"
+              inputs:
+                scriptSource: inline
+                script: |-
+                  import json
+                  import os
+
+                  from collections import OrderedDict
+
+                  values = OrderedDict()
+
+                  builder_dir = os.path.join(r"$(azure_artifacts_directory)", "Builder")
+
+                  # Get the release directory
+                  for potential_dirname in os.listdir(builder_dir):
+                      potential_fullpath = os.path.join(builder_dir, potential_dirname)
+                      if not os.path.isdir(potential_fullpath):
+                          continue
+
+                      file_attributes_filename = os.path.join(potential_fullpath, "Microsoft MLFeaturizers.FileAttributes.json")
+                      if not os.path.isfile(file_attributes_filename):
+                          raise Exception("'{}' does not exist".format(file_attributes_filename))
+
+                      with open(file_attributes_filename) as f:
+                          file_attributes_data = json.load(f)
+
+                      assert "product_version_full" in file_attributes_data, file_attributes_data
+                      values["product_version"] = file_attributes_data["product_version_full"]
+
+                      break
+
+                  for k, v in values.items():
+                      print("{} = {}".format(k, v))
+
+                  with open(os.path.join(builder_dir, "product_version_information.json"), "w") as f:
+                      json.dump(values, f)
+              condition: succeededOrFailed()
+              timeoutInMinutes: 180
+
             - task: PublishPipelineArtifact@0
               displayName: "Publish Artifacts"
               inputs:

CI/CI.yaml

@@ -12,13 +12,29 @@ resources:
       image: universal_linux
       endpoint: featurizersbuild
 
+# Note that a variable set in this file will always override a value set within the editor.
+# Therefore, do not set the default value to False here but instead do it in an initialization
+# script.
+#
+#   variables:
+#     release_build: False                  # Set to true to create a release build
+#     prerelease_build_name: ""             # Set to a value (e.g. "preview.1" to create a preview build)
+
 stages:
+  - template: Prerequisites.stage_template.yaml
+    parameters:
+      agent_pool: windows-2019
+      release_build: $(release_build)
+      prerelease_build_name: $(prerelease_build_name)
+
   # Windows/clang/x64 [Official Build]
   - template: BuildAndTest.stage_template.yaml
     parameters:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x64
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: True
       esrp_connected_service_name: "ESRP CodeSigning Connection"
 
@@ -28,6 +44,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x86
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       esrp_connected_service_name: "ESRP CodeSigning Connection"
 
@@ -38,6 +56,8 @@ stages:
   # TODO:     agent_pool_container: universal_linux
   # TODO:     operating_system: Linux
   # TODO:     configuration: universal_linux
+  # TODO:     dependencies:
+  # TODO:       - Prerequisites_Stage
   # TODO:     enable_code_coverage: False
   # TODO:     esrp_connected_service_name: "ESRP CodeSigning Connection"
 
@@ -47,6 +67,8 @@ stages:
       agent_pool: macOS-10.14
       operating_system: MacOS
       configuration: system_compiler
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       esrp_connected_service_name: "ESRP CodeSigning Connection"
 
@@ -56,6 +78,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x64_MSVC
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing.
 
@@ -65,6 +89,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x86_MSVC
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing.
 
@@ -74,6 +100,8 @@ stages:
       agent_pool: ubuntu-16.04
       operating_system: Linux
       configuration: x64
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing.
 
@@ -83,6 +111,8 @@ stages:
       agent_pool: ubuntu-16.04
       operating_system: Linux
       configuration: system_compiler
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
       # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing.
 

CI/Initialize.steps_template.yaml

@@ -4,79 +4,73 @@ parameters:
     enable_code_coverage: "" # True|False
 
 steps:
-    # These steps are an ugly hack to set Azure DevOps variables conditionally. It really seems like there should be a better way to do this.
+    - task: DownloadPipelineArtifact@2
-    - ${{ if eq(parameters.operating_system, 'Windows') }}:
+      displayName: "[IMPL] Download Global Build Information"
-          - script: |-
+      inputs:
-                echo ##vso[task.setvariable variable=azure_activate_script]call Activate.cmd
+          artifactName: "Prerequisites"
-                echo ##vso[task.setvariable variable=azure_script_extension].cmd
+          targetPath: "$(Pipeline.Workspace)/Prerequisites"
-                echo ##vso[task.setvariable variable=azure_display_all_environment_vars]set
+      timeoutInMinutes: 180
-                echo ##vso[task.setvariable variable=azure_bootstrap_command]bootstrap.cmd
-                echo ##vso[task.setvariable variable=azure_agent_temp_directory]%AGENT_TEMPDIRECTORY%
-                echo ##vso[task.setvariable variable=azure_artifacts_directory]%BUILD_ARTIFACTSTAGINGDIRECTORY%
-                echo ##vso[task.setvariable variable=azure_single_threaded_build_arg]
-            displayName: "[IMPL] Set Environment-Specific Variables (Standard)"
 
-          # Dynamic Code Coverage Value
+    - task: PythonScript@0
-          - ${{ if eq(parameters.enable_code_coverage, 'true') }}:
+      displayName: "[IMPL] Populate Stage Variables"
-                - script: |-
+      inputs:
-                      echo ##vso[task.setvariable variable=azure_code_coverage_arg]/code_coverage
+          scriptSource: inline
-                  displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)"
+          script: |-
+              import os
+              import json
+              from collections import OrderedDict
 
-          - ${{ if not(eq(parameters.enable_code_coverage, 'true')) }}:
+              values = OrderedDict()
-                - script: |-
-                      echo ##vso[task.setvariable variable=azure_code_coverage_arg]
-                  displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)"
 
-    - ? ${{ if or(eq(parameters.operating_system, 'Linux'), eq(parameters.operating_system, 'MacOS')) }}
+              # Create standard values
-      : - ${{ if eq(parameters.configuration, 'universal_linux') }}:
+              operating_system = "${{ parameters.operating_system }}"
-              - script: |-
-                    sudo yum install -y git
-                displayName: "[IMPL] Install git"
 
-        - script: |-
+              if operating_system == "Windows":
-              echo "##vso[task.setvariable variable=azure_activate_script]source ./Activate.sh"
+                  values["azure_activate_script"] = "call Activate.cmd"
-              echo "##vso[task.setvariable variable=azure_script_extension].sh"
+                  values["azure_script_extension"] = ".cmd"
-              echo "##vso[task.setvariable variable=azure_display_all_environment_vars]export"
+                  values["azure_display_all_environment_vars"] = "set"
-              echo "##vso[task.setvariable variable=azure_bootstrap_command]sudo ./bootstrap.sh"
+                  values["azure_bootstrap_command"] = "bootstrap.cmd"
-              echo "##vso[task.setvariable variable=azure_agent_temp_directory]${AGENT_TEMPDIRECTORY}"
+                  values["azure_single_threaded_build_arg"] = ""
-              echo "##vso[task.setvariable variable=azure_artifacts_directory]${BUILD_ARTIFACTSTAGINGDIRECTORY}"
+              elif operating_system in ["Linux", "MacOS"]:
-          displayName: "[IMPL] Set Environment-Specific Variables (Standard)"
+                  values["azure_activate_script"] = "source ./Activate.sh"
+                  values["azure_script_extension"] = ".sh"
+                  values["azure_display_all_environment_vars"] = "export"
+                  values["azure_bootstrap_command"] = "sudo ./bootstrap.sh"
 
-        # Dynamic Code Coverage Value
+                  # The MacOS compiler hangs when invoking the compiler in parallel. As a result, force single threaded
-        - ${{ if eq(parameters.enable_code_coverage, 'true') }}:
+                  # execution.
-              - script: |-
+                  if operating_system == "MacOS":
-                    echo "##vso[task.setvariable variable=azure_code_coverage_arg]/code_coverage"
+                      values["azure_single_threaded_build_arg"] = "/single_threaded"
-                displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)"
+                  else:
+                      values["azure_single_threaded_build_arg"] = ""
 
-        - ${{ if not(eq(parameters.enable_code_coverage, 'true')) }}:
+              values["azure_agent_temp_directory"] = os.getenv("AGENT_TEMPDIRECTORY")
-              - script: |-
+              values["azure_artifacts_directory"] = os.getenv("BUILD_ARTIFACTSTAGINGDIRECTORY")
-                    echo "##vso[task.setvariable variable=azure_code_coverage_arg]"
-                displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)"
 
-        # Dynamic Single Threaded Build Value.
+              if "${{ parameters.enable_code_coverage }}".lower() in ["true", "yes"]:
-        #   The MacOS compiler hangs when invoking the compiler in parallel. As a result, force single threaded
+                  values["azure_code_coverage_arg"] = "/code_coverage"
-        #   execution.
+              else:
-        - ${{ if eq(parameters.operating_system, 'MacOS') }}:
+                  values["azure_code_coverage_arg"] = ""
-              - script: |-
-                    echo "##vso[task.setvariable variable=azure_single_threaded_build_arg]/single_threaded"
-                displayName: "[IMPL] Set Environment-Specific Variables (Single Threaded Build)"
 
-        - ${{ if not(eq(parameters.operating_system, 'MacOS')) }}:
+              # Load global values previously generated
-              - script: |-
+              json_filename = os.path.join(r"$(Pipeline.Workspace)", "Prerequisites", "global_build_information.json")
-                    echo "##vso[task.setvariable variable=azure_single_threaded_build_arg]"
+              assert os.path.isfile(json_filename), json_filename
-                displayName: "[IMPL] Set Environment-Specific Variables (Single Threaded Build)"
 
-    - script: |-
+              with open(json_filename) as f:
-          echo "operating_system - ${{ parameters.operating_system }}"
+                  content = json.load(f)
-          echo "configuration - ${{ parameters.configuration }}"
-          echo "enable_code_coverage - ${{ parameters.enable_code_coverage }}"
 
-          echo "activate_script - $(azure_activate_script)"
+              for k, v in content.items():
-          echo "script_extension - $(azure_script_extension)"
+                  k = "azure_{}".format(k)
-          echo "display_all_environment_vars - $(azure_display_all_environment_vars)"
+
-          echo "bootstrap_command - $(azure_bootstrap_command)"
+                  if isinstance(v, bool):
-          echo "agent_temp_directory - $(azure_agent_temp_directory)"
+                      v = 1 if v else 0
-          echo "artifacts_directory - $(azure_artifacts_directory)"
+                  elif v is None:
-          echo "code_coverage_arg - $(azure_code_coverage_arg)"
+                      v = ""
-          echo "single_threaded_arg - $(azure_single_threaded_build_arg)"
+
-      displayName: "[DEBUG] Display Environment-Specific Variables"
+                  values[k] = v
+
+              for k, v in values.items():
+                  print("{} = {}".format(k, v))
+                  print("##vso[task.setvariable variable={}]{}".format(k, v))
+
+      timeoutInMinutes: 180

CI/NugetPublish.stage_template.yaml

@@ -0,0 +1,45 @@
+parameters:
+  agent_pool: "" # windows-2019
+  azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts)
+  azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline)
+  service_connection: "" # Service connection name setup in Azure DevOps
+
+stages:
+  - stage: NuGetPublish_Stage
+    displayName: "Publish NuGet Packages"
+    dependsOn: [] # No dependencies
+
+    jobs:
+      - job: NuGetPublish_Job
+        timeoutInMinutes: 240
+
+        displayName: "Publish"
+
+        pool:
+          vmImage: "${{ parameters.agent_pool }}"
+
+        steps:
+          - template: PublishInitialize.steps_template.yaml
+            parameters:
+              azure_dev_ops_project_id: ${{ parameters.azure_dev_ops_project_id }}
+              azure_dev_ops_build_definition_id: ${{ parameters.azure_dev_ops_build_definition_id }}
+
+          - task: NuGetCommand@2
+            displayName: "Push to Internal Store"
+            condition: not(or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1)))
+            inputs:
+              command: push
+              publishVstsFeed: "DataPipelines"
+              allowPackageConflicts: false
+              packagesToPush: "$(Pipeline.Workspace)/Package/Packages/**/*.nupkg;!$(Pipeline.Workspace)/Package/Packages/**/*.symbols.nupkg"
+            timeoutInMinutes: 180
+
+          - task: NuGetCommand@2
+            displayName: "Push to External Store"
+            condition: or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1))
+            inputs:
+              command: push
+              nuGetFeedType: external
+              packagesToPush: "$(Pipeline.Workspace)/Package/Packages/Release/**/*.nupkg;!$(Pipeline.Workspace)/Package/Packages/Release/**/*.symbols.nupkg"
+              publishFeedCredentials: ${{ parameters.service_connection }}
+            timeoutInMinutes: 180

CI/PR.yaml

@@ -9,12 +9,20 @@ resources:
       endpoint: featurizersbuild
 
 stages:
+  - template: Prerequisites.stage_template.yaml
+    parameters:
+      agent_pool: windows-2019
+      release_build: False
+      prerelease_build_name: ""
+
   # Windows/clang/x64 [Official build]
   - template: BuildAndTest.stage_template.yaml
     parameters:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x64
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: True
 
   # Windows/clang/x86 [Official build]
@@ -23,6 +31,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x86
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   # TODO: # Linux/GCC/universal_linux [Official build]
@@ -32,6 +42,8 @@ stages:
   # TODO:     agent_pool_container: universal_linux
   # TODO:     operating_system: Linux
   # TODO:     configuration: universal_linux
+  # TODO:     dependencies:
+  # TODO:       - Prerequisites_Stage
   # TODO:     enable_code_coverage: False
 
   # MacOS/clang/x64
@@ -40,6 +52,8 @@ stages:
       agent_pool: macOS-10.14
       operating_system: MacOS
       configuration: system_compiler
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   # Windows/MSVC/x64
@@ -48,6 +62,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x64_MSVC
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   # Windows/MSVC/x86
@@ -56,6 +72,8 @@ stages:
       agent_pool: windows-2019
       operating_system: Windows
       configuration: x86_MSVC
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   # Linux/clang/x64
@@ -64,6 +82,8 @@ stages:
       agent_pool: ubuntu-16.04
       operating_system: Linux
       configuration: x64
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   # Linux/GCC/system_compiler
@@ -72,6 +92,8 @@ stages:
       agent_pool: ubuntu-16.04
       operating_system: Linux
       configuration: system_compiler
+      dependencies:
+        - Prerequisites_Stage
       enable_code_coverage: False
 
   - template: Package.stage_template.yaml

CI/Package.stage_template.yaml

@@ -1,7 +1,6 @@
 parameters:
   operating_system: "" # Windows|Linux|MacOS
   esrp_connected_service_name: "None" # <Name>|None
-
   dependencies: []
   configuration: ""
   agent_pool: ""
@@ -35,7 +34,7 @@ stages:
               enable_code_coverage: False
 
           - script: |-
-              echo "esrp_connected_service_name- ${{ parameters.esrp_connected_service_name }}"
+              echo "esrp_connected_service_name - ${{ parameters.esrp_connected_service_name }}"
               echo "dependencies - ${{ join(', ', parameters.dependencies) }}"
               echo "configuration - ${{ parameters.configuration }}"
               echo "agent_pool - ${{ parameters.agent_pool }}
@@ -101,6 +100,7 @@ stages:
                   SourceFolder: "$(azure_artifacts_directory)/Artifacts/Packages"
                   TargetFolder: "$(azure_artifacts_directory)/Artifacts/Packages.original"
                   CleanTargetFolder: true
+                timeoutInMinutes: 180
 
               - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1
                 displayName: "ESRP CodeSign"
@@ -126,6 +126,7 @@ stages:
                             "ToolVersion" : "1.0"
                         }
                     ]
+                timeoutInMinutes: 180
 
           - task: PublishPipelineArtifact@0
             displayName: "Publish Artifacts"

CI/Prerequisites.stage_template.yaml

@@ -0,0 +1,87 @@
+parameters:
+  agent_pool: ""
+  release_build: False
+  prerelease_build_name: ""
+
+stages:
+  # This is a bit wonky, but we need to create values that can be used from stage-to-stage to ensure
+  # that builds that include datetime information in their product version numbers use a value that is
+  # consistent regardless of when the build was invoked within a particual stage. Create the value here,
+  # and then use that value when invoking the build in `BuildAndTest.stage_template.yaml`.
+  - stage: Prerequisites_Stage
+    displayName: "Persist Global Build Information"
+
+    jobs:
+      - job: Prerequistes_Job
+        timeoutInMinutes: 240
+
+        displayName: "Persist"
+        pool:
+          vmImage: ${{ parameters.agent_pool }}
+
+        steps:
+          - checkout: none
+
+          - script: |-
+              echo "agent_pool - ${{ parameters.agent_pool }}"
+              echo "release_build - ${{ parameters.release_build }}"
+              echo "prerelease_build_name - ${{ parameters.prerelease_build_name }}"
+            displayName: "[DEBUG] Display BuildAndTest Variables"
+
+          - task: PythonScript@0
+            displayName: "[IMPL] Create Global Build Information"
+            inputs:
+              scriptSource: inline
+              script: |-
+                from datetime import datetime
+                import json
+                import os
+                import sys
+
+                values = {}
+
+                release_build = sys.argv[1].lower() in ["true", "yes"]
+                values["is_release_build"] = release_build
+
+                # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of
+                # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value.
+                # Therefore, a non-standard comparison is necessary in this case
+                if len(sys.argv) > 2 and sys.argv[2] != "{}(prerelease_build_name)".format("$"):
+                    build_name = sys.argv[2]
+                    is_prerelease_build = True
+                else:
+                    is_prerelease_build = False
+
+                    if not release_build:
+                        now = datetime.now()
+
+                        # The build should compare as:
+                        #   "manual" < "pipeline" < "preview"
+                        build_name = "pipeline.{year}.{month}.{day}.{hour}.{minute}.{second}".format(
+                             year=now.year,
+                             month=now.month,
+                             day=now.day,
+                             hour=now.hour,
+                             minute=now.minute,
+                             second=now.second,
+                        )
+                    else:
+                        build_name = None
+
+                values["is_prerelease_build"] = is_prerelease_build
+                values["build_name"] = build_name
+
+                for k, v in values.items():
+                    print("{} = {}".format(k, v))
+
+                with open(os.path.join(r"$(Pipeline.Workspace)", "global_build_information.json"), "w") as f:
+                    json.dump(values, f)
+              arguments: "${{ parameters.release_build }} ${{ parameters.prerelease_build_name }}"
+            timeoutInMinutes: 180
+
+          - task: PublishPipelineArtifact@0
+            displayName: "Publish Artifacts"
+            inputs:
+              targetPath: "$(Pipeline.Workspace)/global_build_information.json"
+              artifactName: "Prerequisites"
+            timeoutInMinutes: 180

CI/Publish.yaml

@@ -6,13 +6,53 @@ trigger: none
 
 # User will need to specify buildId if publishing with non-triggered build
 stages:
-    - template: AzureUpload.stage_template.yaml
+  - template: AzureUpload.stage_template.yaml
-      parameters:
+    parameters:
-          agent_pool: vs2017-win2016
+      agent_pool: windows-2019
-          project: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32
+      azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32
-          definition: 717
+      azure_dev_ops_build_definition_id: 717
-          subscription: AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)
+      subscription: AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87)
-          storage: featurizersbuild
+      storage: featurizersbuild
-          containerName: Archive
+      containerName: Archive
 
+  - template: NugetPublish.stage_template.yaml
+    parameters:
+      agent_pool: windows-2019
+      azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32
+      azure_dev_ops_build_definition_id: 717
+      service_connection: "NuGet"
 
+  - stage: TagSource_Stage
+    displayName: "Tag Sources (for release builds)"
+    dependsOn:
+      - AzureUpload_Stage
+      - NuGetPublish_Stage
+
+    jobs:
+      - job: TagSource_Job
+        timeoutInMinutes: 240
+
+        displayName: "Tag"
+
+        pool:
+          vmImage: windows-2019
+
+        steps:
+          - checkout: self
+            persistCredentials: true
+            clean: true
+
+          - template: PublishInitialize.steps_template.yaml
+            parameters:
+              azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32
+              azure_dev_ops_build_definition_id: 717
+
+          - script: |-
+              git config --global user.email "featurizer_dev@microsoft.com"
+              git config --global user.name "Azure DevOps"
+
+              git tag "$(azure_product_version)"
+              git push origin "$(azure_product_version)"
+            displayName: "<Tag Release Sources [Optional]>"
+            condition: or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1))
+            timeoutInMinutes: 180

CI/PublishInitialize.steps_template.yaml

@@ -0,0 +1,102 @@
+parameters:
+  azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts)
+  azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline)
+
+steps:
+  # This build task will try to download artifacts from the triggering build.
+  # If there is no triggering build from the specified pipeline, it will download
+  # artifacts from the build specified in the options below
+  - task: PythonScript@0
+    displayName: "[IMPL] Validate Variables"
+    inputs:
+      scriptSource: inline
+      script: |-
+        import sys
+
+        triggeredBuildNumber = sys.argv[1]
+        buildId = sys.argv[2]
+
+        # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of
+        # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value.
+        # Therefore, a non-standard comparison is necessary in this case
+        if triggeredBuildNumber == "{}(Build.TriggeredBy.BuildNumber)".format("$"):
+            if buildId == "{}(buildId)".format("$"):
+               raise ValueError('Please specify variable buildId since this is a publish with non-triggered build. e.g. set buildId = 104648')
+
+      arguments: "$(Build.TriggeredBy.BuildNumber) $(buildId)"
+
+  # Download from a specific id
+  - task: DownloadPipelineArtifact@2
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual'))
+    displayName: "[IMPL] Download Pipeline Artifact for Manual build"
+    inputs:
+      buildType: "specific"
+      project: "${{ parameters.azure_dev_ops_project_id }}"
+      definition: "${{ parameters.azure_dev_ops_build_definition_id }}"
+      specificBuildWithTriggering: false
+      buildVersionToDownload: specific
+      pipelineId: "$(buildId)"
+      artifactName: ""
+      targetPath: "$(Pipeline.Workspace)"
+    timeoutInMinutes: 180
+
+  # Download from the triggering build
+  - task: DownloadPipelineArtifact@2
+    condition: and(succeeded(), eq(variables['Build.Reason'], 'BuildCompletion'))
+    displayName: "[IMPL] Download Pipeline Artifact for Build triggered by BuildCompletion"
+    inputs:
+      buildType: "specific"
+      project: "${{ parameters.azure_dev_ops_project_id }}"
+      definition: "${{ parameters.azure_dev_ops_build_definition_id }}"
+      specificBuildWithTriggering: true
+      buildVersionToDownload: latestFromBranch
+      artifactName: ""
+      targetPath: "$(Pipeline.Workspace)"
+    timeoutInMinutes: 180
+
+  - task: PythonScript@0
+    displayName: "[IMPL] Populate Stage Variables"
+    inputs:
+      scriptSource: inline
+      script: |-
+        import json
+        import os
+
+        from collections import OrderedDict
+
+        values = OrderedDict()
+
+        # Version info comes from these places:
+        #
+        #     - Global build Information
+        #     - Build Instance Information
+        #
+        for info_filename in [
+            os.path.join(r"$(Pipeline.Workspace)", "Prerequisites", "global_build_information.json"),
+
+            # It doesn't matter which configuration that we use, as they will all contain
+            # identical builder info files.
+            os.path.join(r"$(Pipeline.Workspace)", "Windows - x64", "Builder", "product_version_information.json"),
+        ]:
+            assert os.path.isdir(os.path.dirname(info_filename)), info_filename
+            assert os.path.isfile(info_filename), info_filename
+
+            with open(info_filename) as f:
+                content = json.load(f)
+
+            for k, v in content.items():
+                k = "azure_{}".format(k)
+                assert k not in values, (k, values[k], info_filename)
+
+                if isinstance(v, bool):
+                    v = 1 if v else 0
+                elif v is None:
+                    v = ""
+
+                values[k] = v
+
+        for k, v in values.items():
+            print("{} = {}".format(k, v))
+            print("##vso[task.setvariable variable={}]{}".format(k, v))
+
+    timeoutInMinutes: 180

CI/Readme.md

@@ -1,3 +1,62 @@
+Updating Build Information
+==========================
+Build information is updated in 1 of 2 ways, depending on which part of the version needs
+to be modified. [Semantic Versioning terminology](https://semver.org) is used in the description
+that follows.
+
+    Example build number:                   <major>.<minor>.<patch>[-<prelease_build_name>][+<build info>]
+    Examples:                               1.2.3-preview.1         ==  1.2.3-preview.1+with.build.info
+                                                < 1.2.3-preview.2   <   1.2.3-preview.11
+                                                < 1.2.3-zulu        <   1.2.3
+
+`<major>`, `<minor>`, and `<patch>` build values are defined in cmake files and should be updated there;
+for an example, see src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake.
+
+`<prerelease_build_name>` and `<build_info>` are defined when invoking the CI build. Additional information
+(including steps to specify these values) follows.
+
+Build Types
+===========
+The yaml build definitions in this directory are able to create difference build
+types, each with different versioning, packaging, and publishing characteristics.
+Build variables specified when invoking `CI.yaml` on Azure DevOps are used to
+specific which build type is invoked.
+
+All version numbers conform to [Semantic Versioning standards](https://semver.org)*.
+
+Continuous Integration / Standard Build
+---------------------------------------
+The build type invoked by default (when no Azure DevOps variables are specified).
+
+    Azure DevOps Variables:                 None
+    Version:                                <major>.<minor>.<patch>-cibuild.<year>.<month>.<day>.<hour>.<minute>.<second>.<debug|release>
+    Example:                                0.2.0-cibuild.2019.10.23.8.47.19.release
+    NuGet Store:                            https://aiinfra.visualstudio.com/DataPipelines/_packaging
+
+Prerelease
+----------
+An unofficial, but public, release.
+
+    Azure DevOps Variables:                             prerelease_build_name=<build_name>
+    Version:                                            <major>.<minor>.<patch>-<build_name>
+    Example (where prerelease_build_name=preview001):   0.2.0-preview001
+    NuGet Store:                                        https://www.nuget.org/
+
+Release
+-------
+An official release.
+
+    Azure DevOps Variables:                 release_build=True
+    Version:                                <major>.<minor>.<patch>
+    Example:                                0.2.0
+    NuGet Store:                            https://www.nuget.org
+
+\* Note that NuGet doesn't support SemVer's build info (e.g. "+\<build info here>") well, which is why
+Continuous Integration datetime info is stored as prerelease info rather than build info.
+
+Azure DevOps Configuration Settings
+===================================
+
 These Azure DevOps build definitions rely on the following Service Connections.
 To add/remove/view Service Connections within Azure DevOps:
 
@@ -24,6 +83,14 @@ Used to sign binaries and packages.
     AAD APP Secret:                                     <created in Azure>
     PRSS Signing Authentication Certificate:            Created through the [ESRP Portal](https://portal.esrp.microsoft.com/)
 
+Nuget
+-----
+Used to push NuGet packages.
+
+    Type:                                               NuGet Service Connection
+    Feed URL:                                           https://api.nuget.org/v3/index.json
+    ApiKey:                                             Created on NuGet.org and then copied to the Azure DevOps connection dialog window for one-tiem usage
+
 featurizersbuild
 ----------------
 Used to pull the universal_linux Docker image for universal_linux builds.
@@ -33,3 +100,8 @@ Used to pull the universal_linux Docker image for universal_linux builds.
     Connection name:                                    featurizersbuild
     Azure subscription:                                 AI Infra Build
     Azure container registry:                           featurizersbuild
+
+Azure DevOps Settings
+=====================
+The Publish build will tag sources upon successful completion. To do this, the Azure DevOps `Project Collection Build Service` account needs
+contribute permissions to create tags for the appropriate repos. Follow the steps [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/git-commands?view=azure-devops&tabs=yaml) to grant those permissions.

src/FeaturizerPrep/SharedLibrary/Build.py

@@ -4,6 +4,7 @@
 # ----------------------------------------------------------------------
 """Builds the Shared Library"""
 
+import datetime
 import hashlib
 import json
 import os
@@ -28,6 +29,7 @@ _script_dir, _script_name                   = os.path.split(_script_fullpath)
 # ----------------------------------------------------------------------
 
 CONFIGURATIONS                              = ["Debug", "Release"]
+JSON_FILENAME                               = "Microsoft MLFeaturizers.FileAttributes.json"
 
 # ----------------------------------------------------------------------
 @CommandLine.EntryPoint
@@ -36,6 +38,9 @@ CONFIGURATIONS                              = ["Debug", "Release"]
     output_dir=CommandLine.DirectoryTypeInfo(
         ensure_exists=False,
     ),
+    prerelease_build_name=CommandLine.StringTypeInfo(
+        arity="?",
+    ),
     cmake_generator=CommandLine.StringTypeInfo(
         arity="?",
     ),
@@ -44,13 +49,26 @@ CONFIGURATIONS                              = ["Debug", "Release"]
 def Build(
     configuration,
     output_dir,
+    release_build=False,
+    prerelease_build_name=None,
+    no_build_info=False,
     keep_temp_dir=False,
-    cmake_generator=(None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" else "Ninja"),
+    cmake_generator=(
+        None
+        if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION")
+        == "universal_linux"
+        else "Ninja"
+    ),
     output_stream=sys.stdout,
     verbose=False,
 ):
     """Builds the Featurizer Shared Library"""
 
+    if release_build and prerelease_build_name:
+        raise CommandLine.UsageException(
+            "A prerelese build name cannot be provided with the 'release_build' flag",
+        )
+
     with StreamDecorator(output_stream).DoneManager(
         line_prefix="",
         prefix="\nResults: ",
@@ -78,21 +96,54 @@ def Build(
             os.chdir(temp_directory)
 
             with CallOnExit(lambda: os.chdir(prev_dir)):
+                if not release_build:
+                    if prerelease_build_name is None:
+                        # This value should compare as:
+                        #   "manual" < "pipeline"
+                        prerelease_build_name = "manual"
+
+                    if not no_build_info:
+                        now = datetime.datetime.now()
+
+                        prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format(
+                            year=now.year,
+                            month=now.month,
+                            day=now.day,
+                            hour=now.hour,
+                            minute=now.minute,
+                            second=now.second,
+                            prerelease_build_name=prerelease_build_name,
+                            configuration=configuration.lower(),
+                        )
+
                 activities = [
                     (
                         "Generating cmake Files",
-                        'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} "{this_dir}"'.format(
+                        'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format(
-                            generator='-G "{}" '.format(cmake_generator) if cmake_generator else "",
+                            generator='-G "{}" '.format(
+                                cmake_generator,
+                            ) if cmake_generator else "",
                             temp_dir=temp_directory,
                             configuration=configuration,
                             this_dir=_script_dir,
+                            prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format(
+                                prerelease_build_name,
+                            ),
                         ),
                     ),
                     ("Building", "cmake --build ."),
                 ]
 
-                if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux":
+                if (
-                    activities.append(("Verifying Universal Linux Binaries", 'libcheck libFeaturizers.so'))
+                    os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION")
+                    == "universal_linux"
+                ):
+                    activities.append(
+                        (
+                            "Verifying Universal Linux Binaries",
+                            "libcheck libFeaturizers.so",
+                        ),
+                    )
 
                 activities += [
                     ("Copying Binaries", _CopyBinaries),
@@ -203,14 +254,14 @@ def Package(
         if len(build_dirs) > 1:
             dm.stream.write("Ensuring that build data matches...")
             with dm.stream.DoneManager() as ensure_dm:
-                ensure_dm.stream.write("Checking 'Featurizers.json'...")
+                ensure_dm.stream.write("Checking '{}'...".format(JSON_FILENAME))
                 with ensure_dm.stream.DoneManager() as this_dm:
                     this_dm.result = (
                         0
                         if _CompareFiles(
                             this_dm.stream,
                             *[
-                                os.path.join(build_dir, "Featurizers.json")
+                                os.path.join(build_dir, JSON_FILENAME)
                                 for build_dir in build_dirs
                             ]
                         )
@@ -234,7 +285,7 @@ def Package(
 
         dm.stream.write("Reading build configuration...")
         with dm.stream.DoneManager() as this_dm:
-            json_filename = os.path.join(build_dirs[0], "Featurizers.json")
+            json_filename = os.path.join(build_dirs[0], JSON_FILENAME)
             if not os.path.isfile(json_filename):
                 this_dm.stream.write(
                     "ERROR: The filename '{}' does not exist.\n".format(json_filename),
@@ -248,7 +299,11 @@ def Package(
 
             build_config["build_dir"] = build_dirs[0]
             build_config["data_dir"] = os.path.join(build_dirs[0], "Data", "**", "*.*")
-            build_config["package_id"] = build_config["product_name"].replace(" ", "")
+            build_config["package_id"] = build_config["product_name"].replace(" ", ".")
+            build_config["product_copyright"] = build_config["product_copyright"].replace(
+                "(C)",
+                "©",
+            )
 
         # Generate the correct nuget file statements based on output in the build_dir
         dm.stream.write("Generating nuget file statements...")
@@ -263,12 +318,10 @@ def Package(
                     this_value_type = None
 
                     if item == "Featurizers.dll":
-                        if "x64" in build_dir:
+                        if "x86" in build_dir:
-                            this_value_type = "runtimes/win-x64/native"
-                        elif "x86" in build_dir:
                             this_value_type = "runtimes/win-x86/native"
                         else:
-                            assert False, build_dir
+                            this_value_type = "runtimes/win-x64/native"
 
                     elif item.startswith("libFeaturizers.so"):
                         this_value_type = "runtimes/linux-x64/native"
@@ -279,7 +332,11 @@ def Package(
                             this_value_type = "runtimes/osx-x64/native"
 
                     if this_value_type is not None:
-                        assert value_type is None or this_value_type == value_type, (value_type, item, this_value_type)
+                        assert value_type is None or this_value_type == value_type, (
+                            value_type,
+                            item,
+                            this_value_type,
+                        )
 
                         value_type = this_value_type
                         these_files.append(os.path.join(build_dir, item))
@@ -344,16 +401,16 @@ _nuget_template                             = textwrap.dedent(
     <package xmlns="http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd">
         <metadata>
             <id>{package_id}</id>
-            <version>{product_version_major}.{product_version_minor}.{product_version_patch}.{product_version_revision}</version>
+            <version>{product_version_full}</version>
             <authors>Microsoft</authors>
             <owners>microsoft</owners>
             <license type="expression">MIT</license>
             <licenseUrl>https://licenses.nuget.org/MIT</licenseUrl>
             <requireLicenseAcceptance>true</requireLicenseAcceptance>
-            <copyright>{product_company_copyright}</copyright>
+            <copyright>{product_copyright}</copyright>
 
             <description>{product_bundle}</description>
-            <projectUrl>https://www.microsoft.com</projectUrl>
+            <projectUrl>https://aka.ms/MLFeaturizers</projectUrl>
 
             <contentFiles>
                 <files include="any/any/Data/**/*.json" buildAction="None" copyToOutput="true" />
@@ -370,13 +427,15 @@ _nuget_template                             = textwrap.dedent(
 
 # ----------------------------------------------------------------------
 def _CopyBinaries(temp_directory, output_dir, output_stream):
-    output_files = ["Featurizers.json"]
+    output_files = [JSON_FILENAME]
 
     if CurrentShell.CategoryName == "Windows":
         output_files += ["Featurizers.dll", "Featurizers.pdb"]
     elif CurrentShell.CategoryName in ["Linux", "BSD"]:
         for item in os.listdir(temp_directory):
-            if item.startswith("libFeaturizers") and not os.path.splitext(item)[1] in [".a"]:
+            if item.startswith("libFeaturizers") and not os.path.splitext(item)[1] in [
+                ".a"
+            ]:
                 output_files.append(item)
     else:
         raise Exception("The Current Shell is not supported")

src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake

@@ -2,86 +2,119 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License
 # ----------------------------------------------------------------------
-set(_project_name Featurizers)
+cmake_minimum_required(VERSION 3.5.0)
-set(_${_project_name}_version_major 0)
-set(_${_project_name}_version_minor 1)
-set(_${_project_name}_version_patch 0)
-set(_${_project_name}_version ${_${_project_name}_version_major}.${_${_project_name}_version_minor}.${_${_project_name}_version_patch})
 
-project(${_project_name} VERSION ${_${_project_name}_version} LANGUAGES CXX)
+set(_featurizers_this_path ${CMAKE_CURRENT_LIST_DIR} CACHE INTERNAL "")
 
-get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
+function(Impl)
+    # Defining a function here to introduce a new scope for local variables
+    set(_project_name Featurizers)
+    set(_version_major 0)                   # '1' in the release 1.2.3-alpha1+201910161322
+    set(_version_minor 3)                   # '2' in the release 1.2.3-alpha1+201910161322
+    set(_version_patch 5)                   # '3' in the release 1.2.3-alpha1+201910161322
+    set(_version_prerelease_info "")        # Optional 'alpha1' in the release 1.2.3-alpha1+201910161322
+    set(_version_build_info "")             # Optional '201910161322' in the release 1.2.3-alpha1+201910161322
 
-include(${_this_path}/generate_shared_library_attributes.cmake)
+    set(_version ${_version_major}.${_version_minor}.${_version_patch})
-include(${_this_path}/../../Featurizers/cmake/FeaturizersCode.cmake)
 
-get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
+    # Alpha version components (which are supported in SemVer) present problems
+    # for cmake when the version is provided inline. However, things work as expected
+    # when setting the version as a property.
+    project(${_project_name} LANGUAGES CXX)
+    set(PROJECT_VERSION ${_version})
 
-generate_shared_library_attributes(
+    set(CMAKE_CXX_STANDARD 14)
-    _shared_library_attributes_sources
+    set(CMAKE_CXX_STANDARD_REQUIRED ON)
-    NAME "Microsoft ML Featurizers"
+    set(CMAKE_CXX_EXTENSIONS OFF)
-    FILE_DESCRIPTION "Microsoft ML Featurizers"
-    COMPANY_NAME "Microsoft Corporation"
-    COPYRIGHT "Copyright (C) Microsoft Corporation. All rights reserved."
-    VERSION_MAJOR ${_${_project_name}_version_major}
-    VERSION_MINOR ${_${_project_name}_version_minor}
-    VERSION_PATCH ${_${_project_name}_version_patch}
-    ICON "NA"
-)
 
-add_library(Featurizers SHARED
+    set(_includes "$ENV{INCLUDE}")
-    ${_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.h
+    set(_libs "$ENV{LIB}")
-    ${_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.cpp
+    set(CMAKE_MODULE_PATH "$ENV{DEVELOPMENT_ENVIRONMENT_CMAKE_MODULE_PATH}")
-    ${_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.h
-    ${_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.cpp
-    ${_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.h
-    ${_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.cpp
 
-    ${_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.h
+    if(NOT WIN32)
-    ${_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.cpp
+        string(REPLACE ":" ";" CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}")
-    ${_this_path}/../GeneratedCode/SharedLibrary_Common.h
+        string(REPLACE ":" ";" _includes "${_includes}")
-    ${_this_path}/../GeneratedCode/SharedLibrary_Common.cpp
+        string(REPLACE ":" ";" _libs "${_libs}")
-    ${_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.h
+    endif()
-    ${_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_PointerTable.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_PointerTable.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.cpp
-    ${_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.h
-    ${_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.cpp
 
-    ${_shared_library_attributes_sources}
+    include(CppCommon)
-)
+    include(GenerateFileAttributes)
 
-target_link_libraries(Featurizers PRIVATE
+    generate_file_attributes(
-    FeaturizersCode
+        _featurizers_file_attribute_sources
-)
+        NAME "Microsoft MLFeaturizers"
+        COMPANY_NAME "Microsoft Corporation"
+        VERSION_MAJOR ${_version_major}
+        VERSION_MINOR ${_version_minor}
+        VERSION_PATCH ${_version_patch}
+        VERSION_PRERELEASE_INFO ${_version_prerelease_info}
+        VERSION_BUILD_INFO ${_version_build_info}
+        COPYRIGHT "(C) Microsoft Corporation. All rights reserved."
+    )
 
-target_include_directories(Featurizers PRIVATE
+    include(${_featurizers_this_path}/../../Featurizers/cmake/FeaturizersCode.cmake)
-    ${_this_path}/../GeneratedCode
-    ${_this_path}/../..
-    ${_this_path}/../../Featurizers
-    ${_includes}
-)
 
-target_link_directories(Featurizers PRIVATE
+    add_library(
-    ${_libs}
+        ${_project_name} SHARED
-)
 
-set_target_properties(
+        ${_featurizers_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.h
-    ${_project_name} PROPERTIES
+        ${_featurizers_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.cpp
-    VERSION ${_${_project_name}_version}
+        ${_featurizers_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.h
-    SOVERSION ${_${_project_name}_version_major}
+        ${_featurizers_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.cpp
-)
+        ${_featurizers_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.h
+        ${_featurizers_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.cpp
+
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_Common.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_Common.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_PointerTable.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_PointerTable.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.cpp
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.h
+        ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.cpp
+
+        ${_featurizers_file_attribute_sources}
+    )
+
+    set_target_properties(
+        ${_project_name} PROPERTIES
+        VERSION ${_version}
+        SOVERSION ${_version_major}
+    )
+
+    target_link_libraries(
+        ${_project_name} PRIVATE
+        FeaturizersCode
+    )
+
+    target_include_directories(
+        ${_project_name} PRIVATE
+        ${_featurizers_this_path}/../GeneratedCode
+        ${_featurizers_this_path}/../..
+        ${_featurizers_this_path}/../../Featurizers
+        ${_includes}
+    )
+
+    target_link_directories(
+        ${_project_name} PRIVATE
+        ${_libs}
+    )
+endfunction()
+
+Impl()

src/FeaturizerPrep/SharedLibrary/cmake/generate_shared_library_attributes.cmake

@@ -1,125 +0,0 @@
-# ----------------------------------------------------------------------
-# Copyright (c) Microsoft Corporation. All rights reserved.
-# Licensed under the MIT License
-# ----------------------------------------------------------------------
-get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY)
-
-# This code is based on code by halex2005, available at https://github.com/halex2005/CMakeHelpers/blob/master/generate_product_version.cmake,
-# which is distributed under the MIT license (https://github.com/halex2005/CMakeHelpers/blob/master/LICENSE).
-
-include (CMakeParseArguments)
-
-# generate_product_version() function
-#
-# This function uses VersionInfo.in template file and VersionResource.rc file
-# to generate WIN32 resource with version information and general resource strings.
-#
-# Usage:
-#   generate_product_version(
-#     SomeOutputResourceVariable
-#     NAME MyGreatProject
-#     ICON ${PATH_TO_APP_ICON}
-#     VERSION_MAJOR 2
-#     VERSION_MINOR 3
-#     VERSION_PATH ${BUILD_COUNTER}
-#     VERSION_REVISION ${BUILD_REVISION}
-#   )
-# where BUILD_COUNTER and BUILD_REVISION could be values from your CI server.
-#
-# You can use generated resource for your executable targets:
-#   add_executable(target-name ${target-files} ${SomeOutputResourceVariable})
-#
-# You can specify resource strings in arguments:
-#   NAME               - name of executable (no defaults, ex: Microsoft Word)
-#   BUNDLE             - bundle (${NAME} is default, ex: Microsoft Office)
-#   ICON               - path to application icon (${CMAKE_SOURCE_DIR}/product.ico by default)
-#   VERSION_MAJOR      - 1 is default
-#   VERSION_MINOR      - 0 is default
-#   VERSION_PATCH      - 0 is default
-#   VERSION_REVISION   - 0 is default
-#   COMPANY_NAME       - your company name (no defaults)
-#   COMPANY_COPYRIGHT  - ${COMPANY_NAME} (C) Copyright ${CURRENT_YEAR} is default
-#   COMMENTS           - ${NAME} v${VERSION_MAJOR}.${VERSION_MINOR} is default
-#   ORIGINAL_FILENAME  - ${NAME} is default
-#   INTERNAL_NAME      - ${NAME} is default
-#   FILE_DESCRIPTION   - ${NAME} is default
-function(generate_shared_library_attributes outfiles)
-    set (options)
-    set (oneValueArgs
-        NAME
-        BUNDLE
-        ICON
-        VERSION_MAJOR
-        VERSION_MINOR
-        VERSION_PATCH
-        VERSION_REVISION
-        COMPANY_NAME
-        COMPANY_COPYRIGHT
-        COMMENTS
-        ORIGINAL_FILENAME
-        INTERNAL_NAME
-        FILE_DESCRIPTION)
-    set (multiValueArgs)
-    cmake_parse_arguments(PRODUCT "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN})
-
-    if (NOT PRODUCT_BUNDLE OR "${PRODUCT_BUNDLE}" STREQUAL "")
-        set(PRODUCT_BUNDLE "${PRODUCT_NAME}")
-    endif()
-    if (NOT PRODUCT_ICON OR "${PRODUCT_ICON}" STREQUAL "")
-        set(PRODUCT_ICON "${CMAKE_SOURCE_DIR}/product.ico")
-    endif()
-
-    if (NOT PRODUCT_VERSION_MAJOR EQUAL 0 AND (NOT PRODUCT_VERSION_MAJOR OR "${PRODUCT_VERSION_MAJOR}" STREQUAL ""))
-        set(PRODUCT_VERSION_MAJOR 1)
-    endif()
-    if (NOT PRODUCT_VERSION_MINOR EQUAL 0 AND (NOT PRODUCT_VERSION_MINOR OR "${PRODUCT_VERSION_MINOR}" STREQUAL ""))
-        set(PRODUCT_VERSION_MINOR 0)
-    endif()
-    if (NOT PRODUCT_VERSION_PATCH EQUAL 0 AND (NOT PRODUCT_VERSION_PATCH OR "${PRODUCT_VERSION_PATCH}" STREQUAL ""))
-        set(PRODUCT_VERSION_PATCH 0)
-    endif()
-    if (NOT PRODUCT_VERSION_REVISION EQUAL 0 AND (NOT PRODUCT_VERSION_REVISION OR "${PRODUCT_VERSION_REVISION}" STREQUAL ""))
-        set(PRODUCT_VERSION_REVISION 0)
-    endif()
-
-    if (NOT PRODUCT_COMPANY_COPYRIGHT OR "${PRODUCT_COMPANY_COPYRIGHT}" STREQUAL "")
-        string(TIMESTAMP PRODUCT_CURRENT_YEAR "%Y")
-        set(PRODUCT_COMPANY_COPYRIGHT "${PRODUCT_COMPANY_NAME} (C) Copyright ${PRODUCT_CURRENT_YEAR}")
-    endif()
-    if (NOT PRODUCT_COMMENTS OR "${PRODUCT_COMMENTS}" STREQUAL "")
-        set(PRODUCT_COMMENTS "${PRODUCT_NAME} v${PRODUCT_VERSION_MAJOR}.${PRODUCT_VERSION_MINOR}")
-    endif()
-    if (NOT PRODUCT_ORIGINAL_FILENAME OR "${PRODUCT_ORIGINAL_FILENAME}" STREQUAL "")
-        set(PRODUCT_ORIGINAL_FILENAME "${PRODUCT_NAME}")
-    endif()
-    if (NOT PRODUCT_INTERNAL_NAME OR "${PRODUCT_INTERNAL_NAME}" STREQUAL "")
-        set(PRODUCT_INTERNAL_NAME "${PRODUCT_NAME}")
-    endif()
-    if (NOT PRODUCT_FILE_DESCRIPTION OR "${PRODUCT_FILE_DESCRIPTION}" STREQUAL "")
-        set(PRODUCT_FILE_DESCRIPTION "${PRODUCT_NAME}")
-    endif()
-
-    set (_SharedLibraryHeaderFile ${CMAKE_CURRENT_BINARY_DIR}/SharedLibraryAttributes.h)
-    set (_SharedLibraryResourceFile ${CMAKE_CURRENT_BINARY_DIR}/SharedLibraryAttributes.rc)
-
-    configure_file(
-        ${_this_path}/../Featurizers.in.json
-        ${CMAKE_CURRENT_BINARY_DIR}/Featurizers.json
-        @ONLY
-    )
-
-    configure_file(
-        ${_this_path}/../SharedLibraryAttributes.in.h
-        ${_SharedLibraryHeaderFile}
-        @ONLY
-    )
-
-    configure_file(
-        ${_this_path}/../SharedLibraryAttributes.rc
-        ${_SharedLibraryResourceFile}
-        COPYONLY
-    )
-
-    list(APPEND ${outfiles} ${_SharedLibraryHeaderFile} ${_SharedLibraryResourceFile})
-    set (${outfiles} ${${outfiles}} PARENT_SCOPE)
-endfunction()