diff --git a/CI/AzureUpload.stage_template.yaml b/CI/AzureUpload.stage_template.yaml index 92fd082..a7112bf 100644 --- a/CI/AzureUpload.stage_template.yaml +++ b/CI/AzureUpload.stage_template.yaml @@ -1,104 +1,70 @@ parameters: agent_pool: "" # vs2017-win2016 (Linux not support in some task) - project: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts) - definition: "" # 717 (The definition ID of the build pipeline) + azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts) + azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline) subscription: "" # AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87) storage: "" # featurizersbuild (The name of storage account in Azure) containerName: "" # Archive (The name of the container to which the files will be copied) stages: - stage: AzureUpload_Stage + displayName: "Upload to Azure" + dependsOn: [] # No dependencies jobs: - job: AzureUpload_Job + timeoutInMinutes: 240 + + displayName: "Upload" pool: name: Azure Pipelines demands: azureps vmImage: "${{ parameters.agent_pool }}" - # This build task will try to download artifacts from the triggering build. - # If there is no triggering build from the specified pipeline, it will download - # artifacts from the build specified in the options below steps: + - template: PublishInitialize.steps_template.yaml + parameters: + azure_dev_ops_project_id: ${{ parameters.azure_dev_ops_project_id }} + azure_dev_ops_build_definition_id: ${{ parameters.azure_dev_ops_build_definition_id }} + + - task: CopyFiles@2 + displayName: Copying Documentation + inputs: + SourceFolder: "$(Pipeline.Workspace)/Documentation" + Contents: "**" + TargetFolder: "$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Documentation/" + + - task: CopyFiles@2 + displayName: Copying Packages + inputs: + SourceFolder: "$(Pipeline.Workspace)/Package" + Contents: "**" + TargetFolder: "$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Package/" + - task: PythonScript@0 - displayName: 'Check Variables' + displayName: "Trim Directory Name" inputs: scriptSource: inline script: | - import sys + import os + import sys - triggeredBuildNumber = sys.argv[1] - buildId = sys.argv[2] - - # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of - # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value. - # Therefore, a non-standard comparison is necessary in this case - if triggeredBuildNumber == "{}(Build.TriggeredBy.BuildNumber)".format("$"): - if buildId == "{}(buildId)".format("$"): - raise ValueError('Please specify variable buildId since this is a publish with non-triggered build. e.g. set buildId = 104648') - arguments: '$(Build.TriggeredBy.BuildNumber) $(buildId)' - - - task: DownloadPipelineArtifact@2 - condition: eq(variables['Build.Reason'], 'Manual') - displayName: 'Download Pipeline Artifact for Mannual build' - inputs: - buildType: 'specific' - project: "${{ parameters.project }}" - definition: "${{ parameters.definition }}" - specificBuildWithTriggering: false - buildVersionToDownload: specific - pipelineId: '$(buildId)' - artifactName: '' - targetPath: '$(Pipeline.Workspace)' - - - task: DownloadPipelineArtifact@2 - condition: eq(variables['Build.Reason'], 'BuildCompletion') - displayName: 'Download Pipeline Artifact for Build triggered by BuildCompletion' - inputs: - buildType: 'specific' - project: "${{ parameters.project }}" - definition: "${{ parameters.definition }}" - specificBuildWithTriggering: true - buildVersionToDownload: latestFromBranch - artifactName: '' - targetPath: '$(Pipeline.Workspace)' - - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Pipeline.Workspace)/Documentation' - Contents: '**' - TargetFolder: '$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Documentation/' - - - task: CopyFiles@2 - inputs: - SourceFolder: '$(Pipeline.Workspace)/Package' - Contents: '**' - TargetFolder: '$(Pipeline.Workspace)/upload/$(Build.TriggeredBy.BuildNumber)/Package/' - - - task: PythonScript@0 - displayName: 'Trim Directory Name' - inputs: - scriptSource: inline - script: | - import os - import sys - - basedir = sys.argv[1] - for fn in os.listdir(basedir): - if not os.path.isdir(os.path.join(basedir, fn)): - continue # Not a directory - if fn == "{}(Build.TriggeredBy.BuildNumber)".format("$"): - os.rename(os.path.join(basedir, fn), os.path.join(basedir, 'Independent Publish_' + sys.argv[2])) - else: - os.rename(os.path.join(basedir, fn), os.path.join(basedir, fn[-13:])) + basedir = sys.argv[1] + for fn in os.listdir(basedir): + if not os.path.isdir(os.path.join(basedir, fn)): + continue # Not a directory + if fn == "{}(Build.TriggeredBy.BuildNumber)".format("$"): + os.rename(os.path.join(basedir, fn), os.path.join(basedir, 'Independent Publish_' + sys.argv[2])) + else: + os.rename(os.path.join(basedir, fn), os.path.join(basedir, fn[-13:])) arguments: '$(Pipeline.Workspace)\upload $(buildId)' - task: AzureFileCopy@2 - displayName: 'AzureBlob File Copy' + displayName: "AzureBlob File Copy" inputs: - SourcePath: '$(Pipeline.Workspace)/upload' + SourcePath: "$(Pipeline.Workspace)/upload" azureSubscription: "${{ parameters.subscription }}" - Destination: 'AzureBlob' + Destination: "AzureBlob" storage: "${{ parameters.storage }}" ContainerName: "${{ parameters.containerName }}" diff --git a/CI/BuildAndTest.stage_template.yaml b/CI/BuildAndTest.stage_template.yaml index e87145e..9f343dc 100644 --- a/CI/BuildAndTest.stage_template.yaml +++ b/CI/BuildAndTest.stage_template.yaml @@ -2,6 +2,7 @@ parameters: operating_system: "" # Windows|Linux|MacOS enable_code_coverage: "" # True|False esrp_connected_service_name: "None" # |None + dependencies: [] configuration: "" agent_pool: "" @@ -22,7 +23,7 @@ parameters: stages: - stage: BuildAndTest_${{ parameters.operating_system }}_${{ parameters.configuration }}_Stage displayName: "${{ parameters.operating_system }} - ${{ parameters.configuration }}: " - dependsOn: [] # No dependencies + dependsOn: ${{ parameters.dependencies }} jobs: - template: BuildAndTest.job_template.yaml @@ -42,6 +43,7 @@ stages: - script: |- echo "esrp_connected_service_name - ${{ parameters.esrp_connected_service_name }}" + echo "dependencies - ${{ join(', ', parameters.dependencies) }}" echo "configuration - ${{ parameters.configuration }}" echo "agent_pool - ${{ parameters.agent_pool }}" echo "agent_pool_container - ${{ parameters.agent_pool_container }}" @@ -62,8 +64,25 @@ stages: # TODO: Code formatting + # Dynamically create the builder args + - task: PythonScript@0 + displayName: "[IMPL] Create Builder Args" + inputs: + scriptSource: inline + script: |- + if "$(azure_is_release_build)" == "1": + args = "/release_build" + else: + args = "/prerelease_build_name=$(azure_build_name)" + + args = '"/custom_build_args=src/FeaturizerPrep/SharedLibrary:{} /no_build_info"'.format(args) + + print("azure_builder_args = {}".format(args)) + print("##vso[task.setvariable variable=azure_builder_args]{}".format(args)) + timeoutInMinutes: 180 + - script: |- - $(azure_activate_script) ${{ parameters.configuration }} && Builder$(azure_script_extension) Execute . "$(azure_artifacts_directory)/Builder" /verbose + $(azure_activate_script) ${{ parameters.configuration }} && Builder$(azure_script_extension) Execute . "$(azure_artifacts_directory)/Builder" /verbose $(azure_builder_args) displayName: "" timeoutInMinutes: 180 @@ -119,6 +138,46 @@ stages: } ] + - task: PythonScript@0 + displayName: "[IMPL] Persist Product Version Information" + inputs: + scriptSource: inline + script: |- + import json + import os + + from collections import OrderedDict + + values = OrderedDict() + + builder_dir = os.path.join(r"$(azure_artifacts_directory)", "Builder") + + # Get the release directory + for potential_dirname in os.listdir(builder_dir): + potential_fullpath = os.path.join(builder_dir, potential_dirname) + if not os.path.isdir(potential_fullpath): + continue + + file_attributes_filename = os.path.join(potential_fullpath, "Microsoft MLFeaturizers.FileAttributes.json") + if not os.path.isfile(file_attributes_filename): + raise Exception("'{}' does not exist".format(file_attributes_filename)) + + with open(file_attributes_filename) as f: + file_attributes_data = json.load(f) + + assert "product_version_full" in file_attributes_data, file_attributes_data + values["product_version"] = file_attributes_data["product_version_full"] + + break + + for k, v in values.items(): + print("{} = {}".format(k, v)) + + with open(os.path.join(builder_dir, "product_version_information.json"), "w") as f: + json.dump(values, f) + condition: succeededOrFailed() + timeoutInMinutes: 180 + - task: PublishPipelineArtifact@0 displayName: "Publish Artifacts" inputs: diff --git a/CI/CI.yaml b/CI/CI.yaml index c800957..1574682 100644 --- a/CI/CI.yaml +++ b/CI/CI.yaml @@ -12,13 +12,29 @@ resources: image: universal_linux endpoint: featurizersbuild +# Note that a variable set in this file will always override a value set within the editor. +# Therefore, do not set the default value to False here but instead do it in an initialization +# script. +# +# variables: +# release_build: False # Set to true to create a release build +# prerelease_build_name: "" # Set to a value (e.g. "preview.1" to create a preview build) + stages: + - template: Prerequisites.stage_template.yaml + parameters: + agent_pool: windows-2019 + release_build: $(release_build) + prerelease_build_name: $(prerelease_build_name) + # Windows/clang/x64 [Official Build] - template: BuildAndTest.stage_template.yaml parameters: agent_pool: windows-2019 operating_system: Windows configuration: x64 + dependencies: + - Prerequisites_Stage enable_code_coverage: True esrp_connected_service_name: "ESRP CodeSigning Connection" @@ -28,6 +44,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x86 + dependencies: + - Prerequisites_Stage enable_code_coverage: False esrp_connected_service_name: "ESRP CodeSigning Connection" @@ -38,6 +56,8 @@ stages: # TODO: agent_pool_container: universal_linux # TODO: operating_system: Linux # TODO: configuration: universal_linux + # TODO: dependencies: + # TODO: - Prerequisites_Stage # TODO: enable_code_coverage: False # TODO: esrp_connected_service_name: "ESRP CodeSigning Connection" @@ -47,6 +67,8 @@ stages: agent_pool: macOS-10.14 operating_system: MacOS configuration: system_compiler + dependencies: + - Prerequisites_Stage enable_code_coverage: False esrp_connected_service_name: "ESRP CodeSigning Connection" @@ -56,6 +78,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x64_MSVC + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing. @@ -65,6 +89,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x86_MSVC + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing. @@ -74,6 +100,8 @@ stages: agent_pool: ubuntu-16.04 operating_system: Linux configuration: x64 + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing. @@ -83,6 +111,8 @@ stages: agent_pool: ubuntu-16.04 operating_system: Linux configuration: system_compiler + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Note that this build doesn't produce "official" binaries and is here for a santity check only. Therefore, no code signing. diff --git a/CI/Initialize.steps_template.yaml b/CI/Initialize.steps_template.yaml index 2de4913..d729587 100644 --- a/CI/Initialize.steps_template.yaml +++ b/CI/Initialize.steps_template.yaml @@ -4,79 +4,73 @@ parameters: enable_code_coverage: "" # True|False steps: - # These steps are an ugly hack to set Azure DevOps variables conditionally. It really seems like there should be a better way to do this. - - ${{ if eq(parameters.operating_system, 'Windows') }}: - - script: |- - echo ##vso[task.setvariable variable=azure_activate_script]call Activate.cmd - echo ##vso[task.setvariable variable=azure_script_extension].cmd - echo ##vso[task.setvariable variable=azure_display_all_environment_vars]set - echo ##vso[task.setvariable variable=azure_bootstrap_command]bootstrap.cmd - echo ##vso[task.setvariable variable=azure_agent_temp_directory]%AGENT_TEMPDIRECTORY% - echo ##vso[task.setvariable variable=azure_artifacts_directory]%BUILD_ARTIFACTSTAGINGDIRECTORY% - echo ##vso[task.setvariable variable=azure_single_threaded_build_arg] - displayName: "[IMPL] Set Environment-Specific Variables (Standard)" + - task: DownloadPipelineArtifact@2 + displayName: "[IMPL] Download Global Build Information" + inputs: + artifactName: "Prerequisites" + targetPath: "$(Pipeline.Workspace)/Prerequisites" + timeoutInMinutes: 180 - # Dynamic Code Coverage Value - - ${{ if eq(parameters.enable_code_coverage, 'true') }}: - - script: |- - echo ##vso[task.setvariable variable=azure_code_coverage_arg]/code_coverage - displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)" + - task: PythonScript@0 + displayName: "[IMPL] Populate Stage Variables" + inputs: + scriptSource: inline + script: |- + import os + import json + from collections import OrderedDict - - ${{ if not(eq(parameters.enable_code_coverage, 'true')) }}: - - script: |- - echo ##vso[task.setvariable variable=azure_code_coverage_arg] - displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)" + values = OrderedDict() - - ? ${{ if or(eq(parameters.operating_system, 'Linux'), eq(parameters.operating_system, 'MacOS')) }} - : - ${{ if eq(parameters.configuration, 'universal_linux') }}: - - script: |- - sudo yum install -y git - displayName: "[IMPL] Install git" + # Create standard values + operating_system = "${{ parameters.operating_system }}" - - script: |- - echo "##vso[task.setvariable variable=azure_activate_script]source ./Activate.sh" - echo "##vso[task.setvariable variable=azure_script_extension].sh" - echo "##vso[task.setvariable variable=azure_display_all_environment_vars]export" - echo "##vso[task.setvariable variable=azure_bootstrap_command]sudo ./bootstrap.sh" - echo "##vso[task.setvariable variable=azure_agent_temp_directory]${AGENT_TEMPDIRECTORY}" - echo "##vso[task.setvariable variable=azure_artifacts_directory]${BUILD_ARTIFACTSTAGINGDIRECTORY}" - displayName: "[IMPL] Set Environment-Specific Variables (Standard)" + if operating_system == "Windows": + values["azure_activate_script"] = "call Activate.cmd" + values["azure_script_extension"] = ".cmd" + values["azure_display_all_environment_vars"] = "set" + values["azure_bootstrap_command"] = "bootstrap.cmd" + values["azure_single_threaded_build_arg"] = "" + elif operating_system in ["Linux", "MacOS"]: + values["azure_activate_script"] = "source ./Activate.sh" + values["azure_script_extension"] = ".sh" + values["azure_display_all_environment_vars"] = "export" + values["azure_bootstrap_command"] = "sudo ./bootstrap.sh" - # Dynamic Code Coverage Value - - ${{ if eq(parameters.enable_code_coverage, 'true') }}: - - script: |- - echo "##vso[task.setvariable variable=azure_code_coverage_arg]/code_coverage" - displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)" + # The MacOS compiler hangs when invoking the compiler in parallel. As a result, force single threaded + # execution. + if operating_system == "MacOS": + values["azure_single_threaded_build_arg"] = "/single_threaded" + else: + values["azure_single_threaded_build_arg"] = "" - - ${{ if not(eq(parameters.enable_code_coverage, 'true')) }}: - - script: |- - echo "##vso[task.setvariable variable=azure_code_coverage_arg]" - displayName: "[IMPL] Set Environment-Specific Variables (Code Coverage)" + values["azure_agent_temp_directory"] = os.getenv("AGENT_TEMPDIRECTORY") + values["azure_artifacts_directory"] = os.getenv("BUILD_ARTIFACTSTAGINGDIRECTORY") - # Dynamic Single Threaded Build Value. - # The MacOS compiler hangs when invoking the compiler in parallel. As a result, force single threaded - # execution. - - ${{ if eq(parameters.operating_system, 'MacOS') }}: - - script: |- - echo "##vso[task.setvariable variable=azure_single_threaded_build_arg]/single_threaded" - displayName: "[IMPL] Set Environment-Specific Variables (Single Threaded Build)" + if "${{ parameters.enable_code_coverage }}".lower() in ["true", "yes"]: + values["azure_code_coverage_arg"] = "/code_coverage" + else: + values["azure_code_coverage_arg"] = "" - - ${{ if not(eq(parameters.operating_system, 'MacOS')) }}: - - script: |- - echo "##vso[task.setvariable variable=azure_single_threaded_build_arg]" - displayName: "[IMPL] Set Environment-Specific Variables (Single Threaded Build)" + # Load global values previously generated + json_filename = os.path.join(r"$(Pipeline.Workspace)", "Prerequisites", "global_build_information.json") + assert os.path.isfile(json_filename), json_filename - - script: |- - echo "operating_system - ${{ parameters.operating_system }}" - echo "configuration - ${{ parameters.configuration }}" - echo "enable_code_coverage - ${{ parameters.enable_code_coverage }}" + with open(json_filename) as f: + content = json.load(f) - echo "activate_script - $(azure_activate_script)" - echo "script_extension - $(azure_script_extension)" - echo "display_all_environment_vars - $(azure_display_all_environment_vars)" - echo "bootstrap_command - $(azure_bootstrap_command)" - echo "agent_temp_directory - $(azure_agent_temp_directory)" - echo "artifacts_directory - $(azure_artifacts_directory)" - echo "code_coverage_arg - $(azure_code_coverage_arg)" - echo "single_threaded_arg - $(azure_single_threaded_build_arg)" - displayName: "[DEBUG] Display Environment-Specific Variables" + for k, v in content.items(): + k = "azure_{}".format(k) + + if isinstance(v, bool): + v = 1 if v else 0 + elif v is None: + v = "" + + values[k] = v + + for k, v in values.items(): + print("{} = {}".format(k, v)) + print("##vso[task.setvariable variable={}]{}".format(k, v)) + + timeoutInMinutes: 180 diff --git a/CI/NugetPublish.stage_template.yaml b/CI/NugetPublish.stage_template.yaml new file mode 100644 index 0000000..9121d52 --- /dev/null +++ b/CI/NugetPublish.stage_template.yaml @@ -0,0 +1,45 @@ +parameters: + agent_pool: "" # windows-2019 + azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts) + azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline) + service_connection: "" # Service connection name setup in Azure DevOps + +stages: + - stage: NuGetPublish_Stage + displayName: "Publish NuGet Packages" + dependsOn: [] # No dependencies + + jobs: + - job: NuGetPublish_Job + timeoutInMinutes: 240 + + displayName: "Publish" + + pool: + vmImage: "${{ parameters.agent_pool }}" + + steps: + - template: PublishInitialize.steps_template.yaml + parameters: + azure_dev_ops_project_id: ${{ parameters.azure_dev_ops_project_id }} + azure_dev_ops_build_definition_id: ${{ parameters.azure_dev_ops_build_definition_id }} + + - task: NuGetCommand@2 + displayName: "Push to Internal Store" + condition: not(or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1))) + inputs: + command: push + publishVstsFeed: "DataPipelines" + allowPackageConflicts: false + packagesToPush: "$(Pipeline.Workspace)/Package/Packages/**/*.nupkg;!$(Pipeline.Workspace)/Package/Packages/**/*.symbols.nupkg" + timeoutInMinutes: 180 + + - task: NuGetCommand@2 + displayName: "Push to External Store" + condition: or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1)) + inputs: + command: push + nuGetFeedType: external + packagesToPush: "$(Pipeline.Workspace)/Package/Packages/Release/**/*.nupkg;!$(Pipeline.Workspace)/Package/Packages/Release/**/*.symbols.nupkg" + publishFeedCredentials: ${{ parameters.service_connection }} + timeoutInMinutes: 180 diff --git a/CI/PR.yaml b/CI/PR.yaml index 8c26e5c..b241160 100644 --- a/CI/PR.yaml +++ b/CI/PR.yaml @@ -9,12 +9,20 @@ resources: endpoint: featurizersbuild stages: + - template: Prerequisites.stage_template.yaml + parameters: + agent_pool: windows-2019 + release_build: False + prerelease_build_name: "" + # Windows/clang/x64 [Official build] - template: BuildAndTest.stage_template.yaml parameters: agent_pool: windows-2019 operating_system: Windows configuration: x64 + dependencies: + - Prerequisites_Stage enable_code_coverage: True # Windows/clang/x86 [Official build] @@ -23,6 +31,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x86 + dependencies: + - Prerequisites_Stage enable_code_coverage: False # TODO: # Linux/GCC/universal_linux [Official build] @@ -32,6 +42,8 @@ stages: # TODO: agent_pool_container: universal_linux # TODO: operating_system: Linux # TODO: configuration: universal_linux + # TODO: dependencies: + # TODO: - Prerequisites_Stage # TODO: enable_code_coverage: False # MacOS/clang/x64 @@ -40,6 +52,8 @@ stages: agent_pool: macOS-10.14 operating_system: MacOS configuration: system_compiler + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Windows/MSVC/x64 @@ -48,6 +62,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x64_MSVC + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Windows/MSVC/x86 @@ -56,6 +72,8 @@ stages: agent_pool: windows-2019 operating_system: Windows configuration: x86_MSVC + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Linux/clang/x64 @@ -64,6 +82,8 @@ stages: agent_pool: ubuntu-16.04 operating_system: Linux configuration: x64 + dependencies: + - Prerequisites_Stage enable_code_coverage: False # Linux/GCC/system_compiler @@ -72,6 +92,8 @@ stages: agent_pool: ubuntu-16.04 operating_system: Linux configuration: system_compiler + dependencies: + - Prerequisites_Stage enable_code_coverage: False - template: Package.stage_template.yaml diff --git a/CI/Package.stage_template.yaml b/CI/Package.stage_template.yaml index d8e654b..81663d6 100644 --- a/CI/Package.stage_template.yaml +++ b/CI/Package.stage_template.yaml @@ -1,7 +1,6 @@ parameters: operating_system: "" # Windows|Linux|MacOS esrp_connected_service_name: "None" # |None - dependencies: [] configuration: "" agent_pool: "" @@ -35,7 +34,7 @@ stages: enable_code_coverage: False - script: |- - echo "esrp_connected_service_name- ${{ parameters.esrp_connected_service_name }}" + echo "esrp_connected_service_name - ${{ parameters.esrp_connected_service_name }}" echo "dependencies - ${{ join(', ', parameters.dependencies) }}" echo "configuration - ${{ parameters.configuration }}" echo "agent_pool - ${{ parameters.agent_pool }} @@ -101,6 +100,7 @@ stages: SourceFolder: "$(azure_artifacts_directory)/Artifacts/Packages" TargetFolder: "$(azure_artifacts_directory)/Artifacts/Packages.original" CleanTargetFolder: true + timeoutInMinutes: 180 - task: SFP.build-tasks.custom-build-task-1.EsrpCodeSigning@1 displayName: "ESRP CodeSign" @@ -126,6 +126,7 @@ stages: "ToolVersion" : "1.0" } ] + timeoutInMinutes: 180 - task: PublishPipelineArtifact@0 displayName: "Publish Artifacts" diff --git a/CI/Prerequisites.stage_template.yaml b/CI/Prerequisites.stage_template.yaml new file mode 100644 index 0000000..b7634bf --- /dev/null +++ b/CI/Prerequisites.stage_template.yaml @@ -0,0 +1,87 @@ +parameters: + agent_pool: "" + release_build: False + prerelease_build_name: "" + +stages: + # This is a bit wonky, but we need to create values that can be used from stage-to-stage to ensure + # that builds that include datetime information in their product version numbers use a value that is + # consistent regardless of when the build was invoked within a particual stage. Create the value here, + # and then use that value when invoking the build in `BuildAndTest.stage_template.yaml`. + - stage: Prerequisites_Stage + displayName: "Persist Global Build Information" + + jobs: + - job: Prerequistes_Job + timeoutInMinutes: 240 + + displayName: "Persist" + pool: + vmImage: ${{ parameters.agent_pool }} + + steps: + - checkout: none + + - script: |- + echo "agent_pool - ${{ parameters.agent_pool }}" + echo "release_build - ${{ parameters.release_build }}" + echo "prerelease_build_name - ${{ parameters.prerelease_build_name }}" + displayName: "[DEBUG] Display BuildAndTest Variables" + + - task: PythonScript@0 + displayName: "[IMPL] Create Global Build Information" + inputs: + scriptSource: inline + script: |- + from datetime import datetime + import json + import os + import sys + + values = {} + + release_build = sys.argv[1].lower() in ["true", "yes"] + values["is_release_build"] = release_build + + # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of + # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value. + # Therefore, a non-standard comparison is necessary in this case + if len(sys.argv) > 2 and sys.argv[2] != "{}(prerelease_build_name)".format("$"): + build_name = sys.argv[2] + is_prerelease_build = True + else: + is_prerelease_build = False + + if not release_build: + now = datetime.now() + + # The build should compare as: + # "manual" < "pipeline" < "preview" + build_name = "pipeline.{year}.{month}.{day}.{hour}.{minute}.{second}".format( + year=now.year, + month=now.month, + day=now.day, + hour=now.hour, + minute=now.minute, + second=now.second, + ) + else: + build_name = None + + values["is_prerelease_build"] = is_prerelease_build + values["build_name"] = build_name + + for k, v in values.items(): + print("{} = {}".format(k, v)) + + with open(os.path.join(r"$(Pipeline.Workspace)", "global_build_information.json"), "w") as f: + json.dump(values, f) + arguments: "${{ parameters.release_build }} ${{ parameters.prerelease_build_name }}" + timeoutInMinutes: 180 + + - task: PublishPipelineArtifact@0 + displayName: "Publish Artifacts" + inputs: + targetPath: "$(Pipeline.Workspace)/global_build_information.json" + artifactName: "Prerequisites" + timeoutInMinutes: 180 diff --git a/CI/Publish.yaml b/CI/Publish.yaml index 4878fa1..9e92ca7 100644 --- a/CI/Publish.yaml +++ b/CI/Publish.yaml @@ -6,13 +6,53 @@ trigger: none # User will need to specify buildId if publishing with non-triggered build stages: - - template: AzureUpload.stage_template.yaml - parameters: - agent_pool: vs2017-win2016 - project: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 - definition: 717 - subscription: AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87) - storage: featurizersbuild - containerName: Archive + - template: AzureUpload.stage_template.yaml + parameters: + agent_pool: windows-2019 + azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 + azure_dev_ops_build_definition_id: 717 + subscription: AI Infra Build (00c06639-6ee4-454e-8058-8d8b1703bd87) + storage: featurizersbuild + containerName: Archive + - template: NugetPublish.stage_template.yaml + parameters: + agent_pool: windows-2019 + azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 + azure_dev_ops_build_definition_id: 717 + service_connection: "NuGet" + - stage: TagSource_Stage + displayName: "Tag Sources (for release builds)" + dependsOn: + - AzureUpload_Stage + - NuGetPublish_Stage + + jobs: + - job: TagSource_Job + timeoutInMinutes: 240 + + displayName: "Tag" + + pool: + vmImage: windows-2019 + + steps: + - checkout: self + persistCredentials: true + clean: true + + - template: PublishInitialize.steps_template.yaml + parameters: + azure_dev_ops_project_id: 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 + azure_dev_ops_build_definition_id: 717 + + - script: |- + git config --global user.email "featurizer_dev@microsoft.com" + git config --global user.name "Azure DevOps" + + git tag "$(azure_product_version)" + git push origin "$(azure_product_version)" + displayName: "" + condition: or(eq(variables['azure_is_release_build'], 1), eq(variables['azure_is_prerelease_build'], 1)) + timeoutInMinutes: 180 diff --git a/CI/PublishInitialize.steps_template.yaml b/CI/PublishInitialize.steps_template.yaml new file mode 100644 index 0000000..d2d4b0b --- /dev/null +++ b/CI/PublishInitialize.steps_template.yaml @@ -0,0 +1,102 @@ +parameters: + azure_dev_ops_project_id: "" # 7b4fddf7-fb53-4b12-bfcb-f87ad3774a32 (The project GUID from which to download the pipeline artifacts) + azure_dev_ops_build_definition_id: "" # 717 (The definition ID of the build pipeline) + +steps: + # This build task will try to download artifacts from the triggering build. + # If there is no triggering build from the specified pipeline, it will download + # artifacts from the build specified in the options below + - task: PythonScript@0 + displayName: "[IMPL] Validate Variables" + inputs: + scriptSource: inline + script: |- + import sys + + triggeredBuildNumber = sys.argv[1] + buildId = sys.argv[2] + + # the intention here is to compare if var == '$(var)', but as the right side of == will be a string of + # "$(var)" if var is empty, or the exact value which is equal to var if var is assigned the value. + # Therefore, a non-standard comparison is necessary in this case + if triggeredBuildNumber == "{}(Build.TriggeredBy.BuildNumber)".format("$"): + if buildId == "{}(buildId)".format("$"): + raise ValueError('Please specify variable buildId since this is a publish with non-triggered build. e.g. set buildId = 104648') + + arguments: "$(Build.TriggeredBy.BuildNumber) $(buildId)" + + # Download from a specific id + - task: DownloadPipelineArtifact@2 + condition: and(succeeded(), eq(variables['Build.Reason'], 'Manual')) + displayName: "[IMPL] Download Pipeline Artifact for Manual build" + inputs: + buildType: "specific" + project: "${{ parameters.azure_dev_ops_project_id }}" + definition: "${{ parameters.azure_dev_ops_build_definition_id }}" + specificBuildWithTriggering: false + buildVersionToDownload: specific + pipelineId: "$(buildId)" + artifactName: "" + targetPath: "$(Pipeline.Workspace)" + timeoutInMinutes: 180 + + # Download from the triggering build + - task: DownloadPipelineArtifact@2 + condition: and(succeeded(), eq(variables['Build.Reason'], 'BuildCompletion')) + displayName: "[IMPL] Download Pipeline Artifact for Build triggered by BuildCompletion" + inputs: + buildType: "specific" + project: "${{ parameters.azure_dev_ops_project_id }}" + definition: "${{ parameters.azure_dev_ops_build_definition_id }}" + specificBuildWithTriggering: true + buildVersionToDownload: latestFromBranch + artifactName: "" + targetPath: "$(Pipeline.Workspace)" + timeoutInMinutes: 180 + + - task: PythonScript@0 + displayName: "[IMPL] Populate Stage Variables" + inputs: + scriptSource: inline + script: |- + import json + import os + + from collections import OrderedDict + + values = OrderedDict() + + # Version info comes from these places: + # + # - Global build Information + # - Build Instance Information + # + for info_filename in [ + os.path.join(r"$(Pipeline.Workspace)", "Prerequisites", "global_build_information.json"), + + # It doesn't matter which configuration that we use, as they will all contain + # identical builder info files. + os.path.join(r"$(Pipeline.Workspace)", "Windows - x64", "Builder", "product_version_information.json"), + ]: + assert os.path.isdir(os.path.dirname(info_filename)), info_filename + assert os.path.isfile(info_filename), info_filename + + with open(info_filename) as f: + content = json.load(f) + + for k, v in content.items(): + k = "azure_{}".format(k) + assert k not in values, (k, values[k], info_filename) + + if isinstance(v, bool): + v = 1 if v else 0 + elif v is None: + v = "" + + values[k] = v + + for k, v in values.items(): + print("{} = {}".format(k, v)) + print("##vso[task.setvariable variable={}]{}".format(k, v)) + + timeoutInMinutes: 180 diff --git a/CI/Readme.md b/CI/Readme.md index 1f25958..2d2d330 100644 --- a/CI/Readme.md +++ b/CI/Readme.md @@ -1,3 +1,62 @@ +Updating Build Information +========================== +Build information is updated in 1 of 2 ways, depending on which part of the version needs +to be modified. [Semantic Versioning terminology](https://semver.org) is used in the description +that follows. + + Example build number: ..[-][+] + Examples: 1.2.3-preview.1 == 1.2.3-preview.1+with.build.info + < 1.2.3-preview.2 < 1.2.3-preview.11 + < 1.2.3-zulu < 1.2.3 + +``, ``, and `` build values are defined in cmake files and should be updated there; +for an example, see src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake. + +`` and `` are defined when invoking the CI build. Additional information +(including steps to specify these values) follows. + +Build Types +=========== +The yaml build definitions in this directory are able to create difference build +types, each with different versioning, packaging, and publishing characteristics. +Build variables specified when invoking `CI.yaml` on Azure DevOps are used to +specific which build type is invoked. + +All version numbers conform to [Semantic Versioning standards](https://semver.org)*. + +Continuous Integration / Standard Build +--------------------------------------- +The build type invoked by default (when no Azure DevOps variables are specified). + + Azure DevOps Variables: None + Version: ..-cibuild....... + Example: 0.2.0-cibuild.2019.10.23.8.47.19.release + NuGet Store: https://aiinfra.visualstudio.com/DataPipelines/_packaging + +Prerelease +---------- +An unofficial, but public, release. + + Azure DevOps Variables: prerelease_build_name= + Version: ..- + Example (where prerelease_build_name=preview001): 0.2.0-preview001 + NuGet Store: https://www.nuget.org/ + +Release +------- +An official release. + + Azure DevOps Variables: release_build=True + Version: .. + Example: 0.2.0 + NuGet Store: https://www.nuget.org + +\* Note that NuGet doesn't support SemVer's build info (e.g. "+\") well, which is why +Continuous Integration datetime info is stored as prerelease info rather than build info. + +Azure DevOps Configuration Settings +=================================== + These Azure DevOps build definitions rely on the following Service Connections. To add/remove/view Service Connections within Azure DevOps: @@ -24,6 +83,14 @@ Used to sign binaries and packages. AAD APP Secret: PRSS Signing Authentication Certificate: Created through the [ESRP Portal](https://portal.esrp.microsoft.com/) +Nuget +----- +Used to push NuGet packages. + + Type: NuGet Service Connection + Feed URL: https://api.nuget.org/v3/index.json + ApiKey: Created on NuGet.org and then copied to the Azure DevOps connection dialog window for one-tiem usage + featurizersbuild ---------------- Used to pull the universal_linux Docker image for universal_linux builds. @@ -33,3 +100,8 @@ Used to pull the universal_linux Docker image for universal_linux builds. Connection name: featurizersbuild Azure subscription: AI Infra Build Azure container registry: featurizersbuild + +Azure DevOps Settings +===================== +The Publish build will tag sources upon successful completion. To do this, the Azure DevOps `Project Collection Build Service` account needs +contribute permissions to create tags for the appropriate repos. Follow the steps [here](https://docs.microsoft.com/en-us/azure/devops/pipelines/scripts/git-commands?view=azure-devops&tabs=yaml) to grant those permissions. diff --git a/src/FeaturizerPrep/SharedLibrary/Build.py b/src/FeaturizerPrep/SharedLibrary/Build.py index cd4e234..a81dd40 100644 --- a/src/FeaturizerPrep/SharedLibrary/Build.py +++ b/src/FeaturizerPrep/SharedLibrary/Build.py @@ -4,6 +4,7 @@ # ---------------------------------------------------------------------- """Builds the Shared Library""" +import datetime import hashlib import json import os @@ -28,6 +29,7 @@ _script_dir, _script_name = os.path.split(_script_fullpath) # ---------------------------------------------------------------------- CONFIGURATIONS = ["Debug", "Release"] +JSON_FILENAME = "Microsoft MLFeaturizers.FileAttributes.json" # ---------------------------------------------------------------------- @CommandLine.EntryPoint @@ -36,6 +38,9 @@ CONFIGURATIONS = ["Debug", "Release"] output_dir=CommandLine.DirectoryTypeInfo( ensure_exists=False, ), + prerelease_build_name=CommandLine.StringTypeInfo( + arity="?", + ), cmake_generator=CommandLine.StringTypeInfo( arity="?", ), @@ -44,13 +49,26 @@ CONFIGURATIONS = ["Debug", "Release"] def Build( configuration, output_dir, + release_build=False, + prerelease_build_name=None, + no_build_info=False, keep_temp_dir=False, - cmake_generator=(None if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux" else "Ninja"), + cmake_generator=( + None + if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") + == "universal_linux" + else "Ninja" + ), output_stream=sys.stdout, verbose=False, ): """Builds the Featurizer Shared Library""" + if release_build and prerelease_build_name: + raise CommandLine.UsageException( + "A prerelese build name cannot be provided with the 'release_build' flag", + ) + with StreamDecorator(output_stream).DoneManager( line_prefix="", prefix="\nResults: ", @@ -78,21 +96,54 @@ def Build( os.chdir(temp_directory) with CallOnExit(lambda: os.chdir(prev_dir)): + if not release_build: + if prerelease_build_name is None: + # This value should compare as: + # "manual" < "pipeline" + prerelease_build_name = "manual" + + if not no_build_info: + now = datetime.datetime.now() + + prerelease_build_name = "{prerelease_build_name}.{year}.{month}.{day}.{hour}.{minute}.{second}.{configuration}".format( + year=now.year, + month=now.month, + day=now.day, + hour=now.hour, + minute=now.minute, + second=now.second, + prerelease_build_name=prerelease_build_name, + configuration=configuration.lower(), + ) + activities = [ ( "Generating cmake Files", - 'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} "{this_dir}"'.format( - generator='-G "{}" '.format(cmake_generator) if cmake_generator else "", + 'cmake {generator}-DCMAKE_BUILD_TYPE={configuration} {prerelease_build_name} "{this_dir}"'.format( + generator='-G "{}" '.format( + cmake_generator, + ) if cmake_generator else "", temp_dir=temp_directory, configuration=configuration, this_dir=_script_dir, + prerelease_build_name="" if not prerelease_build_name else "-DPRODUCT_VERSION_PRERELEASE_INFO={}".format( + prerelease_build_name, + ), ), ), ("Building", "cmake --build ."), ] - if os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") == "universal_linux": - activities.append(("Verifying Universal Linux Binaries", 'libcheck libFeaturizers.so')) + if ( + os.getenv("DEVELOPMENT_ENVIRONMENT_REPOSITORY_CONFIGURATION") + == "universal_linux" + ): + activities.append( + ( + "Verifying Universal Linux Binaries", + "libcheck libFeaturizers.so", + ), + ) activities += [ ("Copying Binaries", _CopyBinaries), @@ -203,14 +254,14 @@ def Package( if len(build_dirs) > 1: dm.stream.write("Ensuring that build data matches...") with dm.stream.DoneManager() as ensure_dm: - ensure_dm.stream.write("Checking 'Featurizers.json'...") + ensure_dm.stream.write("Checking '{}'...".format(JSON_FILENAME)) with ensure_dm.stream.DoneManager() as this_dm: this_dm.result = ( 0 if _CompareFiles( this_dm.stream, *[ - os.path.join(build_dir, "Featurizers.json") + os.path.join(build_dir, JSON_FILENAME) for build_dir in build_dirs ] ) @@ -234,7 +285,7 @@ def Package( dm.stream.write("Reading build configuration...") with dm.stream.DoneManager() as this_dm: - json_filename = os.path.join(build_dirs[0], "Featurizers.json") + json_filename = os.path.join(build_dirs[0], JSON_FILENAME) if not os.path.isfile(json_filename): this_dm.stream.write( "ERROR: The filename '{}' does not exist.\n".format(json_filename), @@ -248,7 +299,11 @@ def Package( build_config["build_dir"] = build_dirs[0] build_config["data_dir"] = os.path.join(build_dirs[0], "Data", "**", "*.*") - build_config["package_id"] = build_config["product_name"].replace(" ", "") + build_config["package_id"] = build_config["product_name"].replace(" ", ".") + build_config["product_copyright"] = build_config["product_copyright"].replace( + "(C)", + "©", + ) # Generate the correct nuget file statements based on output in the build_dir dm.stream.write("Generating nuget file statements...") @@ -263,12 +318,10 @@ def Package( this_value_type = None if item == "Featurizers.dll": - if "x64" in build_dir: - this_value_type = "runtimes/win-x64/native" - elif "x86" in build_dir: + if "x86" in build_dir: this_value_type = "runtimes/win-x86/native" else: - assert False, build_dir + this_value_type = "runtimes/win-x64/native" elif item.startswith("libFeaturizers.so"): this_value_type = "runtimes/linux-x64/native" @@ -279,7 +332,11 @@ def Package( this_value_type = "runtimes/osx-x64/native" if this_value_type is not None: - assert value_type is None or this_value_type == value_type, (value_type, item, this_value_type) + assert value_type is None or this_value_type == value_type, ( + value_type, + item, + this_value_type, + ) value_type = this_value_type these_files.append(os.path.join(build_dir, item)) @@ -344,16 +401,16 @@ _nuget_template = textwrap.dedent( {package_id} - {product_version_major}.{product_version_minor}.{product_version_patch}.{product_version_revision} + {product_version_full} Microsoft microsoft MIT https://licenses.nuget.org/MIT true - {product_company_copyright} + {product_copyright} {product_bundle} - https://www.microsoft.com + https://aka.ms/MLFeaturizers @@ -370,13 +427,15 @@ _nuget_template = textwrap.dedent( # ---------------------------------------------------------------------- def _CopyBinaries(temp_directory, output_dir, output_stream): - output_files = ["Featurizers.json"] + output_files = [JSON_FILENAME] if CurrentShell.CategoryName == "Windows": output_files += ["Featurizers.dll", "Featurizers.pdb"] elif CurrentShell.CategoryName in ["Linux", "BSD"]: for item in os.listdir(temp_directory): - if item.startswith("libFeaturizers") and not os.path.splitext(item)[1] in [".a"]: + if item.startswith("libFeaturizers") and not os.path.splitext(item)[1] in [ + ".a" + ]: output_files.append(item) else: raise Exception("The Current Shell is not supported") diff --git a/src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake b/src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake index 17ef2fe..f37b1b9 100644 --- a/src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake +++ b/src/FeaturizerPrep/SharedLibrary/cmake/Featurizers.cmake @@ -2,86 +2,119 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License # ---------------------------------------------------------------------- -set(_project_name Featurizers) -set(_${_project_name}_version_major 0) -set(_${_project_name}_version_minor 1) -set(_${_project_name}_version_patch 0) -set(_${_project_name}_version ${_${_project_name}_version_major}.${_${_project_name}_version_minor}.${_${_project_name}_version_patch}) +cmake_minimum_required(VERSION 3.5.0) -project(${_project_name} VERSION ${_${_project_name}_version} LANGUAGES CXX) +set(_featurizers_this_path ${CMAKE_CURRENT_LIST_DIR} CACHE INTERNAL "") -get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY) +function(Impl) + # Defining a function here to introduce a new scope for local variables + set(_project_name Featurizers) + set(_version_major 0) # '1' in the release 1.2.3-alpha1+201910161322 + set(_version_minor 3) # '2' in the release 1.2.3-alpha1+201910161322 + set(_version_patch 5) # '3' in the release 1.2.3-alpha1+201910161322 + set(_version_prerelease_info "") # Optional 'alpha1' in the release 1.2.3-alpha1+201910161322 + set(_version_build_info "") # Optional '201910161322' in the release 1.2.3-alpha1+201910161322 -include(${_this_path}/generate_shared_library_attributes.cmake) -include(${_this_path}/../../Featurizers/cmake/FeaturizersCode.cmake) + set(_version ${_version_major}.${_version_minor}.${_version_patch}) -get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY) + # Alpha version components (which are supported in SemVer) present problems + # for cmake when the version is provided inline. However, things work as expected + # when setting the version as a property. + project(${_project_name} LANGUAGES CXX) + set(PROJECT_VERSION ${_version}) -generate_shared_library_attributes( - _shared_library_attributes_sources - NAME "Microsoft ML Featurizers" - FILE_DESCRIPTION "Microsoft ML Featurizers" - COMPANY_NAME "Microsoft Corporation" - COPYRIGHT "Copyright (C) Microsoft Corporation. All rights reserved." - VERSION_MAJOR ${_${_project_name}_version_major} - VERSION_MINOR ${_${_project_name}_version_minor} - VERSION_PATCH ${_${_project_name}_version_patch} - ICON "NA" -) + set(CMAKE_CXX_STANDARD 14) + set(CMAKE_CXX_STANDARD_REQUIRED ON) + set(CMAKE_CXX_EXTENSIONS OFF) -add_library(Featurizers SHARED - ${_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.h - ${_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.cpp - ${_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.h - ${_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.cpp - ${_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.h - ${_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.cpp + set(_includes "$ENV{INCLUDE}") + set(_libs "$ENV{LIB}") + set(CMAKE_MODULE_PATH "$ENV{DEVELOPMENT_ENVIRONMENT_CMAKE_MODULE_PATH}") - ${_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_Common.h - ${_this_path}/../GeneratedCode/SharedLibrary_Common.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_PointerTable.h - ${_this_path}/../GeneratedCode/SharedLibrary_PointerTable.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.cpp - ${_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.h - ${_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.cpp + if(NOT WIN32) + string(REPLACE ":" ";" CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH}") + string(REPLACE ":" ";" _includes "${_includes}") + string(REPLACE ":" ";" _libs "${_libs}") + endif() - ${_shared_library_attributes_sources} -) + include(CppCommon) + include(GenerateFileAttributes) -target_link_libraries(Featurizers PRIVATE - FeaturizersCode -) + generate_file_attributes( + _featurizers_file_attribute_sources + NAME "Microsoft MLFeaturizers" + COMPANY_NAME "Microsoft Corporation" + VERSION_MAJOR ${_version_major} + VERSION_MINOR ${_version_minor} + VERSION_PATCH ${_version_patch} + VERSION_PRERELEASE_INFO ${_version_prerelease_info} + VERSION_BUILD_INFO ${_version_build_info} + COPYRIGHT "(C) Microsoft Corporation. All rights reserved." + ) -target_include_directories(Featurizers PRIVATE - ${_this_path}/../GeneratedCode - ${_this_path}/../.. - ${_this_path}/../../Featurizers - ${_includes} -) + include(${_featurizers_this_path}/../../Featurizers/cmake/FeaturizersCode.cmake) -target_link_directories(Featurizers PRIVATE - ${_libs} -) + add_library( + ${_project_name} SHARED -set_target_properties( - ${_project_name} PROPERTIES - VERSION ${_${_project_name}_version} - SOVERSION ${_${_project_name}_version_major} -) + ${_featurizers_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.h + ${_featurizers_this_path}/../SharedLibrary_DateTimeFeaturizerCustom.cpp + ${_featurizers_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.h + ${_featurizers_this_path}/../SharedLibrary_RobustScalarFeaturizerCustom.cpp + ${_featurizers_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.h + ${_featurizers_this_path}/../SharedLibrary_TimeSeriesImputerFeaturizer.cpp + + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_CatImputerFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_Common.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_Common.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_DateTimeFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_HashOneHotVectorizerFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_ImputationMarkerFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_LabelEncoderFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MinMaxScalarFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_MissingDummiesFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_OneHotEncoderFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_PointerTable.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_PointerTable.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_RobustScalarFeaturizer.cpp + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.h + ${_featurizers_this_path}/../GeneratedCode/SharedLibrary_StringFeaturizer.cpp + + ${_featurizers_file_attribute_sources} + ) + + set_target_properties( + ${_project_name} PROPERTIES + VERSION ${_version} + SOVERSION ${_version_major} + ) + + target_link_libraries( + ${_project_name} PRIVATE + FeaturizersCode + ) + + target_include_directories( + ${_project_name} PRIVATE + ${_featurizers_this_path}/../GeneratedCode + ${_featurizers_this_path}/../.. + ${_featurizers_this_path}/../../Featurizers + ${_includes} + ) + + target_link_directories( + ${_project_name} PRIVATE + ${_libs} + ) +endfunction() + +Impl() diff --git a/src/FeaturizerPrep/SharedLibrary/cmake/generate_shared_library_attributes.cmake b/src/FeaturizerPrep/SharedLibrary/cmake/generate_shared_library_attributes.cmake deleted file mode 100644 index 40cffe9..0000000 --- a/src/FeaturizerPrep/SharedLibrary/cmake/generate_shared_library_attributes.cmake +++ /dev/null @@ -1,125 +0,0 @@ -# ---------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License -# ---------------------------------------------------------------------- -get_filename_component(_this_path ${CMAKE_CURRENT_LIST_FILE} DIRECTORY) - -# This code is based on code by halex2005, available at https://github.com/halex2005/CMakeHelpers/blob/master/generate_product_version.cmake, -# which is distributed under the MIT license (https://github.com/halex2005/CMakeHelpers/blob/master/LICENSE). - -include (CMakeParseArguments) - -# generate_product_version() function -# -# This function uses VersionInfo.in template file and VersionResource.rc file -# to generate WIN32 resource with version information and general resource strings. -# -# Usage: -# generate_product_version( -# SomeOutputResourceVariable -# NAME MyGreatProject -# ICON ${PATH_TO_APP_ICON} -# VERSION_MAJOR 2 -# VERSION_MINOR 3 -# VERSION_PATH ${BUILD_COUNTER} -# VERSION_REVISION ${BUILD_REVISION} -# ) -# where BUILD_COUNTER and BUILD_REVISION could be values from your CI server. -# -# You can use generated resource for your executable targets: -# add_executable(target-name ${target-files} ${SomeOutputResourceVariable}) -# -# You can specify resource strings in arguments: -# NAME - name of executable (no defaults, ex: Microsoft Word) -# BUNDLE - bundle (${NAME} is default, ex: Microsoft Office) -# ICON - path to application icon (${CMAKE_SOURCE_DIR}/product.ico by default) -# VERSION_MAJOR - 1 is default -# VERSION_MINOR - 0 is default -# VERSION_PATCH - 0 is default -# VERSION_REVISION - 0 is default -# COMPANY_NAME - your company name (no defaults) -# COMPANY_COPYRIGHT - ${COMPANY_NAME} (C) Copyright ${CURRENT_YEAR} is default -# COMMENTS - ${NAME} v${VERSION_MAJOR}.${VERSION_MINOR} is default -# ORIGINAL_FILENAME - ${NAME} is default -# INTERNAL_NAME - ${NAME} is default -# FILE_DESCRIPTION - ${NAME} is default -function(generate_shared_library_attributes outfiles) - set (options) - set (oneValueArgs - NAME - BUNDLE - ICON - VERSION_MAJOR - VERSION_MINOR - VERSION_PATCH - VERSION_REVISION - COMPANY_NAME - COMPANY_COPYRIGHT - COMMENTS - ORIGINAL_FILENAME - INTERNAL_NAME - FILE_DESCRIPTION) - set (multiValueArgs) - cmake_parse_arguments(PRODUCT "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) - - if (NOT PRODUCT_BUNDLE OR "${PRODUCT_BUNDLE}" STREQUAL "") - set(PRODUCT_BUNDLE "${PRODUCT_NAME}") - endif() - if (NOT PRODUCT_ICON OR "${PRODUCT_ICON}" STREQUAL "") - set(PRODUCT_ICON "${CMAKE_SOURCE_DIR}/product.ico") - endif() - - if (NOT PRODUCT_VERSION_MAJOR EQUAL 0 AND (NOT PRODUCT_VERSION_MAJOR OR "${PRODUCT_VERSION_MAJOR}" STREQUAL "")) - set(PRODUCT_VERSION_MAJOR 1) - endif() - if (NOT PRODUCT_VERSION_MINOR EQUAL 0 AND (NOT PRODUCT_VERSION_MINOR OR "${PRODUCT_VERSION_MINOR}" STREQUAL "")) - set(PRODUCT_VERSION_MINOR 0) - endif() - if (NOT PRODUCT_VERSION_PATCH EQUAL 0 AND (NOT PRODUCT_VERSION_PATCH OR "${PRODUCT_VERSION_PATCH}" STREQUAL "")) - set(PRODUCT_VERSION_PATCH 0) - endif() - if (NOT PRODUCT_VERSION_REVISION EQUAL 0 AND (NOT PRODUCT_VERSION_REVISION OR "${PRODUCT_VERSION_REVISION}" STREQUAL "")) - set(PRODUCT_VERSION_REVISION 0) - endif() - - if (NOT PRODUCT_COMPANY_COPYRIGHT OR "${PRODUCT_COMPANY_COPYRIGHT}" STREQUAL "") - string(TIMESTAMP PRODUCT_CURRENT_YEAR "%Y") - set(PRODUCT_COMPANY_COPYRIGHT "${PRODUCT_COMPANY_NAME} (C) Copyright ${PRODUCT_CURRENT_YEAR}") - endif() - if (NOT PRODUCT_COMMENTS OR "${PRODUCT_COMMENTS}" STREQUAL "") - set(PRODUCT_COMMENTS "${PRODUCT_NAME} v${PRODUCT_VERSION_MAJOR}.${PRODUCT_VERSION_MINOR}") - endif() - if (NOT PRODUCT_ORIGINAL_FILENAME OR "${PRODUCT_ORIGINAL_FILENAME}" STREQUAL "") - set(PRODUCT_ORIGINAL_FILENAME "${PRODUCT_NAME}") - endif() - if (NOT PRODUCT_INTERNAL_NAME OR "${PRODUCT_INTERNAL_NAME}" STREQUAL "") - set(PRODUCT_INTERNAL_NAME "${PRODUCT_NAME}") - endif() - if (NOT PRODUCT_FILE_DESCRIPTION OR "${PRODUCT_FILE_DESCRIPTION}" STREQUAL "") - set(PRODUCT_FILE_DESCRIPTION "${PRODUCT_NAME}") - endif() - - set (_SharedLibraryHeaderFile ${CMAKE_CURRENT_BINARY_DIR}/SharedLibraryAttributes.h) - set (_SharedLibraryResourceFile ${CMAKE_CURRENT_BINARY_DIR}/SharedLibraryAttributes.rc) - - configure_file( - ${_this_path}/../Featurizers.in.json - ${CMAKE_CURRENT_BINARY_DIR}/Featurizers.json - @ONLY - ) - - configure_file( - ${_this_path}/../SharedLibraryAttributes.in.h - ${_SharedLibraryHeaderFile} - @ONLY - ) - - configure_file( - ${_this_path}/../SharedLibraryAttributes.rc - ${_SharedLibraryResourceFile} - COPYONLY - ) - - list(APPEND ${outfiles} ${_SharedLibraryHeaderFile} ${_SharedLibraryResourceFile}) - set (${outfiles} ${${outfiles}} PARENT_SCOPE) -endfunction()