microsoft
/
FeaturizersLibrary
зеркало из https://github.com/microsoft/FeaturizersLibrary.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Added MSVC 2017 builds (#29)

This commit is contained in:
David Brownell 2019-12-12 08:08:48 -08:00 коммит произвёл GitHub
Родитель cd596fadd2
Коммит a2a2a40de2
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
17 изменённых файлов: 520 добавлений и 336 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

56
CI/CI.yaml
Просмотреть файл

@ -15,6 +15,10 @@ resources:
image: universal_linux
endpoint: featurizersbuild
- container: centos7
image: centos7
endpoint: featurizersbuild
# Note that any variable set in this file will always override a value set within the editor.
# Therefore, do not set default values here but instead do it in an initialization script.
#
@ -42,6 +46,7 @@ stages:
agent_pool_is_custom: true
configuration: x64
esrp_connected_service_name: "ESRP CodeSigning Connection"
job_display_name: "Windows-x64-Clang 8"
operating_system: Windows
tester_code_coverage: True
@ -51,6 +56,7 @@ stages:
agent_pool_is_custom: true
configuration: x86
esrp_connected_service_name: "ESRP CodeSigning Connection"
job_display_name: "Windows-x86-Clang 8"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
@ -59,13 +65,15 @@ stages:
agent_pool_is_custom: true
configuration: system_compiler
esrp_connected_service_name: "ESRP CodeSigning Connection"
job_display_name: "Ubuntu 16.04-x64-GCC"
operating_system: Linux
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: macOS-10.14
configuration: system_compiler
esrp_connected_service_name: "ESRP CodeSigning Connection"
job_display_name: "MacOS 10.14-x64-Clang"
job_name_unique_id: "1014"
operating_system: MacOS
# ----------------------------------------------------------------------
@ -77,7 +85,7 @@ stages:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64
job_suffix: " (test only)"
job_display_name: "[Test Only] Windows-x64-Clang 8"
operating_system: Windows
test_only: true
tester_release_only: true
@ -86,23 +94,51 @@ stages:
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64_MSVC
configuration: x64_MSVC_2019
job_display_name: "Windows-x64-MSVC 2019"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x86_MSVC
configuration: x86_MSVC_2019
job_display_name: "Windows-x86-MSVC 2019"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64
configuration: x64_MSVC_2017
job_display_name: "Windows-x64-MSVC 2017"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x86_MSVC_2017
job_display_name: "Windows-x86-MSVC 2017"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64_Clang_8
job_display_name: "Ubuntu 16.04-x64-Clang 8"
operating_system: Linux
# TODO: Disabled for now - template: "templates/BuildAndTest.job_template.yaml"
# TODO: Disabled for now parameters:
# TODO: Disabled for now agent_pool: FeaturizersLibrary
# TODO: Disabled for now agent_pool_is_custom: true
# TODO: Disabled for now agent_pool_container: centos7
# TODO: Disabled for now configuration: system_compiler
# TODO: Disabled for now job_display_name: "CentOS 7-x64-GCC"
# TODO: Disabled for now operating_system: Linux
# Note that the universal_linux build is generating code that produces runtime errors. This build remains to ensure
# that the code compiles on older platforms.
- template: "templates/BuildAndTest.job_template.yaml"
@ -112,9 +148,17 @@ stages:
agent_pool_container: universal_linux
configuration: universal_linux
operating_system: Linux
job_suffix: " (build only)"
job_display_name: "[Build Only] Holy Build Box-x64-GCC"
tester_build_only: true
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: macOS-10.13
configuration: system_compiler
job_display_name: "MacOS 10.13-x64-Clang"
job_name_unique_id: "1013"
operating_system: MacOS
- stage: PostBuild_Stage
displayName: "Post Build:"
dependsOn: BuildAndTest_Stage

55
CI/PR.yaml
Просмотреть файл

@ -11,6 +11,10 @@ resources:
image: universal_linux
endpoint: featurizersbuild
- container: centos7
image: centos7
endpoint: featurizersbuild
stages:
- template: "templates/BuildAndTest.prologue_stage_template.yaml"
parameters:
@ -31,6 +35,7 @@ stages:
agent_pool_is_custom: true
configuration: x64
operating_system: Windows
job_display_name: "Windows-x64-Clang 8"
tester_code_coverage: True
- template: "templates/BuildAndTest.job_template.yaml"
@ -38,6 +43,7 @@ stages:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x86
job_display_name: "Windows-x86-Clang 8"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
@ -45,12 +51,15 @@ stages:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: system_compiler
job_display_name: "Ubuntu 16.04-x64-GCC"
operating_system: Linux
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: macOS-10.14
configuration: system_compiler
job_display_name: "MacOS 10.14-x64-Clang"
job_name_unique_id: "1014"
operating_system: MacOS
# ----------------------------------------------------------------------
@ -62,7 +71,7 @@ stages:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64
job_suffix: " (test only)"
job_display_name: "[Test Only] Windows-x64-Clang 8"
operating_system: Windows
test_only: true
tester_release_only: true
@ -71,23 +80,51 @@ stages:
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64_MSVC
configuration: x64_MSVC_2019
job_display_name: "Windows-x64-MSVC 2019"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x86_MSVC
configuration: x86_MSVC_2019
job_display_name: "Windows-x86-MSVC 2019"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64
configuration: x64_MSVC_2017
job_display_name: "Windows-x64-MSVC 2017"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x86_MSVC_2017
job_display_name: "Windows-x86-MSVC 2017"
operating_system: Windows
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: FeaturizersLibrary
agent_pool_is_custom: true
configuration: x64_Clang_8
job_display_name: "Ubuntu 16.04-x64-Clang 8"
operating_system: Linux
# TODO: Disabled for now - template: "templates/BuildAndTest.job_template.yaml"
# TODO: Disabled for now parameters:
# TODO: Disabled for now agent_pool: FeaturizersLibrary
# TODO: Disabled for now agent_pool_is_custom: true
# TODO: Disabled for now agent_pool_container: centos7
# TODO: Disabled for now configuration: system_compiler
# TODO: Disabled for now job_display_name: "CentOS 7-x64-GCC"
# TODO: Disabled for now operating_system: Linux
# Note that the universal_linux build is generating code that produces runtime errors. This build remains to ensure
# that the code compiles on older platforms.
- template: "templates/BuildAndTest.job_template.yaml"
@ -97,9 +134,17 @@ stages:
agent_pool_container: universal_linux
configuration: universal_linux
operating_system: Linux
job_suffix: " (build only)"
job_display_name: "[Build Only] Holy Build Box-x64-GCC"
tester_build_only: true
- template: "templates/BuildAndTest.job_template.yaml"
parameters:
agent_pool: macOS-10.13
configuration: system_compiler
job_display_name: "MacOS 10.13-x64-Clang"
job_name_unique_id: "1013"
operating_system: MacOS
- stage: PostBuild_Stage
displayName: "Post Build:"
dependsOn: BuildAndTest_Stage

283
CI/docker/Build.py Normal file
Просмотреть файл

@ -0,0 +1,283 @@
# ----------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License
# ----------------------------------------------------------------------
"""Tools to create and publish a docker image that can be used to create universal linux binaries."""
import os
import sys
import six
import CommonEnvironment
from CommonEnvironment import BuildImpl
from CommonEnvironment.CallOnExit import CallOnExit
from CommonEnvironment import CommandLine
from CommonEnvironment import Process
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
DOCKER_USER_NAME = "featurizersbuild"
IMAGES = [
os.path.basename(potential_dir)
for potential_dir in [
os.path.join(_script_dir, item) for item in os.listdir(_script_dir)
]
if os.path.isdir(potential_dir)
]
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
image_name=CommandLine.EnumTypeInfo(IMAGES),
tag=CommandLine.StringTypeInfo(
arity="*",
),
output_stream=None,
)
def Build(
image_name,
tag=None,
output_stream=sys.stdout,
verbose=False,
):
"""Creates a docker image"""
tags = tag
del tag
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
prev_dir = os.getcwd()
os.chdir(os.path.join(_script_dir, image_name))
with CallOnExit(lambda: os.chdir(prev_dir)):
image_name = "{}/{}".format(DOCKER_USER_NAME, image_name)
dm.stream.write("Building docker image...")
with dm.stream.DoneManager(
line_prefix=" ",
suffix="\n",
) as this_dm:
this_dm.result = Process.Execute(
"docker build --tag {} .".format(image_name),
this_dm.stream,
)
if this_dm.result != 0:
return this_dm.result
if tags:
dm.stream.write("Applying tags...")
with dm.stream.DoneManager() as tag_dm:
for index, tag in enumerate(tags):
tag_dm.stream.write(
"'{}' ({} of {})...".format(tag, index + 1, len(tags)),
)
with tag_dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
"docker tag {image_name} {image_name}:{tag}".format(
image_name=image_name,
tag=tag,
),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
image_name=CommandLine.EnumTypeInfo(IMAGES),
tag=CommandLine.StringTypeInfo(
arity="*",
),
output_stream=None,
)
def Clean(
image_name,
tag=None,
all=False,
output_stream=sys.stdout,
):
"""Cleans previously built content"""
tags = tag
del tag
if all and tags:
raise CommandLine.UsageException(
"Individual tag values should not be provided when 'all' is provided on the command line",
)
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
prev_dir = os.getcwd()
os.chdir(os.path.join(_script_dir, image_name))
with CallOnExit(lambda: os.chdir(prev_dir)):
image_name = "{}/{}".format(DOCKER_USER_NAME, image_name)
if all:
image_id = _GetImageId(image_name, dm)
command_line = "docker image rm --force {}".format(image_id)
else:
command_line = "docker image rm {}".format(image_name)
dm.stream.write("Removing docker image(s)...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(command_line)
if this_dm.result != 0:
# If the user provided tags, it may be that they have already deleted this
# image, in which case this is not an error.
this_dm.stream.write(output)
if tags:
this_dm.result = 0
else:
return this_dm.result
if tags:
dm.stream.write("Removing tags...")
with dm.stream.DoneManager() as tag_dm:
for index, tag in enumerate(tags):
tag_dm.stream.write(
"'{}' ({} of {})...".format(tag, index + 1, len(tags)),
)
with tag_dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
"docker image rm {}:{}".format(image_name, tag),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint(
registry_name=CommandLine.EntryPoint.Parameter(
"Name of the container registry to push to (e.g. 'featurizersbuild.azurecr.io')",
),
)
@CommandLine.Constraints(
image_name=CommandLine.EnumTypeInfo(IMAGES),
registry_name=CommandLine.StringTypeInfo(),
tag=CommandLine.StringTypeInfo(
arity="?",
),
output_stream=None,
)
def Publish(
image_name,
registry_name,
tag=None,
output_stream=sys.stdout,
):
"""Publishes previously built content to a docker registry"""
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
prev_dir = os.getcwd()
os.chdir(os.path.join(_script_dir, image_name))
with CallOnExit(lambda: os.chdir(prev_dir)):
image_name = "{}/{}".format(DOCKER_USER_NAME, image_name)
image_id = _GetImageId(
image_name,
dm,
tag=tag,
)
new_image_name = "{}/{}".format(registry_name, image_name.split("/")[-1])
if new_image_name is None:
assert dm.result != 0
return dm.result
dm.stream.write("Renaming image...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
"docker tag {} {}{}".format(
image_id,
new_image_name,
":{}".format(tag) if tag else "",
),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
dm.stream.write("Pushing image...")
with dm.stream.DoneManager(
line_prefix=" ",
) as this_dm:
this_dm.result = Process.Execute(
"docker push {}".format(new_image_name),
this_dm.stream,
)
if this_dm.result != 0:
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def _GetImageId(
image_name,
dm,
tag=None,
):
dm.stream.write("Retrieving image id...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
"docker images {}{} --quiet".format(
image_name,
":{}".format(tag) if tag else "",
),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return None
lines = output.strip().split("\n")
return lines[0]
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
try:
sys.exit(
BuildImpl.Main(
BuildImpl.Configuration(
"universal_linux_build",
requires_output_dir=False,
),
),
)
except KeyboardInterrupt:
pass

0
CI/universal_linux_image/Build.py-ignore → CI/docker/Build.py-ignore
Просмотреть файл

13
CI/universal_linux_image/Readme.md → CI/docker/Readme.md
Просмотреть файл

@ -1,18 +1,11 @@
Publishing a New Image for CI
=============================
The Azure DevOps functionality that makes containers available during builds
assumes that `sudo` is available on the image. Unfortunately, the image that
we use for universal linux builds (`phusion/holy-build-box-64`) does not enable
`sudo` by default.
Building the Image
------------------
==================
These are the steps required to build an image locally and push it to an Azure
Container Registry so that it can be used during builds.
1) Build the image:
python Build.py Build
python Build.py Build <image_name>
2) Login to the ACR:
@ -27,7 +20,7 @@ Container Registry so that it can be used during builds.
3) Publish the image:
python Build.py Publish featurizersbuild.azurecr.io
python Build.py Publish <image_name> featurizersbuild.azurecr.io
Note that ../PR.yaml and ../CI.yaml (and potentially other files) are configured to use
this image in this Container Registry.

21
CI/docker/centos7/Dockerfile Normal file
Просмотреть файл

@ -0,0 +1,21 @@
FROM centos:7
# Install packages on the image:
#
# - git: required during bootstrapping
# - sudo: required by Azure DevOps
# - various development tools: required by everything
#
RUN yum install -y \
git \
sudo \
gcc-4.8.5 \
gcc-c++-4.8.5 \
kernel-devel-3.10.0 \
make-3.82
# All of the build tools want to use Ninja as the cmake generator, but ninja doesn't work in this older
# environment. Rather than specifying a generator, use whichever happens to be the default on the machine.
ENV DEVELOPMENT_ENVIRONMENT_CPP_USE_DEFAULT_CMAKE_GENERATOR=1 \
CXX=gcc \
CC=gcc

0
CI/universal_linux_image/Dockerfile → CI/docker/universal_linux/Dockerfile
Просмотреть файл

9
CI/templates/BuildAndTest.job_template.yaml
Просмотреть файл

@ -7,7 +7,8 @@ parameters:
agent_pool_is_custom: False # Set this value to True if the agent is part of a custom pool
configuration: "" # Environment Activation Configuration
esrp_connected_service_name: "" # Name of service connection to use for code signing (or empty string if code signing is not required)
job_suffix: "" # Optional value to distinguish between standard- and custom-builds if both configurations are invoked as part of the same stage
job_display_name: ""
job_name_unique_id: ""
operating_system: "" # Windows|Linux|MacOS
test_only: False # Set to true to only run tests (no build)
tester_code_coverage: False # Enable code coverage for tests
@ -20,8 +21,8 @@ jobs:
agent_pool: ${{ parameters.agent_pool }}
agent_pool_container: ${{ parameters.agent_pool_container }}
agent_pool_is_custom: ${{ parameters.agent_pool_is_custom }}
job_name: BuildAndTest_${{ parameters.operating_system }}_${{ parameters.configuration }}_${{ parameters.agent_pool_container }}_${{ parameters.test_only }}_Job
job_display_name: "${{ parameters.operating_system }} - ${{ parameters.configuration }}${{ parameters.job_suffix }}"
job_name: BuildAndTest_${{ parameters.operating_system }}_${{ parameters.configuration }}_${{ parameters.agent_pool_container }}_${{ parameters.test_only }}_Job${{ parameters.job_name_unique_id }}
job_display_name: "${{ parameters.job_display_name }}"
operating_system: ${{ parameters.operating_system }}
steps:
- template: BuildAndTest.steps_template.yaml
@ -30,7 +31,7 @@ jobs:
esrp_connected_service_name: ${{ parameters.esrp_connected_service_name }}
operating_system: ${{ parameters.operating_system }}
test_only: ${{ parameters.test_only }}
artifact_suffix: ${{ parameters.job_suffix }}
artifact_name: "${{ parameters.job_display_name }}"
tester_code_coverage: ${{ parameters.tester_code_coverage }}
tester_release_only: ${{ parameters.tester_release_only }}
tester_build_only: ${{ parameters.tester_build_only }}

10
CI/templates/BuildAndTest.package_steps_template.yaml
Просмотреть файл

@ -13,11 +13,11 @@ parameters:
- Release
artifact_configurations:
- "Windows - x64"
- "Windows - x86"
# - 'Linux - universal_linux'
- "Linux - system_compiler"
- "MacOS - system_compiler"
- "Windows-x64-Clang 8"
- "Windows-x86-Clang 8"
# - 'Linux-Holy Build Box-GCC'
- "Ubuntu 16.04-x64-GCC"
- "MacOS 10.14-x64-Clang"
steps:
- task: PythonScript@0

11
CI/templates/BuildAndTest.prologue_stage_template.yaml
Просмотреть файл

@ -106,7 +106,16 @@ stages:
)
if build_name and git_branch != "master":
build_name = "%s-%s" % (git_branch, build_name)
# Update the branch name to only include semantic version-supported chars
sem_ver_chars = []
for c in git_branch:
if not c.isalnum() and c != '.':
c = '.'
sem_ver_chars.append(c)
build_name = "%s.%s" % (''.join(sem_ver_chars), build_name)
values["git_branch"] = git_branch
values["is_prerelease_build"] = is_prerelease_build

6
CI/templates/BuildAndTest.steps_template.yaml
Просмотреть файл

@ -7,7 +7,7 @@ parameters:
operating_system: "" # Windows|Linux|MacOS
test_only: False # Set to true to only run tests (no build)
build_only: False # Set to true to only build (no test execution)
artifact_suffix: "" # Optional value to distinguish between standard- and test-only-builds if both configurations are invoked as part of the same stage
artifact_name: ""
tester_code_coverage: False # Enable code coverage for tests
tester_release_only: False # Only run release tests
tester_build_only: False # Set to true to only build tests (no test execution)
@ -43,7 +43,7 @@ steps:
operating_system: ${{ parameters.operating_system }}
test_only: ${{ parameters.test_only }}
build_only: ${{ parameters.build_only }}
artifact_suffix: ${{ parameters.artifact_suffix }}
artifact_name: ${{ parameters.artifact_name }}
tester_code_coverage: ${{ parameters.tester_code_coverage }}
tester_release_only: ${{ parameters.tester_release_only }}
tester_build_only: ${{ parameters.tester_build_only }}
@ -220,4 +220,4 @@ steps:
inputs:
targetPath: $(pipeline_artifacts_directory)
artifactName: "${{ parameters.operating_system }} - ${{ parameters.configuration }}${{ parameters.artifact_suffix }}"
artifactName: "${{ parameters.artifact_name }}"

237
CI/universal_linux_image/Build.py
Просмотреть файл

@ -1,237 +0,0 @@
# ----------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License
# ----------------------------------------------------------------------
"""Tools to create and publish a docker image that can be used to create universal linux binaries."""
import os
import sys
import six
import CommonEnvironment
from CommonEnvironment import BuildImpl
from CommonEnvironment.CallOnExit import CallOnExit
from CommonEnvironment import CommandLine
from CommonEnvironment import Process
from CommonEnvironment.StreamDecorator import StreamDecorator
# ----------------------------------------------------------------------
_script_fullpath = CommonEnvironment.ThisFullpath()
_script_dir, _script_name = os.path.split(_script_fullpath)
# ----------------------------------------------------------------------
DEFAULT_IMAGE_NAME = "featurizersbuild/universal_linux"
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
tag=CommandLine.StringTypeInfo(
arity="*",
),
output_stream=None,
)
def Build(
image_name=DEFAULT_IMAGE_NAME,
tag=None,
output_stream=sys.stdout,
verbose=False,
):
"""Creates a docker image"""
tags = tag
del tag
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
dm.stream.write("Building docker image...")
with dm.stream.DoneManager(
line_prefix=" ",
suffix="\n",
) as this_dm:
prev_dir = os.getcwd()
os.chdir(_script_dir)
with CallOnExit(lambda: os.chdir(prev_dir)):
this_dm.result = Process.Execute('docker build --tag {} .'.format(image_name), this_dm.stream)
if this_dm.result != 0:
return this_dm.result
if tags:
dm.stream.write("Applying tags...")
with dm.stream.DoneManager() as tag_dm:
for index, tag in enumerate(tags):
tag_dm.stream.write("'{}' ({} of {})...".format(tag, index + 1, len(tags)))
with tag_dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
'docker tag {image_name} {image_name}:{tag}'.format(
image_name=image_name,
tag=tag,
),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint
@CommandLine.Constraints(
tag=CommandLine.StringTypeInfo(
arity="*",
),
output_stream=None,
)
def Clean(
image_name=DEFAULT_IMAGE_NAME,
tag=None,
all=False,
output_stream=sys.stdout,
):
"""Cleans previously built content"""
tags = tag
del tag
if all and tags:
raise CommandLine.UsageException("Individual tag values should not be provided when 'all' is provided on the command line")
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
if all:
image_id = _GetImageId(image_name, dm)
command_line = 'docker image rm --force {}'.format(image_id)
else:
command_line = 'docker image rm {}'.format(image_name)
dm.stream.write("Removing docker image(s)...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(command_line)
if this_dm.result != 0:
# If the user provided tags, it may be that they have already deleted this
# image, in which case this is not an error.
this_dm.stream.write(output)
if tags:
this_dm.result = 0
else:
return this_dm.result
if tags:
dm.stream.write("Removing tags...")
with dm.stream.DoneManager() as tag_dm:
for index, tag in enumerate(tags):
tag_dm.stream.write("'{}' ({} of {})...".format(tag, index + 1, len(tags)))
with tag_dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute('docker image rm {}:{}'.format(image_name, tag))
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
@CommandLine.EntryPoint(
registry_name=CommandLine.EntryPoint.Parameter("Name of the container registry to push to (e.g. 'featurizersbuild.azurecr.io')"),
)
@CommandLine.Constraints(
registry_name=CommandLine.StringTypeInfo(),
tag=CommandLine.StringTypeInfo(
arity="?",
),
output_stream=None,
)
def Publish(
registry_name,
image_name=DEFAULT_IMAGE_NAME,
tag=None,
output_stream=sys.stdout,
):
"""Publishes previously built content to a docker registry"""
with StreamDecorator(output_stream).DoneManager(
line_prefix="",
prefix="\nResults: ",
suffix="\n",
) as dm:
image_id = _GetImageId(
image_name,
dm,
tag=tag,
)
new_image_name = "{}/{}".format(registry_name, image_name.split("/")[-1])
if new_image_name is None:
assert dm.result != 0
return dm.result
dm.stream.write("Renaming image...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute(
'docker tag {} {}{}'.format(
image_id,
new_image_name,
":{}".format(tag) if tag else "",
),
)
if this_dm.result != 0:
this_dm.stream.write(output)
return this_dm.result
dm.stream.write("Pushing image...")
with dm.stream.DoneManager(
line_prefix=" ",
) as this_dm:
this_dm.result = Process.Execute('docker push {}'.format(new_image_name), this_dm.stream)
if this_dm.result != 0:
return this_dm.result
return dm.result
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
def _GetImageId(
image_name,
dm,
tag=None,
):
dm.stream.write("Retrieving image id...")
with dm.stream.DoneManager() as this_dm:
this_dm.result, output = Process.Execute('docker images {}{} --quiet'.format(image_name, ":{}".format(tag) if tag else ""))
if this_dm.result != 0:
this_dm.stream.write(output)
return None
lines = output.strip().split("\n")
return lines[0]
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
# ----------------------------------------------------------------------
if __name__ == "__main__":
try:
sys.exit(
BuildImpl.Main(
BuildImpl.Configuration(
"universal_linux_build",
requires_output_dir=False,
),
),
)
except KeyboardInterrupt:
pass

99
Setup_custom.py
Просмотреть файл

@ -12,7 +12,6 @@
# |
# ----------------------------------------------------------------------
import copy
import os
import sys
@ -87,44 +86,56 @@ def GetDependencies():
d = OrderedDict()
d["x64"] = Configuration(
"Builds using Clang for a x64 architecture",
[
Dependency(
"3DE9F3430E494A6C8429B26A1503C895",
"Common_cpp_Clang_8",
"x64-ex",
"https://github.com/davidbrownell/Common_cpp_Clang_8.git",
),
],
)
architectures = ["x64"]
if CurrentShell.CategoryName == "Windows":
d["x86"] = Configuration(
"Builds using Clang for a x86 architecture",
[
Dependency(
"3DE9F3430E494A6C8429B26A1503C895",
"Common_cpp_Clang_8",
"x86-ex",
"https://github.com/davidbrownell/Common_cpp_Clang_8.git",
),
],
)
architectures.append("x86")
for architecture in ["x64", "x86"]:
d["{}_MSVC".format(architecture)] = Configuration(
"Builds using MSVC 2019 for a {} architecture".format(architecture),
# Clang
for version, guid in [("8", "3DE9F3430E494A6C8429B26A1503C895")]:
for architecture in architectures:
d["{}_Clang_{}".format(architecture, version)] = Configuration(
"Builds using Clang {} for a {} architecture".format(
version,
architecture,
),
[
Dependency(
"AB7D87C49C2449F79D9F42E5195030FD",
"Common_cpp_MSVC_2019",
architecture,
"https://github.com/davidbrownell/Common_cpp_MSVC_2019.git",
guid,
"Common_cpp_Clang_{}".format(version),
"{}-ex".format(architecture),
"https://github.com/davidbrownell/Common_cpp_Clang_{}".format(
version,
),
),
],
)
# MSVC
if CurrentShell.CategoryName == "Windows":
for version, guid in [
("2019", "AB7D87C49C2449F79D9F42E5195030FD"),
("2017", "8FC8ACE80A594D2EA996CAC5DBFFEBBC"),
]:
for architecture in architectures:
d["{}_MSVC_{}".format(architecture, version)] = Configuration(
"Builds using MSVC {} for a {} architecture".format(
version,
architecture,
),
[
Dependency(
guid,
"Common_cpp_MSVC_{}".format(version),
architecture,
"https://github.com/davidbrownell/Common_cpp_MSVC_{}.git".format(
version,
),
),
],
)
# Misc
d["system_compiler"] = Configuration(
"Enables basic C++ tools (cmake, ninja, doxygen, etc.)",
[
@ -141,13 +152,27 @@ def GetDependencies():
d["universal_linux"] = Configuration(
"Builds using the Holy Build Box Docker Image (phusion/holy-build-box-64). More info at http://phusion.github.io/holy-build-box/",
[
Dependency("F33C43DA6BB54336A7573B39509CDAD7",
"Common_cpp_Common",
"x64",
"https://github.com/davidbrownell/Common_cpp_Common.git",
),
],
)
Dependency(
"F33C43DA6BB54336A7573B39509CDAD7",
"Common_cpp_Common",
"x64",
"https://github.com/davidbrownell/Common_cpp_Common.git",
),
],
)
# Set the defaults
d["x64_Clang"] = d["x64_Clang_8"]
if "x86" in architectures:
d["x86_Clang"] = d["x86_Clang_8"]
if CurrentShell.CategoryName == "Windows":
d["x64_MSVC"] = d["x64_MSVC_2019"]
d["x86_MSVC"] = d["x86_MSVC_2019"]
d["x64"] = d["x64_Clang"]
if "x86" in architectures:
d["x86"] = d["x86_Clang"]
return d

4
src/Featurizers/Components/Details/EstimatorTraits.h
Просмотреть файл

@ -33,13 +33,13 @@ private:
template <typename U>
static constexpr std::true_type Check(
U *,
std::enable_if_t<
typename std::enable_if<
std::is_same<
decltype(std::declval<U>().has_created_transformer(std::declval<ArgTs>()...)),
ReturnT
>::value,
void *
>
>::type
);
public:

36
src/Featurizers/Components/Details/PipelineExecutionEstimatorImpl_details.h
Просмотреть файл

@ -42,7 +42,7 @@ template <int N, typename EstimatorTupleT>
struct ValidateEstimators<
N,
EstimatorTupleT,
std::enable_if_t<N != std::tuple_size<EstimatorTupleT>::value - 1>
typename std::enable_if<N != std::tuple_size<EstimatorTupleT>::value - 1>::type
> {
using ThisEstimator = typename std::tuple_element<N, EstimatorTupleT>::type;
using NextEstimator = typename std::tuple_element<N + 1, EstimatorTupleT>::type;
@ -67,7 +67,7 @@ template <int N, typename EstimatorTupleT>
struct ValidateEstimators<
N,
EstimatorTupleT,
std::enable_if_t<N == std::tuple_size<EstimatorTupleT>::value - 1>
typename std::enable_if<N == std::tuple_size<EstimatorTupleT>::value - 1>::type
> {
static constexpr bool const value = true;
};
@ -399,8 +399,8 @@ template <int N, typename EstimatorTupleT>
class EstimatorChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N == std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>
typename std::enable_if<N == std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>::type
> :
public EstimatorChainElement_TerminalMixin<
EstimatorChainElement<N, EstimatorTupleT>,
@ -464,8 +464,8 @@ template <int N, typename EstimatorTupleT>
class EstimatorChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N == std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>
typename std::enable_if<N == std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>::type
> :
public EstimatorChainElement_TerminalMixin<
EstimatorChainElement<N, EstimatorTupleT>,
@ -547,8 +547,8 @@ template <int N, typename EstimatorTupleT>
class EstimatorChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N != std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>
typename std::enable_if<N != std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>::type
> :
public EstimatorChainElement_IntraMixin<
EstimatorChainElement<N, EstimatorTupleT>,
@ -629,8 +629,8 @@ template <int N, typename EstimatorTupleT>
class EstimatorChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N != std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>
typename std::enable_if<N != std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>::type
> :
public EstimatorChainElement_IntraMixin<
EstimatorChainElement<N, EstimatorTupleT>,
@ -798,8 +798,8 @@ template <int N, typename EstimatorTupleT>
class TransformerChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N == std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>
typename std::enable_if<N == std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>::type
> {
private:
// ----------------------------------------------------------------------
@ -842,8 +842,8 @@ template <int N, typename EstimatorTupleT>
class TransformerChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N == std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>
typename std::enable_if<N == std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>::type
> {
private:
// ----------------------------------------------------------------------
@ -906,8 +906,8 @@ template <int N, typename EstimatorTupleT>
class TransformerChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N != std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>
typename std::enable_if<N != std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value == false>::type
> :
private TransformerChainElement<N + 1, EstimatorTupleT> {
private:
@ -958,8 +958,8 @@ template <int N, typename EstimatorTupleT>
class TransformerChainElement<
N,
EstimatorTupleT,
std::enable_if_t<N != std::tuple_size<EstimatorTupleT>::value - 1>,
std::enable_if_t<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>
typename std::enable_if<N != std::tuple_size<EstimatorTupleT>::value - 1>::type,
typename std::enable_if<IsTransformerEstimator<typename std::tuple_element<N, EstimatorTupleT>::type>::value>::type
> :
private TransformerChainElement<N + 1, EstimatorTupleT> {
private:

4
src/Featurizers/Components/GrainEstimatorImpl.h
Просмотреть файл

@ -252,7 +252,7 @@ class GrainEstimatorImpl<
GrainT,
EstimatorT,
MaxNumTrainingItemsV,
std::enable_if_t<Details::IsTransformerEstimator<EstimatorT>::value == false>
typename std::enable_if<Details::IsTransformerEstimator<EstimatorT>::value == false>::type
> :
public Impl::GrainEstimatorImplBase<
FitEstimator<typename GrainEstimatorTraits<GrainT, EstimatorT>::InputType>,
@ -298,7 +298,7 @@ class GrainEstimatorImpl<
GrainT,
EstimatorT,
MaxNumTrainingItemsV,
std::enable_if_t<Details::IsTransformerEstimator<EstimatorT>::value>
typename std::enable_if<Details::IsTransformerEstimator<EstimatorT>::value>::type
> :
public Impl::GrainEstimatorImplBase<
TransformerEstimator<

12
src/Traits.h
Просмотреть файл

@ -871,40 +871,40 @@ struct Traits<std::tuple<Types...>> : public TraitsImpl<std::tuple<Types...>> {
}
private:
template<std::size_t N> static inline std::enable_if_t<N < sizeof...(Types) - 1> ToStringHelper(std::tuple<Types...> const& value, std::ostringstream& streamObj) {
template<std::size_t N> static inline typename std::enable_if<N < sizeof...(Types) - 1>::type ToStringHelper(std::tuple<Types...> const& value, std::ostringstream& streamObj) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
streamObj << Traits<type>::ToString(std::get<N>(value)) << ",";
ToStringHelper<N + 1>(value, streamObj);
}
template<std::size_t N> static inline std::enable_if_t<N == (sizeof...(Types) - 1)> ToStringHelper(std::tuple<Types...> const& value, std::ostringstream& streamObj) {
template<std::size_t N> static inline typename std::enable_if<N == (sizeof...(Types) - 1)>::type ToStringHelper(std::tuple<Types...> const& value, std::ostringstream& streamObj) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
streamObj << Traits<type>::ToString(std::get<N>(value));
}
template <std::size_t N, typename ArchiveT> static inline std::enable_if_t<N < sizeof...(Types) - 1> SerializeHelper(ArchiveT &ar, std::tuple<Types...> const &value) {
template <std::size_t N, typename ArchiveT> static inline typename std::enable_if<N < sizeof...(Types) - 1>::type SerializeHelper(ArchiveT &ar, std::tuple<Types...> const &value) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
Traits<type>::serialize(ar, std::get<N>(value));
SerializeHelper<N + 1>(ar, value);
}
template <std::size_t N, typename ArchiveT> static inline std::enable_if_t<N == sizeof...(Types) - 1> SerializeHelper(ArchiveT &ar, std::tuple<Types...> const &value) {
template <std::size_t N, typename ArchiveT> static inline typename std::enable_if<N == sizeof...(Types) - 1>::type SerializeHelper(ArchiveT &ar, std::tuple<Types...> const &value) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
Traits<type>::serialize(ar, std::get<N>(value));
}
template <std::size_t N, typename ArchiveT> static inline std::enable_if_t<N < sizeof...(Types) - 1> DeserializeHelper(ArchiveT &ar, std::tuple<Types...> &value) {
template <std::size_t N, typename ArchiveT> static inline typename std::enable_if<N < sizeof...(Types) - 1>::type DeserializeHelper(ArchiveT &ar, std::tuple<Types...> &value) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
std::get<N>(value) = Traits<type>::deserialize(ar);
DeserializeHelper<N + 1>(ar, value);
}
template <std::size_t N, typename ArchiveT> static inline std::enable_if_t<N == sizeof...(Types) - 1> DeserializeHelper(ArchiveT &ar, std::tuple<Types...> &value) {
template <std::size_t N, typename ArchiveT> static inline typename std::enable_if<N == sizeof...(Types) - 1>::type DeserializeHelper(ArchiveT &ar, std::tuple<Types...> &value) {
using type = typename std::tuple_element<N, std::tuple<Types...>>::type;
std::get<N>(value) = Traits<type>::deserialize(ar);