2016-04-21 15:58:45 +03:00
// Copyright 2013--2014 Xamarin Inc. All rights reserved.
using System ;
using System.Collections.Generic ;
using System.Linq ;
using System.IO ;
using System.Text ;
using System.Threading.Tasks ;
2016-05-11 14:27:51 +03:00
using System.Xml ;
2016-04-21 15:58:45 +03:00
using MonoTouch.Tuner ;
using Mono.Cecil ;
using Mono.Tuner ;
using Mono.Linker ;
using Xamarin.Linker ;
using Xamarin.Utils ;
2018-02-05 18:26:29 +03:00
using Registrar ;
2016-05-11 14:10:35 +03:00
2016-04-21 15:58:45 +03:00
namespace Xamarin.Bundler
{
2017-01-26 15:19:00 +03:00
public class BundleFileInfo
{
public HashSet < string > Sources = new HashSet < string > ( ) ;
public bool DylibToFramework ;
}
2016-04-21 15:58:45 +03:00
public partial class Target {
public string TargetDirectory ;
public string AppTargetDirectory ;
public MonoTouchManifestResolver ManifestResolver = new MonoTouchManifestResolver ( ) ;
public AssemblyDefinition ProductAssembly ;
// directories used during the build process
public string ArchDirectory ;
2016-11-09 14:05:59 +03:00
public string PreBuildDirectory ;
2016-04-21 15:58:45 +03:00
public string BuildDirectory ;
public string LinkDirectory ;
// Note that each 'Target' can have multiple abis: armv7+armv7s for instance.
public List < Abi > Abis ;
2017-01-26 15:19:00 +03:00
public Dictionary < string , BundleFileInfo > BundleFiles = new Dictionary < string , BundleFileInfo > ( ) ;
2016-04-21 15:58:45 +03:00
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
Dictionary < Abi , CompileTask > pinvoke_tasks = new Dictionary < Abi , CompileTask > ( ) ;
List < CompileTask > link_with_task_output = new List < CompileTask > ( ) ;
List < AOTTask > aot_dependencies = new List < AOTTask > ( ) ;
2017-04-12 12:05:01 +03:00
List < LinkTask > embeddinator_tasks = new List < LinkTask > ( ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
CompilerFlags linker_flags ;
NativeLinkTask link_task ;
2016-04-21 15:58:45 +03:00
// If the assemblies were symlinked.
public bool Symlinked ;
2018-05-29 18:14:10 +03:00
// If we're targetting a 32 bit arch for this target.
bool? is32bits ;
public bool Is32Build {
get {
if ( ! is32bits . HasValue )
is32bits = Application . IsArchEnabled ( Abis , Abi . Arch32Mask ) ;
return is32bits . Value ;
}
}
// If we're targetting a 64 bit arch for this target.
bool? is64bits ;
public bool Is64Build {
get {
if ( ! is64bits . HasValue )
is64bits = Application . IsArchEnabled ( Abis , Abi . Arch64Mask ) ;
return is64bits . Value ;
}
}
2016-04-21 15:58:45 +03:00
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
// If we didn't link the final executable because the existing binary is up-to-date.
public bool CachedExecutable {
get {
if ( link_task = = null )
return false ;
return ! link_task . Rebuilt ;
}
}
2018-02-21 11:27:26 +03:00
// If this is an app extension, this returns the equivalent (32/64bit) target for the container app.
// This may be null (it's possible to build an extension for 32+64bit, and the main app only for 64-bit, for instance.
public Target ContainerTarget {
get {
return App . ContainerApp . Targets . FirstOrDefault ( ( v ) = > v . Is32Build = = Is32Build ) ;
}
}
2017-01-26 15:19:00 +03:00
// This is a list of all the architectures we need to build, which may include any architectures
// in any extensions (but not the main app).
List < Abi > all_architectures ;
public List < Abi > AllArchitectures {
get {
if ( all_architectures = = null ) {
all_architectures = new List < Abi > ( ) ;
var mask = Is32Build ? Abi . Arch32Mask : Abi . Arch64Mask ;
foreach ( var abi in App . AllArchitectures ) {
var a = abi & mask ;
if ( a ! = 0 )
2017-04-25 13:49:16 +03:00
all_architectures . Add ( abi ) ;
2017-01-26 15:19:00 +03:00
}
}
return all_architectures ;
}
}
List < Abi > GetArchitectures ( AssemblyBuildTarget build_target )
{
switch ( build_target ) {
case AssemblyBuildTarget . StaticObject :
case AssemblyBuildTarget . DynamicLibrary :
return Abis ;
case AssemblyBuildTarget . Framework :
return AllArchitectures ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , build_target ) ;
2017-01-26 15:19:00 +03:00
}
}
public void AddToBundle ( string source , string bundle_path = null , bool dylib_to_framework_conversion = false )
{
BundleFileInfo info ;
if ( bundle_path = = null ) {
if ( source . EndsWith ( ".framework" , StringComparison . Ordinal ) ) {
var bundle_name = Path . GetFileNameWithoutExtension ( source ) ;
bundle_path = $"Frameworks/{bundle_name}.framework" ;
} else {
bundle_path = Path . GetFileName ( source ) ;
}
}
if ( ! BundleFiles . TryGetValue ( bundle_path , out info ) )
BundleFiles [ bundle_path ] = info = new BundleFileInfo ( ) { DylibToFramework = dylib_to_framework_conversion } ;
if ( info . DylibToFramework ! = dylib_to_framework_conversion )
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , "Internal error: 'invalid value for framework conversion'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." ) ;
2017-01-26 15:19:00 +03:00
info . Sources . Add ( source ) ;
}
2017-01-24 13:10:20 +03:00
void LinkWithBuildTarget ( AssemblyBuildTarget build_target , string name , CompileTask link_task , IEnumerable < Assembly > assemblies )
{
switch ( build_target ) {
case AssemblyBuildTarget . StaticObject :
LinkWithTaskOutput ( link_task ) ;
break ;
case AssemblyBuildTarget . DynamicLibrary :
2017-04-25 18:39:58 +03:00
if ( ! ( ! App . HasFrameworksDirectory & & assemblies . Any ( ( asm ) = > asm . IsCodeShared ) ) )
2017-01-24 13:10:20 +03:00
AddToBundle ( link_task . OutputFile ) ;
LinkWithTaskOutput ( link_task ) ;
break ;
case AssemblyBuildTarget . Framework :
2017-04-25 18:39:58 +03:00
if ( ! ( ! App . HasFrameworksDirectory & & assemblies . Any ( ( asm ) = > asm . IsCodeShared ) ) )
2017-01-24 13:10:20 +03:00
AddToBundle ( link_task . OutputFile , $"Frameworks/{name}.framework/{name}" , dylib_to_framework_conversion : true ) ;
LinkWithTaskOutput ( link_task ) ;
break ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , build_target ) ;
2017-01-24 13:10:20 +03:00
}
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
public void LinkWithTaskOutput ( CompileTask task )
{
if ( task . SharedLibrary ) {
LinkWithDynamicLibrary ( task . OutputFile ) ;
} else {
LinkWithStaticLibrary ( task . OutputFile ) ;
}
link_with_task_output . Add ( task ) ;
}
public void LinkWithTaskOutput ( IEnumerable < CompileTask > tasks )
{
foreach ( var t in tasks )
LinkWithTaskOutput ( t ) ;
}
public void LinkWithStaticLibrary ( string path )
{
linker_flags . AddLinkWith ( path ) ;
}
public void LinkWithStaticLibrary ( IEnumerable < string > paths )
{
linker_flags . AddLinkWith ( paths ) ;
}
2017-01-26 15:19:00 +03:00
public void LinkWithFramework ( string path )
{
linker_flags . AddFramework ( path ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
public void LinkWithDynamicLibrary ( string path )
{
linker_flags . AddLinkWith ( path ) ;
}
2016-10-06 15:16:55 +03:00
2016-10-06 17:13:01 +03:00
PInvokeWrapperGenerator pinvoke_state ;
PInvokeWrapperGenerator MarshalNativeExceptionsState {
get {
if ( ! App . RequiresPInvokeWrappers )
return null ;
if ( pinvoke_state = = null ) {
pinvoke_state = new PInvokeWrapperGenerator ( )
{
2016-12-23 20:50:35 +03:00
App = App ,
2016-10-06 17:13:01 +03:00
SourcePath = Path . Combine ( ArchDirectory , "pinvokes.m" ) ,
HeaderPath = Path . Combine ( ArchDirectory , "pinvokes.h" ) ,
Registrar = ( StaticRegistrar ) StaticRegistrar ,
} ;
}
return pinvoke_state ;
}
}
2016-04-21 15:58:45 +03:00
public string Executable {
get {
return Path . Combine ( TargetDirectory , App . ExecutableName ) ;
}
}
public void Initialize ( bool show_warnings )
{
// we want to load our own mscorlib[-runtime].dll, not something else we're being feeded
// (e.g. bug #6612) since it might not match the libmono[-sgen].a library we'll link with,
// so load the corlib we want first.
var corlib_path = Path . Combine ( Resolver . FrameworkDirectory , "mscorlib.dll" ) ;
var corlib = ManifestResolver . Load ( corlib_path ) ;
if ( corlib = = null )
throw new MonoTouchException ( 2006 , true , "Can not load mscorlib.dll from: '{0}'. Please reinstall Xamarin.iOS." , corlib_path ) ;
2017-04-07 15:13:11 +03:00
var roots = new List < AssemblyDefinition > ( ) ;
foreach ( var root_assembly in App . RootAssemblies ) {
var root = ManifestResolver . Load ( root_assembly ) ;
if ( root = = null ) {
// We check elsewhere that the path exists, so I'm not sure how we can get into this.
throw ErrorHelper . CreateError ( 2019 , "Can not load the root assembly '{0}'." , root_assembly ) ;
}
2017-04-19 20:10:48 +03:00
roots . Add ( root ) ;
2017-01-25 12:47:51 +03:00
}
2016-04-21 15:58:45 +03:00
foreach ( var reference in App . References ) {
var ad = ManifestResolver . Load ( reference ) ;
if ( ad = = null )
throw new MonoTouchException ( 2002 , true , "Can not resolve reference: {0}" , reference ) ;
2017-01-25 12:47:51 +03:00
2017-04-07 15:13:11 +03:00
var root_assembly = roots . FirstOrDefault ( ( v ) = > v . MainModule . FileName = = ad . MainModule . FileName ) ;
if ( root_assembly ! = null ) {
// If we asked the manifest resolver for assembly X and got back a root assembly, it means the requested assembly has the same identity as the root assembly, which is not allowed.
throw ErrorHelper . CreateError ( 23 , "The root assembly {0} conflicts with another assembly ({1})." , root_assembly . MainModule . FileName , reference ) ;
2017-01-25 12:47:51 +03:00
}
2016-04-21 15:58:45 +03:00
if ( ad . MainModule . Runtime > TargetRuntime . Net_4_0 )
ErrorHelper . Show ( new MonoTouchException ( 11 , false , "{0} was built against a more recent runtime ({1}) than Xamarin.iOS supports." , Path . GetFileName ( reference ) , ad . MainModule . Runtime ) ) ;
// Figure out if we're referencing Xamarin.iOS or monotouch.dll
2016-12-23 20:50:35 +03:00
if ( Path . GetFileNameWithoutExtension ( ad . MainModule . FileName ) = = Driver . GetProductAssembly ( App ) )
2016-04-21 15:58:45 +03:00
ProductAssembly = ad ;
2017-01-25 12:47:51 +03:00
if ( ad ! = ProductAssembly & & GetRealPath ( ad . MainModule . FileName ) ! = GetRealPath ( reference ) & & ! ad . MainModule . FileName . EndsWith ( ".resources.dll" , StringComparison . Ordinal ) )
ErrorHelper . Show ( ErrorHelper . CreateWarning ( 109 , "The assembly '{0}' was loaded from a different path than the provided path (provided path: {1}, actual path: {2})." , Path . GetFileName ( reference ) , reference , ad . MainModule . FileName ) ) ;
2016-04-21 15:58:45 +03:00
}
ComputeListOfAssemblies ( ) ;
if ( App . LinkMode = = LinkMode . None & & App . I18n ! = I18nAssemblies . None )
AddI18nAssemblies ( ) ;
2017-09-28 18:31:55 +03:00
if ( ! App . Embeddinator ) {
if ( ! Assemblies . Any ( ( v ) = > v . AssemblyDefinition . Name . Name = = Driver . GetProductAssembly ( App ) ) )
throw ErrorHelper . CreateError ( 123 , $"The executable assembly {App.RootAssemblies [0]} does not reference {Driver.GetProductAssembly (App)}.dll." ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
linker_flags = new CompilerFlags ( this ) ;
2018-10-10 18:02:28 +03:00
// Verify that there are no entries in our list of intepreted assemblies that doesn't match
// any of the assemblies we know about.
if ( App . UseInterpreter ) {
var exceptions = new List < Exception > ( ) ;
foreach ( var entry in App . InterpretedAssemblies ) {
var assembly = entry ;
if ( string . IsNullOrEmpty ( assembly ) )
continue ;
if ( assembly [ 0 ] = = '-' )
assembly = assembly . Substring ( 1 ) ;
if ( assembly = = "all" )
continue ;
if ( Assemblies . ContainsKey ( assembly ) )
continue ;
exceptions . Add ( ErrorHelper . CreateWarning ( 138 , $"Cannot find the assembly '{assembly}', passed as an argument to --interpreter." ) ) ;
}
ErrorHelper . ThrowIfErrors ( exceptions ) ;
}
2016-04-21 15:58:45 +03:00
}
// This is to load the symbols for all assemblies, so that we can give better error messages
// (with file name / line number information).
public void LoadSymbols ( )
{
foreach ( var a in Assemblies )
a . LoadSymbols ( ) ;
}
IEnumerable < AssemblyDefinition > GetAssemblies ( )
{
if ( App . LinkMode = = LinkMode . None )
2016-05-11 14:10:35 +03:00
return ManifestResolver . GetAssemblies ( ) ;
2016-04-21 15:58:45 +03:00
List < AssemblyDefinition > assemblies = new List < AssemblyDefinition > ( ) ;
if ( LinkContext = = null ) {
// use data from cache
foreach ( var assembly in Assemblies )
assemblies . Add ( assembly . AssemblyDefinition ) ;
} else {
foreach ( var assembly in LinkContext . GetAssemblies ( ) ) {
if ( LinkContext . Annotations . GetAction ( assembly ) = = AssemblyAction . Delete )
continue ;
assemblies . Add ( assembly ) ;
}
}
return assemblies ;
}
/ /
// Gets a flattened list of all the assemblies pulled by the root assembly
/ /
public void ComputeListOfAssemblies ( )
{
var exceptions = new List < Exception > ( ) ;
var assemblies = new HashSet < string > ( ) ;
2017-05-24 08:26:36 +03:00
var cache_file = Path . Combine ( this . ArchDirectory , "assembly-references.txt" ) ;
if ( File . Exists ( cache_file ) ) {
assemblies . UnionWith ( File . ReadAllLines ( cache_file ) ) ;
// Check if any of the referenced assemblies changed after we cached the complete set of references
if ( Application . IsUptodate ( assemblies , new string [ ] { cache_file } ) ) {
// Load all the assemblies in the cached list of assemblies
[mtouch] Load assemblies correctly when loading cached list. (#2124)
Make sure to instantiate an Assembly instance for every assembly loaded using
the cached list of assemblies.
Fixes the following test failures:
1. Xamarin.MTouch.RebuildTest_WithExtensions("single","",False,System.String[]) : second build
Expected: 0
But was: 1
2. Xamarin.MTouch.RebuildTest_WithExtensions("dual","armv7,arm64",False,System.String[]) : second build
Expected: 0
But was: 1
3. Xamarin.MTouch.RebuildTest_WithExtensions("llvm","armv7+llvm",False,System.String[]) : second build
Expected: 0
But was: 1
4. Xamarin.MTouch.RebuildTest_WithExtensions("debug","",True,System.String[]) : second build
Expected: 0
But was: 1
5. Xamarin.MTouch.RebuildTest_WithExtensions("single-framework","",False,System.String[]) : second build
Expected: 0
But was: 1
2017-05-24 22:03:32 +03:00
foreach ( var assembly in assemblies ) {
var ad = ManifestResolver . Load ( assembly ) ;
var asm = new Assembly ( this , ad ) ;
asm . ComputeSatellites ( ) ;
this . Assemblies . Add ( asm ) ;
}
2017-05-24 08:26:36 +03:00
return ;
}
// We must manually find all the references.
assemblies . Clear ( ) ;
}
2016-04-21 15:58:45 +03:00
try {
2017-04-07 15:13:11 +03:00
foreach ( var root in App . RootAssemblies ) {
var assembly = ManifestResolver . Load ( root ) ;
ComputeListOfAssemblies ( assemblies , assembly , exceptions ) ;
}
2016-04-21 15:58:45 +03:00
} catch ( MonoTouchException mte ) {
exceptions . Add ( mte ) ;
} catch ( Exception e ) {
2016-10-28 00:45:33 +03:00
exceptions . Add ( new MonoTouchException ( 9 , true , e , "Error while loading assemblies: {0}" , e . Message ) ) ;
2016-04-21 15:58:45 +03:00
}
if ( App . LinkMode = = LinkMode . None )
exceptions . AddRange ( ManifestResolver . list ) ;
if ( exceptions . Count > 0 )
throw new AggregateException ( exceptions ) ;
2017-05-24 08:26:36 +03:00
// Cache all the assemblies we found.
Directory . CreateDirectory ( Path . GetDirectoryName ( cache_file ) ) ;
File . WriteAllLines ( cache_file , assemblies ) ;
2016-04-21 15:58:45 +03:00
}
void ComputeListOfAssemblies ( HashSet < string > assemblies , AssemblyDefinition assembly , List < Exception > exceptions )
{
if ( assembly = = null )
return ;
2016-08-24 16:43:35 +03:00
var fqname = assembly . MainModule . FileName ;
2016-04-21 15:58:45 +03:00
if ( assemblies . Contains ( fqname ) )
return ;
[mtouch/mmp] Print assembly references in verbose mode. (#2139)
* [mtouch/mmp] Print assembly references in verbose mode.
Sample output:
Loaded assembly 'unifiedtestapp, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null' from /Users/rolf/Projects/TestApp/bin/iPhoneSimulator/Debug/unifiedtestapp.exe
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Xamarin.iOS, Version=0.0.0.0, Culture=neutral, PublicKeyToken=84e04ff9cfb79065
Loaded assembly 'mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/mscorlib.dll
Loaded assembly 'Xamarin.iOS, Version=0.0.0.0, Culture=neutral, PublicKeyToken=84e04ff9cfb79065' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/../../64bits/Xamarin.iOS.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System.Core, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756
References: System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756
References: System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/Mono.Security.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.Xml.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System.Core, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.Core.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
I believe this will make it easier to diagnose cases where something references a desktop assembly.
* [mtouch/mmp] Share more code between mmp and mtouch.
2017-05-29 17:15:22 +03:00
PrintAssemblyReferences ( assembly ) ;
2016-04-21 15:58:45 +03:00
assemblies . Add ( fqname ) ;
var asm = new Assembly ( this , assembly ) ;
asm . ComputeSatellites ( ) ;
this . Assemblies . Add ( asm ) ;
var main = assembly . MainModule ;
foreach ( AssemblyNameReference reference in main . AssemblyReferences ) {
// Verify that none of the references references an incorrect platform assembly.
switch ( reference . Name ) {
case "monotouch" :
case "Xamarin.iOS" :
case "Xamarin.TVOS" :
case "Xamarin.WatchOS" :
2016-12-23 20:50:35 +03:00
if ( reference . Name ! = Driver . GetProductAssembly ( App ) )
2016-04-21 15:58:45 +03:00
exceptions . Add ( ErrorHelper . CreateError ( 34 , "Cannot reference '{0}.dll' in a {1} project - it is implicitly referenced by '{2}'." , reference . Name , Driver . TargetFramework . Identifier , assembly . FullName ) ) ;
break ;
}
var reference_assembly = ManifestResolver . Resolve ( reference ) ;
2018-07-30 11:55:23 +03:00
if ( reference_assembly = = null ) {
ErrorHelper . Warning ( 136 , "Cannot find the assembly '{0}' referenced from '{1}'." , reference . FullName , main . FileName ) ;
continue ;
}
2016-04-21 15:58:45 +03:00
ComputeListOfAssemblies ( assemblies , reference_assembly , exceptions ) ;
}
[mtouch] Don't look for assembly references in attributes in assemblies we ship. Partially fixes #49087. (#2121)
* [mtouch] Don't look for assembly references in attributes in assemblies we ship. Partially fixes #49087.
Don't look for assembly references in attributes in assemblies we ship,
because it takes a significant amount of time to do this, and we can
precompute the fact that there aren't any such assembly references.
Additionally add a test to ensure we catch any changes to this assumption.
For a simple test app this makes rebuilding (without any changes) go from
~1.1s to ~0.4s.
https://bugzilla.xamarin.com/show_bug.cgi?id=49087
* [tests] Simplify tests to not use [TestCaseSource].
Using [TestCaseSource] is nice when running from the IDE, since it shows all
test cases in the test tree.
Unfortunately it causes the console runner to freak out [1], because the method
that lists all the test cases calls Configuration's cctor, which calls
TestContext.CurrentContext.TestDirectory, which is apparently not safe this
early in the test run.
[1] I think 'freak out' is the appropriate term for this behavior, which has
absolutely no direct nor obvious connection to the cause of the problem:
System.Runtime.Remoting.RemotingException: Cannot create channel sink to connect to URL 93a78115_c0da_4b6a_9661_9f9b9d9fb935/6669afd6_4.rem. An appropriate channel has probably not been registered.
Server stack trace:
at System.Runtime.Remoting.RemotingServices.GetClientChannelSinkChain (System.String url, System.Object channelData, System.String& objectUri) [0x00019] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.RemotingServices.GetOrCreateClientIdentity (System.Runtime.Remoting.ObjRef objRef, System.Type proxyType, System.Object& clientProxy) [0x0001d] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.RemotingServices.GetRemoteObject (System.Runtime.Remoting.ObjRef objRef, System.Type proxyType) [0x00000] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.RemotingServices.GetProxyForRemoteObject (System.Runtime.Remoting.ObjRef objref, System.Type classToProxy) [0x0001b] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.RemotingServices.Unmarshal (System.Runtime.Remoting.ObjRef objectRef, System.Boolean fRefine) [0x0007a] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.RemotingServices.Unmarshal (System.Runtime.Remoting.ObjRef objectRef) [0x00000] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.ObjRef.GetRealObject (System.Runtime.Serialization.StreamingContext context) [0x0000f] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.ObjectManager.ResolveObjectReference (System.Runtime.Serialization.ObjectHolder holder) [0x00010] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.ObjectManager.DoFixups () [0x0007f] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.Formatters.Binary.ObjectReader.Deserialize (System.Runtime.Remoting.Messaging.HeaderHandler handler, System.Runtime.Serialization.Formatters.Binary.__BinaryParser serParser, System.Boolean fCheck, System.Boolean isCrossAppDomain, System.Runtime.Remoting.Messaging.IMethodCallMessage methodCallMessage) [0x00077] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.Formatters.Binary.BinaryFormatter.Deserialize (System.IO.Stream serializationStream, System.Runtime.Remoting.Messaging.HeaderHandler handler, System.Boolean fCheck, System.Boolean isCrossAppDomain, System.Runtime.Remoting.Messaging.IMethodCallMessage methodCallMessage) [0x000a2] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.Formatters.Binary.BinaryFormatter.Deserialize (System.IO.Stream serializationStream, System.Runtime.Remoting.Messaging.HeaderHandler handler, System.Boolean fCheck, System.Runtime.Remoting.Messaging.IMethodCallMessage methodCallMessage) [0x00000] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Serialization.Formatters.Binary.BinaryFormatter.DeserializeMethodResponse (System.IO.Stream serializationStream, System.Runtime.Remoting.Messaging.HeaderHandler handler, System.Runtime.Remoting.Messaging.IMethodCallMessage methodCallMessage) [0x00000] in <04300341516a482b9708b764d58af7ca>:0
at System.Runtime.Remoting.Channels.BinaryClientFormatterSink.SyncProcessMessage (System.Runtime.Remoting.Messaging.IMessage msg) [0x00083] in <270c90abbc234cde9d33eb198a97cf71>:0
2017-05-24 22:05:21 +03:00
if ( Profile . IsSdkAssembly ( assembly ) | | Profile . IsProductAssembly ( assembly ) )
return ; // We know there are no new assembly references from attributes in assemblies we ship
2016-04-21 15:58:45 +03:00
// Custom Attribute metadata can include references to other assemblies, e.g. [X (typeof (Y)],
// but it is not reflected in AssemblyReferences :-( ref: #37611
// so we must scan every custom attribute to look for System.Type
2018-07-30 11:55:23 +03:00
GetCustomAttributeReferences ( main , main , assemblies , exceptions ) ;
foreach ( var ca in main . GetCustomAttributes ( ) )
GetCustomAttributeReferences ( main , ca , assemblies , exceptions ) ;
2016-04-21 15:58:45 +03:00
}
2018-07-30 11:55:23 +03:00
void GetCustomAttributeReferences ( ModuleDefinition main , ICustomAttributeProvider cap , HashSet < string > assemblies , List < Exception > exceptions )
2016-04-21 15:58:45 +03:00
{
if ( ! cap . HasCustomAttributes )
return ;
2017-01-19 12:24:00 +03:00
foreach ( var ca in cap . CustomAttributes )
2018-07-30 11:55:23 +03:00
GetCustomAttributeReferences ( main , ca , assemblies , exceptions ) ;
2017-01-19 12:24:00 +03:00
}
2018-07-30 11:55:23 +03:00
void GetCustomAttributeReferences ( ModuleDefinition main , CustomAttribute ca , HashSet < string > assemblies , List < Exception > exceptions )
2017-01-19 12:24:00 +03:00
{
if ( ca . HasConstructorArguments ) {
foreach ( var arg in ca . ConstructorArguments )
2018-07-30 11:55:23 +03:00
GetCustomAttributeArgumentReference ( main , ca , arg , assemblies , exceptions ) ;
2017-01-19 12:24:00 +03:00
}
if ( ca . HasFields ) {
foreach ( var arg in ca . Fields )
2018-07-30 11:55:23 +03:00
GetCustomAttributeArgumentReference ( main , ca , arg . Argument , assemblies , exceptions ) ;
2017-01-19 12:24:00 +03:00
}
if ( ca . HasProperties ) {
foreach ( var arg in ca . Properties )
2018-07-30 11:55:23 +03:00
GetCustomAttributeArgumentReference ( main , ca , arg . Argument , assemblies , exceptions ) ;
2016-04-21 15:58:45 +03:00
}
}
2018-07-30 11:55:23 +03:00
void GetCustomAttributeArgumentReference ( ModuleDefinition main , CustomAttribute ca , CustomAttributeArgument arg , HashSet < string > assemblies , List < Exception > exceptions )
2016-04-21 15:58:45 +03:00
{
if ( ! arg . Type . Is ( "System" , "Type" ) )
return ;
var ar = ( arg . Value as TypeReference ) ? . Scope as AssemblyNameReference ;
if ( ar = = null )
return ;
var reference_assembly = ManifestResolver . Resolve ( ar ) ;
2018-07-30 11:55:23 +03:00
if ( reference_assembly = = null ) {
ErrorHelper . Warning ( 137 , "Cannot find the assembly '{0}', referenced by a {2} attribute in '{1}'." , ar . FullName , main . Name , ca . AttributeType . FullName ) ;
return ;
}
2016-04-21 15:58:45 +03:00
ComputeListOfAssemblies ( assemblies , reference_assembly , exceptions ) ;
}
bool IncludeI18nAssembly ( Mono . Linker . I18nAssemblies assembly )
{
return ( App . I18n & assembly ) ! = 0 ;
}
public void AddI18nAssemblies ( )
{
Assemblies . Add ( LoadI18nAssembly ( "I18N" ) ) ;
if ( IncludeI18nAssembly ( Mono . Linker . I18nAssemblies . CJK ) )
Assemblies . Add ( LoadI18nAssembly ( "I18N.CJK" ) ) ;
if ( IncludeI18nAssembly ( Mono . Linker . I18nAssemblies . MidEast ) )
Assemblies . Add ( LoadI18nAssembly ( "I18N.MidEast" ) ) ;
if ( IncludeI18nAssembly ( Mono . Linker . I18nAssemblies . Other ) )
Assemblies . Add ( LoadI18nAssembly ( "I18N.Other" ) ) ;
if ( IncludeI18nAssembly ( Mono . Linker . I18nAssemblies . Rare ) )
Assemblies . Add ( LoadI18nAssembly ( "I18N.Rare" ) ) ;
if ( IncludeI18nAssembly ( Mono . Linker . I18nAssemblies . West ) )
Assemblies . Add ( LoadI18nAssembly ( "I18N.West" ) ) ;
}
Assembly LoadI18nAssembly ( string name )
{
var assembly = ManifestResolver . Resolve ( AssemblyNameReference . Parse ( name ) ) ;
return new Assembly ( this , assembly ) ;
}
2017-01-24 13:10:20 +03:00
public void LinkAssemblies ( out List < AssemblyDefinition > assemblies , string output_dir , IEnumerable < Target > sharedCodeTargets )
2016-04-21 15:58:45 +03:00
{
2017-09-05 21:48:09 +03:00
var cache = ( Dictionary < string , AssemblyDefinition > ) Resolver . ResolverCache ;
2017-01-24 13:10:20 +03:00
var resolver = new AssemblyResolver ( cache ) ;
2016-04-21 15:58:45 +03:00
resolver . AddSearchDirectory ( Resolver . RootDirectory ) ;
resolver . AddSearchDirectory ( Resolver . FrameworkDirectory ) ;
2017-01-24 13:10:20 +03:00
var main_assemblies = new List < AssemblyDefinition > ( ) ;
2017-04-07 15:13:11 +03:00
foreach ( var root in App . RootAssemblies )
main_assemblies . Add ( Resolver . Load ( root ) ) ;
foreach ( var appex in sharedCodeTargets ) {
foreach ( var root in appex . App . RootAssemblies )
main_assemblies . Add ( Resolver . Load ( root ) ) ;
}
2017-01-24 13:10:20 +03:00
if ( Driver . Verbosity > 0 )
Console . WriteLine ( "Linking {0} into {1} using mode '{2}'" , string . Join ( ", " , main_assemblies . Select ( ( v ) = > v . MainModule . FileName ) ) , output_dir , App . LinkMode ) ;
2016-05-11 14:10:35 +03:00
LinkerOptions = new LinkerOptions {
2017-01-24 13:10:20 +03:00
MainAssemblies = main_assemblies ,
2016-04-21 15:58:45 +03:00
OutputDirectory = output_dir ,
LinkMode = App . LinkMode ,
Resolver = resolver ,
SkippedAssemblies = App . LinkSkipped ,
I18nAssemblies = App . I18n ,
LinkSymbols = true ,
LinkAway = App . LinkAway ,
ExtraDefinitions = App . Definitions ,
Device = App . IsDeviceBuild ,
DebugBuild = App . EnableDebug ,
IsDualBuild = App . IsDualBuild ,
DumpDependencies = App . LinkerDumpDependencies ,
2016-05-11 14:27:51 +03:00
RuntimeOptions = App . RuntimeOptions ,
2016-10-06 17:13:01 +03:00
MarshalNativeExceptionsState = MarshalNativeExceptionsState ,
2017-01-18 12:25:58 +03:00
Target = this ,
2016-04-21 15:58:45 +03:00
} ;
2017-01-24 22:12:48 +03:00
MonoTouch . Tuner . Linker . Process ( LinkerOptions , out LinkContext , out assemblies ) ;
2016-04-21 15:58:45 +03:00
2019-02-07 09:57:01 +03:00
ErrorHelper . Show ( LinkContext . Exceptions ) ;
2016-04-21 15:58:45 +03:00
Driver . Watch ( "Link Assemblies" , 1 ) ;
}
2017-01-24 13:10:20 +03:00
bool linked ;
2016-04-21 15:58:45 +03:00
public void ManagedLink ( )
{
2017-01-24 13:10:20 +03:00
if ( linked )
return ;
2016-04-21 15:58:45 +03:00
var cache_path = Path . Combine ( ArchDirectory , "linked-assemblies.txt" ) ;
2017-01-24 13:10:20 +03:00
// Get all the Target instances we're sharing code with. Make sure to only select targets with matching pointer size.
var sharingTargets = App . SharedCodeApps . SelectMany ( ( v ) = > v . Targets ) . Where ( ( v ) = > v . Is32Build = = Is32Build ) . ToList ( ) ;
var allTargets = new List < Target > ( ) ;
allTargets . Add ( this ) ; // We want ourselves first in this list.
allTargets . AddRange ( sharingTargets ) ;
// Include any assemblies from appex's we're sharing code with.
foreach ( var target in sharingTargets ) {
var targetAssemblies = target . Assemblies . ToList ( ) ; // We need to clone the list of assemblies, since we'll be modifying the original
foreach ( var asm in targetAssemblies ) {
Assembly main_asm ;
if ( ! Assemblies . TryGetValue ( asm . Identity , out main_asm ) ) {
// The appex has an assembly that's not present in the main app.
// Re-load it into the main app.
main_asm = new Assembly ( this , asm . FullPath ) ;
main_asm . LoadAssembly ( main_asm . FullPath ) ;
Assemblies . Add ( main_asm ) ;
Driver . Log ( 1 , "Added '{0}' from {1} to the set of assemblies to be linked." , main_asm . Identity , Path . GetFileNameWithoutExtension ( target . App . AppDirectory ) ) ;
} else {
asm . IsCodeShared = true ;
}
// Use the same AOT information between both Assembly instances.
target . Assemblies [ main_asm . Identity ] . AotInfos = main_asm . AotInfos ;
main_asm . IsCodeShared = true ;
}
}
2016-04-21 15:58:45 +03:00
foreach ( var a in Assemblies )
a . CopyToDirectory ( LinkDirectory , false , check_case : true ) ;
// Check if we can use a previous link result.
2017-01-24 13:10:20 +03:00
var cached_output = new Dictionary < string , List < string > > ( ) ;
2016-04-21 15:58:45 +03:00
if ( ! Driver . Force ) {
if ( File . Exists ( cache_path ) ) {
2017-01-24 13:10:20 +03:00
using ( var reader = new StreamReader ( cache_path ) ) {
[mtouch/mmp] Fix tracking of whether the static registrar should run again or not. Fixes #641. (#3534)
* [tests] Improve debug spew for the RebuildTest_WithExtensions test.
* [mtouch/mmp] Store/load if the dynamic registrar is removed or not into the cached link results.
Store/load if the dynamic registrar is removed or not into the cached link
results, so that we generate the correct main.m even if cached linker results
are used.
* [mtouch/mmp] The static registrar must not execute if we're loading cached results from the linker.
The static registrar must not execute if we're loading cached results from the
linker, because the static registrar needs information from the linker that's
not restored from the cache.
* [mtouch/mmp] Share Touch code.
* [mtouch/mmp] Make it possible to touch inexistent files (to create them).
* [mtouch/mmp] Fix tracking of whether the static registrar should run again or not.
The recent changes to support optimizing away the dynamic registrar caused the
Xamarin.MTouch.RebuildTest_WithExtensions test to regress.
The problem
-----------
* The linker now collects and stores information the static registrar needs.
* This information is not restored from disk when the linker realizes that it
can reload previously linked assemblies instead of executing again.
* The static registrar runs again (for another reason).
* The information the static registrar needs isn't available, and incorrect
output follows.
So fix 1: show an error if the static registrar runs when the linker loaded
cached results.
The exact scenario the test ran into is this:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk (this is an optimization to avoid
compiling the registrar.m file again unless needed).
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is newer than
registrar.m's timestamp and run again, but doesn't produce the right result
because it doesn't have the information it needs.
Considered solutions
--------------------
1. Only track timestamps, not file contents. This is not ideal, since it will
result in more work done: in particular for the case above, it would add a
registrar.m compilation in build #2, and linker rerun + static registrar
rerun + registrar.m compilation + final native link in build #3.
2. Always write the output of the static registrar, even if it hasn't changed.
This is not ideal either, since it will also result in more work done: for
the case above, it would add a registrar.m compilation + final native link
in build #3.
3. Always write the output of the static registrar, but track if it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it.
This only means the final native link in build #3 is added (see #5 for why
this is worse than it sounds).
4. Always write the output of the static registrar, but track it it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it,
and track that too, so that the final native link in build #3 isn't needed
anymore. Unfortunately this may result in incorrect behavior, because now
the msbuild tasks will detect that the executable has changed, and may run
dsymutil + strip again. The executable didn't actually change, which means
it would be the previously stripped executable, and thus we'd end up with
an empty .dSYM because we ran dsymtil on an already stripped executable.
5. Idea #4, but write the output of the final link into a temporary directory
instead of the .app, so that we could track whether we should update the
executable in the .app or not. This is not optimal either, because
executables can be *big* (I've seen multi-GB tvOS bitcode executables), and
extra copies of such files should not be taken lightly.
6. Idea #4, but tell the MSBuild tasks that dsymutil/strip doesn't need to be
rerun even if the timestamp of the executable changed. This might actually
work, but now the solution's become quite complex.
Implemented solution
--------------------
Use stamp files to detect whether a file is up-to-date or not.
In particular:
* When we don't write to a file because the new contents are identical to the
old contents, we now touch a .stamp file. This stamp file means "the
accompanying file was determined to be up-to-date when the stamp was
touched."
* When checking whether a file is up-to-date, also check for the presence of a
.stamp file, and if it exists, use the highest timestamp between the stamp
file and the actual file.
Now the test scenario becomes:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk, but it creates a registrar.m.stamp
file to indicate the point in time when registrar.m was considered up-to-
date.
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is *older* than
registrar.m.stamp's timestamp and doesn't run again.
We only use the stamp file for source code (registrar.[m|h], main.[m|h],
pinvokes.[m|h]), since using it every time has too much potential for running
into other problems (for instance we should never create .stamp files inside
the .app).
Fixes these test failures:
1) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single","",False,System.String[])
single
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory371/testApp.app/testApp is modified, timestamp: 2/15/2018 3:04:11 PM > 2/15/2018 3:04:09 PM" >
2) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("dual","armv7,arm64",False,System.String[])
dual
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory375/testApp.app/testApp is modified, timestamp: 2/15/2018 3:06:03 PM > 2/15/2018 3:06:00 PM" >
3) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("llvm","armv7+llvm",False,System.String[])
llvm
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory379/testApp.app/testApp is modified, timestamp: 2/15/2018 3:07:14 PM > 2/15/2018 3:07:12 PM" >
4) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("debug","",True,System.String[])
debug
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory383/testApp.app/testApp is modified, timestamp: 2/15/2018 3:08:16 PM > 2/15/2018 3:08:13 PM" >
5) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single-framework","",False,System.String[])
single-framework
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory387/testApp.app/testApp is modified, timestamp: 2/15/2018 3:09:18 PM > 2/15/2018 3:09:16 PM" >
Fixes https://github.com/xamarin/maccore/issues/641
2018-02-19 22:28:04 +03:00
string line = null ;
2017-01-24 13:10:20 +03:00
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
var colon = line . IndexOf ( ':' ) ;
if ( colon = = - 1 )
continue ;
[mtouch/mmp] Fix tracking of whether the static registrar should run again or not. Fixes #641. (#3534)
* [tests] Improve debug spew for the RebuildTest_WithExtensions test.
* [mtouch/mmp] Store/load if the dynamic registrar is removed or not into the cached link results.
Store/load if the dynamic registrar is removed or not into the cached link
results, so that we generate the correct main.m even if cached linker results
are used.
* [mtouch/mmp] The static registrar must not execute if we're loading cached results from the linker.
The static registrar must not execute if we're loading cached results from the
linker, because the static registrar needs information from the linker that's
not restored from the cache.
* [mtouch/mmp] Share Touch code.
* [mtouch/mmp] Make it possible to touch inexistent files (to create them).
* [mtouch/mmp] Fix tracking of whether the static registrar should run again or not.
The recent changes to support optimizing away the dynamic registrar caused the
Xamarin.MTouch.RebuildTest_WithExtensions test to regress.
The problem
-----------
* The linker now collects and stores information the static registrar needs.
* This information is not restored from disk when the linker realizes that it
can reload previously linked assemblies instead of executing again.
* The static registrar runs again (for another reason).
* The information the static registrar needs isn't available, and incorrect
output follows.
So fix 1: show an error if the static registrar runs when the linker loaded
cached results.
The exact scenario the test ran into is this:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk (this is an optimization to avoid
compiling the registrar.m file again unless needed).
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is newer than
registrar.m's timestamp and run again, but doesn't produce the right result
because it doesn't have the information it needs.
Considered solutions
--------------------
1. Only track timestamps, not file contents. This is not ideal, since it will
result in more work done: in particular for the case above, it would add a
registrar.m compilation in build #2, and linker rerun + static registrar
rerun + registrar.m compilation + final native link in build #3.
2. Always write the output of the static registrar, even if it hasn't changed.
This is not ideal either, since it will also result in more work done: for
the case above, it would add a registrar.m compilation + final native link
in build #3.
3. Always write the output of the static registrar, but track if it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it.
This only means the final native link in build #3 is added (see #5 for why
this is worse than it sounds).
4. Always write the output of the static registrar, but track it it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it,
and track that too, so that the final native link in build #3 isn't needed
anymore. Unfortunately this may result in incorrect behavior, because now
the msbuild tasks will detect that the executable has changed, and may run
dsymutil + strip again. The executable didn't actually change, which means
it would be the previously stripped executable, and thus we'd end up with
an empty .dSYM because we ran dsymtil on an already stripped executable.
5. Idea #4, but write the output of the final link into a temporary directory
instead of the .app, so that we could track whether we should update the
executable in the .app or not. This is not optimal either, because
executables can be *big* (I've seen multi-GB tvOS bitcode executables), and
extra copies of such files should not be taken lightly.
6. Idea #4, but tell the MSBuild tasks that dsymutil/strip doesn't need to be
rerun even if the timestamp of the executable changed. This might actually
work, but now the solution's become quite complex.
Implemented solution
--------------------
Use stamp files to detect whether a file is up-to-date or not.
In particular:
* When we don't write to a file because the new contents are identical to the
old contents, we now touch a .stamp file. This stamp file means "the
accompanying file was determined to be up-to-date when the stamp was
touched."
* When checking whether a file is up-to-date, also check for the presence of a
.stamp file, and if it exists, use the highest timestamp between the stamp
file and the actual file.
Now the test scenario becomes:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk, but it creates a registrar.m.stamp
file to indicate the point in time when registrar.m was considered up-to-
date.
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is *older* than
registrar.m.stamp's timestamp and doesn't run again.
We only use the stamp file for source code (registrar.[m|h], main.[m|h],
pinvokes.[m|h]), since using it every time has too much potential for running
into other problems (for instance we should never create .stamp files inside
the .app).
Fixes these test failures:
1) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single","",False,System.String[])
single
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory371/testApp.app/testApp is modified, timestamp: 2/15/2018 3:04:11 PM > 2/15/2018 3:04:09 PM" >
2) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("dual","armv7,arm64",False,System.String[])
dual
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory375/testApp.app/testApp is modified, timestamp: 2/15/2018 3:06:03 PM > 2/15/2018 3:06:00 PM" >
3) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("llvm","armv7+llvm",False,System.String[])
llvm
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory379/testApp.app/testApp is modified, timestamp: 2/15/2018 3:07:14 PM > 2/15/2018 3:07:12 PM" >
4) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("debug","",True,System.String[])
debug
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory383/testApp.app/testApp is modified, timestamp: 2/15/2018 3:08:16 PM > 2/15/2018 3:08:13 PM" >
5) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single-framework","",False,System.String[])
single-framework
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory387/testApp.app/testApp is modified, timestamp: 2/15/2018 3:09:18 PM > 2/15/2018 3:09:16 PM" >
Fixes https://github.com/xamarin/maccore/issues/641
2018-02-19 22:28:04 +03:00
var key = line . Substring ( 0 , colon ) ;
var value = line . Substring ( colon + 1 ) ;
switch ( key ) {
case "RemoveDynamicRegistrar" :
switch ( value ) {
case "true" :
App . Optimizations . RemoveDynamicRegistrar = true ;
break ;
case "false" :
App . Optimizations . RemoveDynamicRegistrar = false ;
break ;
default :
App . Optimizations . RemoveDynamicRegistrar = null ;
break ;
}
2018-03-09 19:24:47 +03:00
foreach ( var t in sharingTargets )
t . App . Optimizations . RemoveDynamicRegistrar = App . Optimizations . RemoveDynamicRegistrar ;
[mtouch/mmp] Fix tracking of whether the static registrar should run again or not. Fixes #641. (#3534)
* [tests] Improve debug spew for the RebuildTest_WithExtensions test.
* [mtouch/mmp] Store/load if the dynamic registrar is removed or not into the cached link results.
Store/load if the dynamic registrar is removed or not into the cached link
results, so that we generate the correct main.m even if cached linker results
are used.
* [mtouch/mmp] The static registrar must not execute if we're loading cached results from the linker.
The static registrar must not execute if we're loading cached results from the
linker, because the static registrar needs information from the linker that's
not restored from the cache.
* [mtouch/mmp] Share Touch code.
* [mtouch/mmp] Make it possible to touch inexistent files (to create them).
* [mtouch/mmp] Fix tracking of whether the static registrar should run again or not.
The recent changes to support optimizing away the dynamic registrar caused the
Xamarin.MTouch.RebuildTest_WithExtensions test to regress.
The problem
-----------
* The linker now collects and stores information the static registrar needs.
* This information is not restored from disk when the linker realizes that it
can reload previously linked assemblies instead of executing again.
* The static registrar runs again (for another reason).
* The information the static registrar needs isn't available, and incorrect
output follows.
So fix 1: show an error if the static registrar runs when the linker loaded
cached results.
The exact scenario the test ran into is this:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk (this is an optimization to avoid
compiling the registrar.m file again unless needed).
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is newer than
registrar.m's timestamp and run again, but doesn't produce the right result
because it doesn't have the information it needs.
Considered solutions
--------------------
1. Only track timestamps, not file contents. This is not ideal, since it will
result in more work done: in particular for the case above, it would add a
registrar.m compilation in build #2, and linker rerun + static registrar
rerun + registrar.m compilation + final native link in build #3.
2. Always write the output of the static registrar, even if it hasn't changed.
This is not ideal either, since it will also result in more work done: for
the case above, it would add a registrar.m compilation + final native link
in build #3.
3. Always write the output of the static registrar, but track if it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it.
This only means the final native link in build #3 is added (see #5 for why
this is worse than it sounds).
4. Always write the output of the static registrar, but track it it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it,
and track that too, so that the final native link in build #3 isn't needed
anymore. Unfortunately this may result in incorrect behavior, because now
the msbuild tasks will detect that the executable has changed, and may run
dsymutil + strip again. The executable didn't actually change, which means
it would be the previously stripped executable, and thus we'd end up with
an empty .dSYM because we ran dsymtil on an already stripped executable.
5. Idea #4, but write the output of the final link into a temporary directory
instead of the .app, so that we could track whether we should update the
executable in the .app or not. This is not optimal either, because
executables can be *big* (I've seen multi-GB tvOS bitcode executables), and
extra copies of such files should not be taken lightly.
6. Idea #4, but tell the MSBuild tasks that dsymutil/strip doesn't need to be
rerun even if the timestamp of the executable changed. This might actually
work, but now the solution's become quite complex.
Implemented solution
--------------------
Use stamp files to detect whether a file is up-to-date or not.
In particular:
* When we don't write to a file because the new contents are identical to the
old contents, we now touch a .stamp file. This stamp file means "the
accompanying file was determined to be up-to-date when the stamp was
touched."
* When checking whether a file is up-to-date, also check for the presence of a
.stamp file, and if it exists, use the highest timestamp between the stamp
file and the actual file.
Now the test scenario becomes:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk, but it creates a registrar.m.stamp
file to indicate the point in time when registrar.m was considered up-to-
date.
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is *older* than
registrar.m.stamp's timestamp and doesn't run again.
We only use the stamp file for source code (registrar.[m|h], main.[m|h],
pinvokes.[m|h]), since using it every time has too much potential for running
into other problems (for instance we should never create .stamp files inside
the .app).
Fixes these test failures:
1) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single","",False,System.String[])
single
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory371/testApp.app/testApp is modified, timestamp: 2/15/2018 3:04:11 PM > 2/15/2018 3:04:09 PM" >
2) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("dual","armv7,arm64",False,System.String[])
dual
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory375/testApp.app/testApp is modified, timestamp: 2/15/2018 3:06:03 PM > 2/15/2018 3:06:00 PM" >
3) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("llvm","armv7+llvm",False,System.String[])
llvm
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory379/testApp.app/testApp is modified, timestamp: 2/15/2018 3:07:14 PM > 2/15/2018 3:07:12 PM" >
4) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("debug","",True,System.String[])
debug
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory383/testApp.app/testApp is modified, timestamp: 2/15/2018 3:08:16 PM > 2/15/2018 3:08:13 PM" >
5) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single-framework","",False,System.String[])
single-framework
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory387/testApp.app/testApp is modified, timestamp: 2/15/2018 3:09:18 PM > 2/15/2018 3:09:16 PM" >
Fixes https://github.com/xamarin/maccore/issues/641
2018-02-19 22:28:04 +03:00
Driver . Log ( 1 , $"Optimization dynamic registrar removal loaded from cached results: {(App.Optimizations.RemoveDynamicRegistrar.HasValue ? (App.Optimizations.RemoveUIThreadChecks.Value ? " enabled " : " disabled ") : " not set ")}" ) ;
break ;
default :
// key: app(ex)
// value: assembly
List < string > asms ;
if ( ! cached_output . TryGetValue ( key , out asms ) )
cached_output [ key ] = asms = new List < string > ( ) ;
asms . Add ( value ) ;
break ;
}
2016-04-21 15:58:45 +03:00
}
}
2017-01-24 13:10:20 +03:00
var cache_valid = true ;
foreach ( var target in allTargets ) {
List < string > cached_files ;
if ( ! cached_output . TryGetValue ( target . App . AppDirectory , out cached_files ) ) {
cache_valid = false ;
Driver . Log ( 2 , $"The cached assemblies are not valid because there are no cached assemblies for {target.App.Name}." ) ;
break ;
2016-04-21 15:58:45 +03:00
}
2017-01-24 13:10:20 +03:00
var outputs = new List < string > ( ) ;
var inputs = new List < string > ( cached_files ) ;
foreach ( var input in inputs . ToArray ( ) ) {
var output = Path . Combine ( PreBuildDirectory , Path . GetFileName ( input ) ) ;
outputs . Add ( output ) ;
if ( File . Exists ( input + ".mdb" ) ) {
// Debug files can change without the assemblies themselves changing
// This should also invalidate the cached linker results, since the non-linked mdbs can't be copied.
inputs . Add ( input + ".mdb" ) ;
outputs . Add ( output + ".mdb" ) ;
2016-04-21 15:58:45 +03:00
}
2017-03-24 19:58:05 +03:00
var pdb = Path . ChangeExtension ( input , "pdb" ) ;
if ( File . Exists ( pdb ) ) {
inputs . Add ( pdb ) ;
outputs . Add ( Path . ChangeExtension ( output , "pdb" ) ) ;
}
2017-02-01 17:47:54 +03:00
if ( File . Exists ( input + ".config" ) ) {
// If a config file changes, then the AOT-compiled output can be different,
// so make sure to take config files into account as well.
inputs . Add ( input + ".config" ) ;
outputs . Add ( output + ".config" ) ;
}
2016-04-21 15:58:45 +03:00
}
2017-01-24 13:10:20 +03:00
if ( ! cache_valid )
break ;
2016-04-21 15:58:45 +03:00
2017-01-24 13:10:20 +03:00
if ( ! Application . IsUptodate ( inputs , outputs ) ) {
Driver . Log ( 2 , $"The cached assemblies are not valid because some of the assemblies in {target.App.Name} are out-of-date." ) ;
cache_valid = false ;
break ;
}
2016-04-21 15:58:45 +03:00
}
2017-02-01 17:52:19 +03:00
cached_link = cache_valid ;
2016-04-21 15:58:45 +03:00
}
}
2017-01-24 13:10:20 +03:00
List < AssemblyDefinition > output_assemblies ;
if ( cached_link ) {
Driver . Log ( 2 , $"Reloading cached assemblies." ) ;
output_assemblies = new List < AssemblyDefinition > ( ) ;
2017-02-01 17:50:20 +03:00
foreach ( var file in cached_output . Values . SelectMany ( ( v ) = > v ) . Select ( ( v ) = > Path . GetFileName ( v ) ) . Distinct ( ) )
output_assemblies . Add ( Resolver . Load ( Path . Combine ( PreBuildDirectory , file ) ) ) ;
2017-01-24 13:10:20 +03:00
Driver . Watch ( "Cached assemblies reloaded" , 1 ) ;
Driver . Log ( "Cached assemblies reloaded." ) ;
} else {
// Load the assemblies into memory.
foreach ( var a in Assemblies )
a . LoadAssembly ( a . FullPath ) ;
// Link!
2018-05-29 20:08:34 +03:00
Driver . Watch ( "Managed Link Preparation" , 1 ) ;
2017-01-24 13:10:20 +03:00
LinkAssemblies ( out output_assemblies , PreBuildDirectory , sharingTargets ) ;
}
2016-04-21 15:58:45 +03:00
2017-02-01 17:51:52 +03:00
// Verify that we don't get multiple identical assemblies from the linker.
foreach ( var group in output_assemblies . GroupBy ( ( v ) = > v . Name . Name ) ) {
if ( group . Count ( ) ! = 1 )
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , "Internal error {0}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." ,
2017-02-01 17:51:52 +03:00
$"The linker output contains more than one assemblies named '{group.Key}':\n\t{string.Join (" \ n \ t ", group.Select ((v) => v.MainModule.FileName).ToArray ())}" ) ;
}
2016-04-21 15:58:45 +03:00
2017-01-24 13:10:20 +03:00
// Update (add/remove) list of assemblies in each app, since the linker may have both added and removed assemblies.
// The logic for updating assemblies when doing code-sharing is not equivalent to when we're not code sharing
// (in particular code sharing is not supported when there are xml linker definitions), so we need
// to maintain two paths here.
if ( sharingTargets . Count = = 0 ) {
Assemblies . Update ( this , output_assemblies ) ;
} else {
// For added assemblies we have to determine exactly which apps need which assemblies.
// Code sharing is only allowed if there are no linker xml definitions, nor any I18N values, which means that
// we can limit ourselves to iterate over assembly references to create the updated list of assemblies.
foreach ( var t in allTargets ) {
// Find the root assembly
// Here we assume that 'AssemblyReference.Name' == 'Assembly.Identity'.
2017-04-07 15:13:11 +03:00
var rootAssemblies = new List < Assembly > ( ) ;
foreach ( var root in t . App . RootAssemblies )
rootAssemblies . Add ( t . Assemblies [ Assembly . GetIdentity ( root ) ] ) ;
2017-01-24 13:10:20 +03:00
var queue = new Queue < string > ( ) ;
var collectedNames = new HashSet < string > ( ) ;
// First collect the set of all assemblies in the app by walking the assembly references.
2017-04-07 15:13:11 +03:00
foreach ( var root in rootAssemblies )
queue . Enqueue ( root . Identity ) ;
2017-01-24 13:10:20 +03:00
do {
var next = queue . Dequeue ( ) ;
collectedNames . Add ( next ) ;
2017-02-01 17:51:52 +03:00
var ad = output_assemblies . SingleOrDefault ( ( AssemblyDefinition v ) = > v . Name . Name = = next ) ;
if ( ad = = null )
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , "Internal error {0}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , $"The assembly {next} was referenced by another assembly, but at the same time linked out by the linker." ) ;
2017-01-24 13:10:20 +03:00
if ( ad . MainModule . HasAssemblyReferences ) {
foreach ( var ar in ad . MainModule . AssemblyReferences ) {
if ( ! collectedNames . Contains ( ar . Name ) & & ! queue . Contains ( ar . Name ) )
queue . Enqueue ( ar . Name ) ;
}
}
} while ( queue . Count > 0 ) ;
// Now update the assembly collection
var appexAssemblies = collectedNames . Select ( ( v ) = > output_assemblies . Single ( ( v2 ) = > v2 . Name . Name = = v ) ) ;
t . Assemblies . Update ( t , appexAssemblies ) ;
// And make sure every Target's assembly resolver knows about all the assemblies.
foreach ( var asm in t . Assemblies )
t . Resolver . Add ( asm . AssemblyDefinition ) ;
}
2016-04-21 15:58:45 +03:00
2018-03-02 11:39:42 +03:00
// Find assemblies that are in more than 1 appex, but not in the container app.
// These assemblies will be bundled once into the container .app instead of in each appex.
var grouped = sharingTargets . SelectMany ( ( v ) = > v . Assemblies ) .
GroupBy ( ( v ) = > Assembly . GetIdentity ( v . AssemblyDefinition ) ) .
Where ( ( v ) = > ! Assemblies . ContainsKey ( v . Key ) ) .
Where ( ( v ) = > v . Count ( ) > 1 ) ;
foreach ( var gr in grouped ) {
var asm = gr . First ( ) ;
Assemblies . Add ( asm ) ;
Resolver . Add ( asm . AssemblyDefinition ) ;
gr . All ( ( v ) = > v . BundleInContainerApp = true ) ;
}
2017-01-31 19:38:41 +03:00
// If any of the appex'es build to a grouped SDK framework, then we must ensure that all SDK assemblies
// in that appex are also in the container app.
foreach ( var st in sharingTargets ) {
if ( ! st . App . ContainsGroupedSdkAssemblyBuildTargets )
2016-04-21 15:58:45 +03:00
continue ;
2017-01-31 19:38:41 +03:00
foreach ( var asm in st . Assemblies . Where ( ( v ) = > Profile . IsSdkAssembly ( v . AssemblyDefinition ) | | Profile . IsProductAssembly ( v . AssemblyDefinition ) ) ) {
if ( ! Assemblies . ContainsKey ( asm . Identity ) ) {
Driver . Log ( 2 , $"The SDK assembly {asm.Identity} will be included in the app because it's referenced by the extension {st.App.Name}" ) ;
Assemblies . Add ( asm ) ;
}
}
2016-04-21 15:58:45 +03:00
}
}
2017-01-24 13:10:20 +03:00
// Write the input files to the cache
using ( var writer = new StreamWriter ( cache_path , false ) ) {
[mtouch/mmp] Fix tracking of whether the static registrar should run again or not. Fixes #641. (#3534)
* [tests] Improve debug spew for the RebuildTest_WithExtensions test.
* [mtouch/mmp] Store/load if the dynamic registrar is removed or not into the cached link results.
Store/load if the dynamic registrar is removed or not into the cached link
results, so that we generate the correct main.m even if cached linker results
are used.
* [mtouch/mmp] The static registrar must not execute if we're loading cached results from the linker.
The static registrar must not execute if we're loading cached results from the
linker, because the static registrar needs information from the linker that's
not restored from the cache.
* [mtouch/mmp] Share Touch code.
* [mtouch/mmp] Make it possible to touch inexistent files (to create them).
* [mtouch/mmp] Fix tracking of whether the static registrar should run again or not.
The recent changes to support optimizing away the dynamic registrar caused the
Xamarin.MTouch.RebuildTest_WithExtensions test to regress.
The problem
-----------
* The linker now collects and stores information the static registrar needs.
* This information is not restored from disk when the linker realizes that it
can reload previously linked assemblies instead of executing again.
* The static registrar runs again (for another reason).
* The information the static registrar needs isn't available, and incorrect
output follows.
So fix 1: show an error if the static registrar runs when the linker loaded
cached results.
The exact scenario the test ran into is this:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk (this is an optimization to avoid
compiling the registrar.m file again unless needed).
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is newer than
registrar.m's timestamp and run again, but doesn't produce the right result
because it doesn't have the information it needs.
Considered solutions
--------------------
1. Only track timestamps, not file contents. This is not ideal, since it will
result in more work done: in particular for the case above, it would add a
registrar.m compilation in build #2, and linker rerun + static registrar
rerun + registrar.m compilation + final native link in build #3.
2. Always write the output of the static registrar, even if it hasn't changed.
This is not ideal either, since it will also result in more work done: for
the case above, it would add a registrar.m compilation + final native link
in build #3.
3. Always write the output of the static registrar, but track if it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it.
This only means the final native link in build #3 is added (see #5 for why
this is worse than it sounds).
4. Always write the output of the static registrar, but track it it changed or
not, and if it didn't, just touch registrar.o instead of recompiling it,
and track that too, so that the final native link in build #3 isn't needed
anymore. Unfortunately this may result in incorrect behavior, because now
the msbuild tasks will detect that the executable has changed, and may run
dsymutil + strip again. The executable didn't actually change, which means
it would be the previously stripped executable, and thus we'd end up with
an empty .dSYM because we ran dsymtil on an already stripped executable.
5. Idea #4, but write the output of the final link into a temporary directory
instead of the .app, so that we could track whether we should update the
executable in the .app or not. This is not optimal either, because
executables can be *big* (I've seen multi-GB tvOS bitcode executables), and
extra copies of such files should not be taken lightly.
6. Idea #4, but tell the MSBuild tasks that dsymutil/strip doesn't need to be
rerun even if the timestamp of the executable changed. This might actually
work, but now the solution's become quite complex.
Implemented solution
--------------------
Use stamp files to detect whether a file is up-to-date or not.
In particular:
* When we don't write to a file because the new contents are identical to the
old contents, we now touch a .stamp file. This stamp file means "the
accompanying file was determined to be up-to-date when the stamp was
touched."
* When checking whether a file is up-to-date, also check for the presence of a
.stamp file, and if it exists, use the highest timestamp between the stamp
file and the actual file.
Now the test scenario becomes:
* 1st build: everything is new and everything is built.
* 2nd build: contents of .exe changes, the linker runs again, the static
registrar runs again, but sees that the generated output didn't change, so
it doesn't write the new content to disk, but it creates a registrar.m.stamp
file to indicate the point in time when registrar.m was considered up-to-
date.
* 3rd build: only the .exe timestamp changes, the linker sees nothing changes
in the contents of the .exe and loads the previously linked assemblies from
disk, the static registrar sees that the .exe's timestamp is *older* than
registrar.m.stamp's timestamp and doesn't run again.
We only use the stamp file for source code (registrar.[m|h], main.[m|h],
pinvokes.[m|h]), since using it every time has too much potential for running
into other problems (for instance we should never create .stamp files inside
the .app).
Fixes these test failures:
1) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single","",False,System.String[])
single
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory371/testApp.app/testApp is modified, timestamp: 2/15/2018 3:04:11 PM > 2/15/2018 3:04:09 PM" >
2) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("dual","armv7,arm64",False,System.String[])
dual
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory375/testApp.app/testApp is modified, timestamp: 2/15/2018 3:06:03 PM > 2/15/2018 3:06:00 PM" >
3) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("llvm","armv7+llvm",False,System.String[])
llvm
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory379/testApp.app/testApp is modified, timestamp: 2/15/2018 3:07:14 PM > 2/15/2018 3:07:12 PM" >
4) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("debug","",True,System.String[])
debug
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory383/testApp.app/testApp is modified, timestamp: 2/15/2018 3:08:16 PM > 2/15/2018 3:08:13 PM" >
5) Failed : Xamarin.MTouch.RebuildTest_WithExtensions("single-framework","",False,System.String[])
single-framework
Expected: <empty>
But was: < "/Users/builder/data/lanes/5746/4123bf7e/source/xamarin-macios/tests/mtouch/bin/Debug/tmp-test-dir/Xamarin.Tests.BundlerTool.CreateTemporaryDirectory387/testApp.app/testApp is modified, timestamp: 2/15/2018 3:09:18 PM > 2/15/2018 3:09:16 PM" >
Fixes https://github.com/xamarin/maccore/issues/641
2018-02-19 22:28:04 +03:00
var opt = App . Optimizations . RemoveDynamicRegistrar ;
writer . WriteLine ( $"RemoveDynamicRegistrar:{(opt.HasValue ? (opt.Value ? " true " : " false ") : string.Empty)}" ) ;
2017-01-24 13:10:20 +03:00
foreach ( var target in allTargets ) {
foreach ( var asm in target . Assemblies ) {
writer . WriteLine ( $"{target.App.AppDirectory}:{asm.FullPath}" ) ;
}
}
2016-04-21 15:58:45 +03:00
}
2017-01-24 13:10:20 +03:00
// Now the assemblies are in PreBuildDirectory, and they need to be in the BuildDirectory for the AOT compiler.
foreach ( var t in allTargets ) {
foreach ( var a in t . Assemblies ) {
// All these assemblies are in the main app's PreBuildDirectory.
a . FullPath = Path . Combine ( PreBuildDirectory , a . FileName ) ;
// The linker can copy files (and not update timestamps), and then we run into this sequence:
// * We run the linker, nothing changes, so the linker copies
// all files to the PreBuild directory, with timestamps intact.
// * This means that for instance SDK assemblies will have the original
// timestamp from their installed location, and the exe will have the
// timestamp of when it was built.
// * mtouch is executed again for some reason, and none of the input assemblies changed.
// We'll still re-execute the linker, because at least one of the input assemblies
// (the .exe) has a newer timestamp than some of the assemblies in the PreBuild directory.
// So here we manually touch all the assemblies we have, to make sure their timestamps
// change (this is us saying 'we know these files are up-to-date at this point in time').
2017-02-01 17:49:32 +03:00
if ( ! cached_link ) {
Driver . Touch ( a . FullPath ) ;
if ( File . Exists ( a . FullPath + ".mdb" ) )
Driver . Touch ( a . FullPath + ".mdb" ) ;
2017-03-24 19:58:05 +03:00
var pdb = Path . ChangeExtension ( a . FullPath , "pdb" ) ;
if ( File . Exists ( pdb ) )
Driver . Touch ( pdb ) ;
2017-05-29 17:14:38 +03:00
var config = a . FullPath + ".config" ;
if ( File . Exists ( config ) )
Driver . Touch ( config ) ;
2017-02-01 17:49:32 +03:00
}
2016-04-21 15:58:45 +03:00
2017-01-24 13:10:20 +03:00
// Now copy to the build directory
var target = Path . Combine ( BuildDirectory , a . FileName ) ;
if ( ! a . CopyAssembly ( a . FullPath , target ) )
Driver . Log ( 3 , "Target '{0}' is up-to-date." , target ) ;
a . FullPath = target ;
}
2017-01-20 12:45:08 +03:00
}
2016-04-21 15:58:45 +03:00
2017-01-24 13:10:20 +03:00
// Set the 'linked' flag for the targets sharing code, so that this method can be called
// again, and it won't do anything for the appex's sharing code with the main app (but
// will still work for any appex's not sharing code).
allTargets . ForEach ( ( v ) = > v . linked = true ) ;
2016-04-21 15:58:45 +03:00
}
public void ProcessAssemblies ( )
{
/ /
// * Linking
// Copy assemblies to LinkDirectory
2016-11-09 14:05:59 +03:00
// Link and save to PreBuildDirectory
2016-05-11 14:27:51 +03:00
// If marshalling native exceptions:
2016-11-09 14:05:59 +03:00
// * Generate/calculate P/Invoke wrappers and save to PreBuildDirectory
2016-04-21 15:58:45 +03:00
// [AOT assemblies in BuildDirectory]
// Strip managed code save to TargetDirectory (or just copy the file if stripping is disabled).
/ /
// * No linking
2016-05-11 14:27:51 +03:00
// If marshalling native exceptions:
2016-11-09 14:05:59 +03:00
// Generate/calculate P/Invoke wrappers and save to PreBuildDirectory.
2016-05-11 14:27:51 +03:00
// If not marshalling native exceptions:
2016-11-09 14:05:59 +03:00
// Copy assemblies to PreBuildDirectory
// Copy unmodified assemblies to BuildDirectory
2016-04-21 15:58:45 +03:00
// [AOT assemblies in BuildDirectory]
// Strip managed code save to TargetDirectory (or just copy the file if stripping is disabled).
/ /
// Note that we end up copying assemblies around quite much,
// this is because we we're comparing contents instead of
// filestamps, so we need the previous file around to be
// able to do the actual comparison. For instance: in the
// 'No linking' case above, we copy the assembly to PreBuild
// before removing the resources and saving that result to Build.
// The copy in PreBuild is required for the next build iteration,
// to see if the original assembly has been modified or not (the
// file in the Build directory might be different due to resource
// removal even if the original assembly didn't change).
/ /
// This can probably be improved by storing digests/hashes instead
// of the entire files, but this turned out a bit messy when
// trying to make it work with the linker, so I decided to go for
// simple file copying for now.
/ /
//
// Other notes:
/ /
// * We need all assemblies in the same directory when doing AOT-compilation.
// * We cannot overwrite in-place, because it will mess up dependency tracking
// and besides if we overwrite in place we might not be able to ignore
// insignificant changes (such as only a GUID change - the code is identical,
// but we still need the original assembly since the AOT-ed image also stores
// the GUID, and we fail at runtime if the GUIDs in the assembly and the AOT-ed
// image don't match - if we overwrite in-place we lose the original assembly and
// its GUID).
//
2018-02-19 23:50:20 +03:00
LinkDirectory = Path . Combine ( ArchDirectory , "1-Link" ) ;
2016-04-21 15:58:45 +03:00
if ( ! Directory . Exists ( LinkDirectory ) )
Directory . CreateDirectory ( LinkDirectory ) ;
2018-02-19 23:50:20 +03:00
PreBuildDirectory = Path . Combine ( ArchDirectory , "2-PreBuild" ) ;
2016-11-09 14:05:59 +03:00
if ( ! Directory . Exists ( PreBuildDirectory ) )
Directory . CreateDirectory ( PreBuildDirectory ) ;
2018-02-19 23:50:20 +03:00
BuildDirectory = Path . Combine ( ArchDirectory , "3-Build" ) ;
2016-04-21 15:58:45 +03:00
if ( ! Directory . Exists ( BuildDirectory ) )
Directory . CreateDirectory ( BuildDirectory ) ;
if ( ! Directory . Exists ( TargetDirectory ) )
Directory . CreateDirectory ( TargetDirectory ) ;
2017-11-16 18:18:18 +03:00
ValidateAssembliesBeforeLink ( ) ;
2016-04-21 15:58:45 +03:00
ManagedLink ( ) ;
2017-04-05 22:38:40 +03:00
GatherFrameworks ( ) ;
2016-04-21 15:58:45 +03:00
}
2016-05-11 14:27:51 +03:00
2017-01-25 14:40:25 +03:00
public void CompilePInvokeWrappers ( )
{
if ( ! App . RequiresPInvokeWrappers )
return ;
2016-11-09 14:05:59 +03:00
2017-04-11 20:20:22 +03:00
if ( ! App . HasFrameworksDirectory & & App . IsCodeShared )
return ;
2017-01-25 14:40:25 +03:00
// Write P/Invokes
var state = MarshalNativeExceptionsState ;
if ( state . Started ) {
// The generator is 'started' by the linker, which means it may not
// be started if the linker was not executed due to re-using cached results.
state . End ( ) ;
2016-11-09 14:05:59 +03:00
}
2016-04-21 15:58:45 +03:00
2017-01-25 14:40:25 +03:00
var ifile = state . SourcePath ;
2017-01-25 21:43:26 +03:00
var mode = App . LibPInvokesLinkMode ;
foreach ( var abi in GetArchitectures ( mode ) ) {
2017-01-25 14:40:25 +03:00
var arch = abi . AsArchString ( ) ;
2017-01-25 21:43:26 +03:00
string ofile ;
2016-04-21 15:58:45 +03:00
2017-01-25 21:43:26 +03:00
switch ( mode ) {
case AssemblyBuildTarget . StaticObject :
ofile = Path . Combine ( App . Cache . Location , arch , "libpinvokes.a" ) ;
break ;
case AssemblyBuildTarget . DynamicLibrary :
ofile = Path . Combine ( App . Cache . Location , arch , "libpinvokes.dylib" ) ;
break ;
case AssemblyBuildTarget . Framework :
ofile = Path . Combine ( App . Cache . Location , arch , "Xamarin.PInvokes.framework" , "Xamarin.PInvokes" ) ;
var plist_path = Path . Combine ( Path . GetDirectoryName ( ofile ) , "Info.plist" ) ;
var fw_name = Path . GetFileNameWithoutExtension ( ofile ) ;
App . CreateFrameworkInfoPList ( plist_path , fw_name , App . BundleId + ".frameworks." + fw_name , fw_name ) ;
break ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , mode ) ;
2017-01-25 21:43:26 +03:00
}
2017-01-25 14:40:25 +03:00
2017-01-25 19:14:28 +03:00
var pinvoke_task = new PinvokesTask
{
Target = this ,
Abi = abi ,
InputFile = ifile ,
OutputFile = ofile ,
2017-01-25 21:43:26 +03:00
SharedLibrary = mode ! = AssemblyBuildTarget . StaticObject ,
2017-01-25 19:14:28 +03:00
Language = "objective-c++" ,
} ;
2018-06-28 15:49:22 +03:00
pinvoke_task . CompilerFlags . AddStandardCppLibrary ( ) ;
2017-01-25 21:43:26 +03:00
if ( pinvoke_task . SharedLibrary ) {
if ( mode = = AssemblyBuildTarget . Framework ) {
var name = Path . GetFileNameWithoutExtension ( ifile ) ;
pinvoke_task . InstallName = $"@rpath/{name}.framework/{name}" ;
AddToBundle ( pinvoke_task . OutputFile , $"Frameworks/{name}.framework/{name}" , dylib_to_framework_conversion : true ) ;
} else {
pinvoke_task . InstallName = $"@rpath/{Path.GetFileName (ofile)}" ;
AddToBundle ( pinvoke_task . OutputFile ) ;
}
2017-01-25 19:14:28 +03:00
pinvoke_task . CompilerFlags . AddFramework ( "Foundation" ) ;
pinvoke_task . CompilerFlags . LinkWithXamarin ( ) ;
2017-01-25 14:40:25 +03:00
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
pinvoke_tasks . Add ( abi , pinvoke_task ) ;
2017-01-25 14:40:25 +03:00
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
LinkWithTaskOutput ( pinvoke_task ) ;
2017-01-25 14:40:25 +03:00
}
2016-04-21 15:58:45 +03:00
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
void AOTCompile ( )
{
if ( App . IsSimulatorBuild )
return ;
// Here we create the tasks to run the AOT compiler.
foreach ( var a in Assemblies ) {
2018-05-05 00:59:31 +03:00
if ( ! a . IsAOTCompiled )
continue ;
2017-01-26 15:19:00 +03:00
foreach ( var abi in GetArchitectures ( a . BuildTarget ) ) {
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
a . CreateAOTTask ( abi ) ;
}
}
2017-01-26 15:19:00 +03:00
// Group the assemblies according to their target name, and link everything together accordingly.
var grouped = Assemblies . GroupBy ( ( arg ) = > arg . BuildTargetName ) ;
foreach ( var @group in grouped ) {
var name = @group . Key ;
var assemblies = @group . AsEnumerable ( ) . ToArray ( ) ;
2018-05-05 00:59:31 +03:00
if ( assemblies . Length < = 0 )
continue ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
// We ensure elsewhere that all assemblies in a group have the same build target.
2017-01-26 15:19:00 +03:00
var build_target = assemblies [ 0 ] . BuildTarget ;
foreach ( var abi in GetArchitectures ( build_target ) ) {
Driver . Log ( 2 , "Building {0} from {1}" , name , string . Join ( ", " , assemblies . Select ( ( arg1 ) = > Path . GetFileName ( arg1 . FileName ) ) ) ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
string install_name ;
string compiler_output ;
var compiler_flags = new CompilerFlags ( this ) ;
var link_dependencies = new List < CompileTask > ( ) ;
2018-05-05 00:59:31 +03:00
var infos = assemblies . Where ( ( asm ) = > asm . AotInfos . ContainsKey ( abi ) ) . Select ( ( asm ) = > asm . AotInfos [ abi ] ) . ToList ( ) ;
2017-01-26 15:19:00 +03:00
var aottasks = infos . Select ( ( info ) = > info . Task ) ;
2018-05-05 00:59:31 +03:00
if ( aottasks = = null )
continue ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
2017-01-24 13:10:20 +03:00
var existingLinkTask = infos . Where ( ( v ) = > v . LinkTask ! = null ) . Select ( ( v ) = > v . LinkTask ) . ToList ( ) ;
if ( existingLinkTask . Count > 0 ) {
if ( existingLinkTask . Count ! = infos . Count )
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , "Internal error: {0}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , $"Not all assemblies for {name} have link tasks" ) ;
2017-01-24 13:10:20 +03:00
if ( ! existingLinkTask . All ( ( v ) = > v = = existingLinkTask [ 0 ] ) )
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , "Internal error: {0}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , $"Link tasks for {name} aren't all the same" ) ;
2017-01-24 13:10:20 +03:00
LinkWithBuildTarget ( build_target , name , existingLinkTask [ 0 ] , assemblies ) ;
continue ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
// We have to compile any source files to object files before we can link.
2017-01-26 15:19:00 +03:00
var sources = infos . SelectMany ( ( info ) = > info . AsmFiles ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
if ( sources . Count ( ) > 0 ) {
foreach ( var src in sources ) {
// We might have to convert .s to bitcode assembly (.ll) first
var assembly = src ;
BitCodeifyTask bitcode_task = null ;
if ( App . EnableAsmOnlyBitCode ) {
bitcode_task = new BitCodeifyTask ( )
{
Input = assembly ,
OutputFile = Path . ChangeExtension ( assembly , ".ll" ) ,
Platform = App . Platform ,
Abi = abi ,
DeploymentTarget = App . DeploymentTarget ,
} ;
2017-01-26 15:19:00 +03:00
bitcode_task . AddDependency ( aottasks ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
assembly = bitcode_task . OutputFile ;
}
// Compile assembly code (either .s or .ll) to object file
var compile_task = new CompileTask
{
Target = this ,
SharedLibrary = false ,
InputFile = assembly ,
OutputFile = Path . ChangeExtension ( assembly , ".o" ) ,
Abi = abi ,
Language = bitcode_task ! = null ? null : "assembler" ,
} ;
compile_task . AddDependency ( bitcode_task ) ;
2017-01-26 15:19:00 +03:00
compile_task . AddDependency ( aottasks ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
link_dependencies . Add ( compile_task ) ;
}
} else {
2017-01-26 15:19:00 +03:00
aot_dependencies . AddRange ( aottasks ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
}
var arch = abi . AsArchString ( ) ;
switch ( build_target ) {
case AssemblyBuildTarget . StaticObject :
LinkWithTaskOutput ( link_dependencies ) ; // Any .s or .ll files from the AOT compiler (compiled to object files)
2017-01-26 15:19:00 +03:00
foreach ( var info in infos ) {
LinkWithStaticLibrary ( info . ObjectFiles ) ;
LinkWithStaticLibrary ( info . BitcodeFiles ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
continue ; // no linking to do here.
case AssemblyBuildTarget . DynamicLibrary :
install_name = $"@rpath/lib{name}.dylib" ;
compiler_output = Path . Combine ( App . Cache . Location , arch , $"lib{name}.dylib" ) ;
break ;
2017-01-26 15:19:00 +03:00
case AssemblyBuildTarget . Framework :
install_name = $"@rpath/{name}.framework/{name}" ;
2017-04-03 12:52:29 +03:00
compiler_output = Path . Combine ( App . Cache . Location , arch , name ) ;
2017-01-26 15:19:00 +03:00
break ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , build_target ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
}
CompileTask pinvoke_task ;
if ( pinvoke_tasks . TryGetValue ( abi , out pinvoke_task ) )
link_dependencies . Add ( pinvoke_task ) ;
2017-01-26 15:19:00 +03:00
foreach ( var info in infos ) {
compiler_flags . AddLinkWith ( info . ObjectFiles ) ;
compiler_flags . AddLinkWith ( info . BitcodeFiles ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
foreach ( var task in link_dependencies )
compiler_flags . AddLinkWith ( task . OutputFile ) ;
2017-01-26 15:19:00 +03:00
foreach ( var a in assemblies ) {
compiler_flags . AddFrameworks ( a . Frameworks , a . WeakFrameworks ) ;
compiler_flags . AddLinkWith ( a . LinkWith , a . ForceLoad ) ;
compiler_flags . AddOtherFlags ( a . LinkerFlags ) ;
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
if ( a . HasLinkWithAttributes ) {
var symbols = GetRequiredSymbols ( a ) ;
switch ( App . SymbolMode ) {
case SymbolMode . Ignore :
break ;
case SymbolMode . Code :
var tasks = GenerateReferencingSource ( Path . Combine ( App . Cache . Location , Path . GetFileNameWithoutExtension ( a . FullPath ) + "-unresolved-externals.m" ) , symbols ) ;
foreach ( var task in tasks )
compiler_flags . AddLinkWith ( task . OutputFile ) ;
link_dependencies . AddRange ( tasks ) ;
break ;
case SymbolMode . Linker :
compiler_flags . ReferenceSymbols ( symbols ) ;
break ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , $"Internal error: invalid symbol mode: {App.SymbolMode}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." ) ;
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
}
}
2017-01-26 15:19:00 +03:00
}
2017-04-12 12:04:15 +03:00
if ( App . Embeddinator ) {
if ( ! string . IsNullOrEmpty ( App . UserGccFlags ) )
compiler_flags . AddOtherFlag ( App . UserGccFlags ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
compiler_flags . LinkWithMono ( ) ;
compiler_flags . LinkWithXamarin ( ) ;
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
if ( GetAllSymbols ( ) . Contains ( "UIApplicationMain" ) )
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
compiler_flags . AddFramework ( "UIKit" ) ;
if ( App . EnableLLVMOnlyBitCode ) {
// The AOT compiler doesn't optimize the bitcode so clang will do it
compiler_flags . AddOtherFlag ( "-fexceptions" ) ;
2017-01-26 15:19:00 +03:00
var optimizations = assemblies . Select ( ( a ) = > App . GetLLVMOptimizations ( a ) ) . Where ( ( opt ) = > opt ! = null ) . Distinct ( ) . ToList ( ) ;
if ( optimizations . Count = = 0 ) {
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
compiler_flags . AddOtherFlag ( "-O2" ) ;
2017-01-26 15:19:00 +03:00
} else if ( optimizations . Count = = 1 ) {
2019-02-11 13:00:02 +03:00
compiler_flags . AddOtherFlag ( optimizations [ 0 ] ) ;
2017-01-26 15:19:00 +03:00
} else {
throw ErrorHelper . CreateError ( 107 , "The assemblies '{0}' have different custom LLVM optimizations ('{1}'), which is not allowed when they are all compiled to a single binary." , string . Join ( ", " , assemblies . Select ( ( v ) = > v . Identity ) ) , string . Join ( "', '" , optimizations ) ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
}
}
2018-10-15 22:39:29 +03:00
HandleMonoNative ( App , compiler_flags ) ;
2019-02-11 13:00:02 +03:00
var link_task = new LinkTask ( )
{
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
Target = this ,
Abi = abi ,
OutputFile = compiler_output ,
InstallName = install_name ,
CompilerFlags = compiler_flags ,
Language = compiler_output . EndsWith ( ".s" , StringComparison . Ordinal ) ? "assembler" : null ,
SharedLibrary = build_target ! = AssemblyBuildTarget . StaticObject ,
} ;
link_task . AddDependency ( link_dependencies ) ;
2017-01-26 15:19:00 +03:00
link_task . AddDependency ( aottasks ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
2017-04-12 12:05:01 +03:00
if ( App . Embeddinator ) {
link_task . AddDependency ( link_with_task_output ) ;
link_task . CompilerFlags . AddLinkWith ( link_with_task_output . Select ( ( v ) = > v . OutputFile ) ) ;
embeddinator_tasks . Add ( link_task ) ;
}
2017-01-24 13:10:20 +03:00
LinkWithBuildTarget ( build_target , name , link_task , assemblies ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
2017-01-26 15:19:00 +03:00
foreach ( var info in infos )
info . LinkTask = link_task ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
}
}
2017-01-25 20:23:10 +03:00
2018-05-05 00:59:31 +03:00
if ( App . UseInterpreter )
2018-10-10 18:02:28 +03:00
/ * TODO : not sure ? we might have to continue here , depending on
* the set of assemblies are AOT ' d ? * /
2018-05-05 00:59:31 +03:00
return ;
2017-01-25 20:23:10 +03:00
// Code in one assembly (either in a P/Invoke or a third-party library) can depend on a third-party library in another assembly.
// This means that we must always build assemblies only when all their dependent assemblies have been built, so that
// we can link (natively) with the frameworks/dylibs for those dependent assemblies.
// Fortunately we can cheat a bit, since this can (currently at least) only happen for assemblies that
// have third-party libraries. This means that we only enforce this order for any assemblies that depend
// on other assemblies that have third-party libraries.
// Example:
// * We can build System.dll and mscorlib.dll in parallel, even if System.dll depends on mscorlib.dll,
// because we know that mscorlib.dll does not have any third-party libraries.
if ( Assemblies . All ( ( arg ) = > arg . HasDependencyMap ) ) {
var dict = Assemblies . ToDictionary ( ( arg ) = > Path . GetFileNameWithoutExtension ( arg . FileName ) ) ;
foreach ( var asm in Assemblies ) {
if ( ! asm . HasDependencyMap )
continue ;
if ( asm . BuildTarget = = AssemblyBuildTarget . StaticObject )
continue ;
if ( Profile . IsSdkAssembly ( asm . AssemblyDefinition ) | | Profile . IsProductAssembly ( asm . AssemblyDefinition ) ) {
//Console.WriteLine ("SDK assembly, so skipping assembly dependency checks: {0}", Path.GetFileNameWithoutExtension (asm.FileName));
continue ;
}
HashSet < Assembly > dependent_assemblies = new HashSet < Assembly > ( ) ;
foreach ( var dep in asm . DependencyMap ) {
Assembly dependentAssembly ;
if ( ! dict . TryGetValue ( Path . GetFileNameWithoutExtension ( dep ) , out dependentAssembly ) ) {
//Console.WriteLine ("Could not find dependency '{0}' of '{1}'", dep, asm.Identity);
continue ;
}
if ( asm = = dependentAssembly )
continue ; // huh?
// Nothing can depend on anything in our SDK, nor does our SDK depend on anything else in our SDK
// So we can remove any SDK dependency
if ( Profile . IsSdkAssembly ( dependentAssembly . AssemblyDefinition ) | | Profile . IsProductAssembly ( dependentAssembly . AssemblyDefinition ) ) {
//Console.WriteLine ("SDK assembly, so not a dependency of anything: {0}", Path.GetFileNameWithoutExtension (dependentAssembly.FileName));
continue ;
}
if ( ! dependentAssembly . HasLinkWithAttributes ) {
//Console.WriteLine ("Assembly {0} does not have LinkWith attributes, so there's nothing we can depend on.", dependentAssembly.Identity);
continue ;
}
if ( dependentAssembly . BuildTargetName = = asm . BuildTargetName ) {
//Console.WriteLine ("{0} is a dependency of {1}, but both are being built into the same target, so no dependency added.", Path.GetFileNameWithoutExtension (dep), Path.GetFileNameWithoutExtension (asm.FileName));
continue ;
}
//Console.WriteLine ("Added {0} as a dependency of {1}", Path.GetFileNameWithoutExtension (dep), Path.GetFileNameWithoutExtension (asm.FileName));
dependent_assemblies . Add ( dependentAssembly ) ;
}
// Circular dependencies shouldn't happen, but still make sure, since it's technically possible
// for users to do it.
foreach ( var abi in GetArchitectures ( asm . BuildTarget ) ) {
var target_task = asm . AotInfos [ abi ] . LinkTask ;
var dependent_tasks = dependent_assemblies . Select ( ( v ) = > v . AotInfos [ abi ] . LinkTask ) ;
var stack = new Stack < BuildTask > ( ) ;
foreach ( var dep in dependent_tasks ) {
stack . Clear ( ) ;
stack . Push ( target_task ) ;
if ( target_task = = dep | | IsCircularTask ( target_task , stack , dep ) ) {
Driver . Log ( "Found circular task." ) ;
Driver . Log ( "Task {0} (with output {1}) depends on:" , target_task . GetType ( ) . Name , target_task . Outputs . First ( ) ) ;
stack = new Stack < BuildTask > ( stack . Reverse ( ) ) ;
while ( stack . Count > 0 ) {
var node = stack . Pop ( ) ;
Driver . Log ( " -> {0} (Output: {1})" , node . GetType ( ) . Name , node . Outputs . First ( ) ) ;
}
} else {
target_task . AddDependency ( dep ) ;
target_task . CompilerFlags . AddLinkWith ( dep . OutputFile ) ;
}
}
}
}
}
}
bool IsCircularTask ( BuildTask root , Stack < BuildTask > stack , BuildTask task )
{
stack . Push ( task ) ;
foreach ( var d in task ? . Dependencies ) {
if ( stack . Contains ( d ) )
return true ;
if ( IsCircularTask ( root , stack , d ) )
return true ;
}
stack . Pop ( ) ;
return false ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
}
2016-05-11 14:10:35 +03:00
public void Compile ( )
{
2017-01-24 21:32:23 +03:00
// Compute the dependency map, and show warnings if there are any problems.
List < Exception > exceptions = new List < Exception > ( ) ;
foreach ( var a in Assemblies )
a . ComputeDependencyMap ( exceptions ) ;
if ( exceptions . Count > 0 ) {
ErrorHelper . Show ( exceptions ) ;
ErrorHelper . Warning ( 3006 , "Could not compute a complete dependency map for the project. This will result in slower build times because Xamarin.iOS can't properly detect what needs to be rebuilt (and what does not need to be rebuilt). Please review previous warnings for more details." ) ;
2016-04-21 15:58:45 +03:00
}
2017-01-25 19:14:28 +03:00
List < string > registration_methods = new List < string > ( ) ;
2016-04-21 15:58:45 +03:00
// The static registrar.
2016-09-30 22:02:17 +03:00
if ( App . Registrar = = RegistrarMode . Static ) {
2016-04-21 15:58:45 +03:00
var registrar_m = Path . Combine ( ArchDirectory , "registrar.m" ) ;
2016-05-11 14:10:35 +03:00
var registrar_h = Path . Combine ( ArchDirectory , "registrar.h" ) ;
2017-01-25 19:14:28 +03:00
var run_registrar_task = new RunRegistrarTask
{
Target = this ,
2017-02-17 12:20:30 +03:00
RegistrarCodePath = registrar_m ,
RegistrarHeaderPath = registrar_h ,
2017-01-25 19:14:28 +03:00
} ;
2016-04-21 15:58:45 +03:00
2017-01-26 15:19:00 +03:00
foreach ( var abi in GetArchitectures ( AssemblyBuildTarget . StaticObject ) ) {
2017-01-25 14:49:27 +03:00
var arch = abi . AsArchString ( ) ;
var ofile = Path . Combine ( App . Cache . Location , arch , Path . GetFileNameWithoutExtension ( registrar_m ) + ".o" ) ;
2017-01-25 19:14:28 +03:00
var registrar_task = new CompileRegistrarTask
{
Target = this ,
Abi = abi ,
InputFile = registrar_m ,
OutputFile = ofile ,
2017-02-17 12:20:30 +03:00
RegistrarCodePath = registrar_m ,
RegistrarHeaderPath = registrar_h ,
2017-01-25 19:14:28 +03:00
SharedLibrary = false ,
Language = "objective-c++" ,
} ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
registrar_task . AddDependency ( run_registrar_task ) ;
2017-01-25 19:14:28 +03:00
// This is because iOS has a forward declaration of NSPortMessage, but no actual declaration.
// They still use NSPortMessage in other API though, so it can't just be removed from our bindings.
registrar_task . CompilerFlags . AddOtherFlag ( "-Wno-receiver-forward-class" ) ;
2017-08-04 20:19:37 +03:00
2018-03-22 20:34:41 +03:00
// clang sometimes detects missing [super ...] calls, but clang doesn't know about
// calling super through managed code, so ignore those warnings.
registrar_task . CompilerFlags . AddOtherFlag ( "-Wno-objc-missing-super-calls" ) ;
2017-08-04 20:19:37 +03:00
if ( Driver . XcodeVersion > = new Version ( 9 , 0 ) )
registrar_task . CompilerFlags . AddOtherFlag ( "-Wno-unguarded-availability-new" ) ;
2018-06-28 15:49:22 +03:00
registrar_task . CompilerFlags . AddStandardCppLibrary ( ) ;
2018-06-09 04:45:24 +03:00
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
LinkWithTaskOutput ( registrar_task ) ;
2016-04-21 15:58:45 +03:00
}
2017-01-25 19:14:28 +03:00
registration_methods . Add ( "xamarin_create_classes" ) ;
2016-04-21 15:58:45 +03:00
}
2016-09-30 22:02:17 +03:00
if ( App . Registrar = = RegistrarMode . Dynamic & & App . IsSimulatorBuild & & App . LinkMode = = LinkMode . None ) {
2016-04-21 15:58:45 +03:00
string method ;
string library ;
switch ( App . Platform ) {
case ApplePlatform . iOS :
method = "xamarin_create_classes_Xamarin_iOS" ;
library = "Xamarin.iOS.registrar.a" ;
break ;
case ApplePlatform . WatchOS :
method = "xamarin_create_classes_Xamarin_WatchOS" ;
library = "Xamarin.WatchOS.registrar.a" ;
break ;
case ApplePlatform . TVOS :
method = "xamarin_create_classes_Xamarin_TVOS" ;
library = "Xamarin.TVOS.registrar.a" ;
break ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 71 , "Unknown platform: {0}. This usually indicates a bug in Xamarin.iOS; please file a bug report at https://github.com/xamarin/xamarin-macios/issues/new with a test case." , App . Platform ) ;
2016-04-21 15:58:45 +03:00
}
2017-06-06 23:21:26 +03:00
var lib = Path . Combine ( Driver . GetProductSdkDirectory ( App ) , "usr" , "lib" , library ) ;
if ( File . Exists ( lib ) ) {
registration_methods . Add ( method ) ;
LinkWithStaticLibrary ( lib ) ;
}
2016-04-21 15:58:45 +03:00
}
// The main method.
2017-01-26 15:19:00 +03:00
foreach ( var abi in GetArchitectures ( AssemblyBuildTarget . StaticObject ) ) {
2017-01-25 14:21:58 +03:00
var arch = abi . AsArchString ( ) ;
var main_m = Path . Combine ( App . Cache . Location , arch , "main.m" ) ;
2016-04-21 15:58:45 +03:00
2017-01-25 19:14:28 +03:00
var generate_main_task = new GenerateMainTask
{
Target = this ,
Abi = abi ,
MainM = main_m ,
RegistrationMethods = registration_methods ,
} ;
2017-01-25 14:21:58 +03:00
var main_o = Path . Combine ( App . Cache . Location , arch , "main.o" ) ;
2017-01-25 19:14:28 +03:00
var main_task = new CompileMainTask
{
Target = this ,
Abi = abi ,
InputFile = main_m ,
OutputFile = main_o ,
SharedLibrary = false ,
Language = "objective-c++" ,
} ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
main_task . AddDependency ( generate_main_task ) ;
2017-01-25 19:14:28 +03:00
main_task . CompilerFlags . AddDefine ( "MONOTOUCH" ) ;
2018-06-28 15:49:22 +03:00
main_task . CompilerFlags . AddStandardCppLibrary ( ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
LinkWithTaskOutput ( main_task ) ;
2016-04-21 15:58:45 +03:00
}
2017-04-12 12:05:01 +03:00
// Compile the managed assemblies into object files, frameworks or shared libraries
AOTCompile ( ) ;
2016-04-21 15:58:45 +03:00
Driver . Watch ( "Compile" , 1 ) ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
public void NativeLink ( BuildTasks build_tasks )
2016-04-21 15:58:45 +03:00
{
2017-04-12 14:24:00 +03:00
if ( App . Embeddinator & & App . IsDeviceBuild ) {
2017-04-12 12:05:01 +03:00
build_tasks . AddRange ( embeddinator_tasks ) ;
return ;
}
2016-04-21 15:58:45 +03:00
if ( ! string . IsNullOrEmpty ( App . UserGccFlags ) )
App . DeadStrip = false ;
if ( App . EnableLLVMOnlyBitCode )
App . DeadStrip = false ;
// Get global frameworks
2017-01-24 17:41:03 +03:00
linker_flags . AddFrameworks ( App . Frameworks , App . WeakFrameworks ) ;
linker_flags . AddFrameworks ( Frameworks , WeakFrameworks ) ;
2016-04-21 15:58:45 +03:00
// Collect all LinkWith flags and frameworks from all assemblies.
foreach ( var a in Assemblies ) {
2017-01-24 17:41:03 +03:00
linker_flags . AddFrameworks ( a . Frameworks , a . WeakFrameworks ) ;
2017-01-26 15:19:00 +03:00
if ( a . BuildTarget = = AssemblyBuildTarget . StaticObject )
2017-01-24 17:41:03 +03:00
linker_flags . AddLinkWith ( a . LinkWith , a . ForceLoad ) ;
linker_flags . AddOtherFlags ( a . LinkerFlags ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
if ( a . BuildTarget = = AssemblyBuildTarget . StaticObject ) {
2017-01-26 15:19:00 +03:00
foreach ( var abi in GetArchitectures ( a . BuildTarget ) ) {
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
AotInfo info ;
if ( ! a . AotInfos . TryGetValue ( abi , out info ) )
continue ;
linker_flags . AddLinkWith ( info . BitcodeFiles ) ;
linker_flags . AddLinkWith ( info . ObjectFiles ) ;
}
}
2016-04-21 15:58:45 +03:00
}
var bitcode = App . EnableBitCode ;
if ( bitcode )
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( App . EnableMarkerOnlyBitCode ? "-fembed-bitcode-marker" : "-fembed-bitcode" ) ;
2016-04-21 15:58:45 +03:00
if ( App . EnablePie . HasValue & & App . EnablePie . Value & & ( App . DeploymentTarget < new Version ( 4 , 2 ) ) )
ErrorHelper . Error ( 28 , "Cannot enable PIE (-pie) when targeting iOS 4.1 or earlier. Please disable PIE (-pie:false) or set the deployment target to at least iOS 4.2" ) ;
if ( ! App . EnablePie . HasValue )
App . EnablePie = true ;
if ( App . Platform = = ApplePlatform . iOS ) {
if ( App . EnablePie . Value & & ( App . DeploymentTarget > = new Version ( 4 , 2 ) ) ) {
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-Wl,-pie" ) ;
2016-04-21 15:58:45 +03:00
} else {
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-Wl,-no_pie" ) ;
2016-04-21 15:58:45 +03:00
}
}
2017-01-24 17:41:03 +03:00
CompileTask . GetArchFlags ( linker_flags , Abis ) ;
2016-04-21 15:58:45 +03:00
if ( App . IsDeviceBuild ) {
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( $"-m{Driver.GetTargetMinSdkName (App)}-version-min={App.DeploymentTarget}" ) ;
2017-06-06 23:32:25 +03:00
linker_flags . AddOtherFlag ( $"-isysroot {StringUtils.Quote (Driver.GetFrameworkDirectory (App))}" ) ;
2016-04-21 15:58:45 +03:00
} else {
2017-01-24 20:26:36 +03:00
CompileTask . GetSimulatorCompilerFlags ( linker_flags , false , App ) ;
2016-04-21 15:58:45 +03:00
}
2017-01-24 17:41:03 +03:00
linker_flags . LinkWithMono ( ) ;
2017-01-26 15:19:00 +03:00
if ( App . LibMonoLinkMode ! = AssemblyBuildTarget . StaticObject )
AddToBundle ( App . GetLibMono ( App . LibMonoLinkMode ) ) ;
2017-01-24 17:41:03 +03:00
linker_flags . LinkWithXamarin ( ) ;
2017-01-26 15:19:00 +03:00
if ( App . LibXamarinLinkMode ! = AssemblyBuildTarget . StaticObject )
AddToBundle ( App . GetLibXamarin ( App . LibXamarinLinkMode ) ) ;
2016-04-21 15:58:45 +03:00
2017-06-06 23:32:25 +03:00
linker_flags . AddOtherFlag ( $"-o {StringUtils.Quote (Executable)}" ) ;
2016-04-21 15:58:45 +03:00
2016-05-20 22:03:36 +03:00
bool need_libcpp = false ;
2016-04-21 15:58:45 +03:00
if ( App . EnableBitCode )
2016-05-20 22:03:36 +03:00
need_libcpp = true ;
#if ENABLE_BITCODE_ON_IOS
need_libcpp = true ;
#endif
if ( need_libcpp )
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-lc++" ) ;
2016-04-21 15:58:45 +03:00
// allow the native linker to remove unused symbols (if the caller was removed by the managed linker)
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
// Note that we include *all* (__Internal) p/invoked symbols here
// We also include any fields from [Field] attributes.
switch ( App . SymbolMode ) {
case SymbolMode . Ignore :
break ;
case SymbolMode . Code :
LinkWithTaskOutput ( GenerateReferencingSource ( Path . Combine ( App . Cache . Location , "reference.m" ) , GetRequiredSymbols ( ) ) ) ;
break ;
case SymbolMode . Linker :
2017-01-24 17:41:03 +03:00
linker_flags . ReferenceSymbols ( GetRequiredSymbols ( ) ) ;
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
break ;
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 99 , $"Internal error: invalid symbol mode: {App.SymbolMode}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." ) ;
2016-04-21 15:58:45 +03:00
}
2017-04-12 19:34:53 +03:00
var libdir = Path . Combine ( Driver . GetProductSdkDirectory ( App ) , "usr" , "lib" ) ;
if ( App . Embeddinator ) {
linker_flags . AddOtherFlag ( "-shared" ) ;
2017-06-06 23:32:25 +03:00
linker_flags . AddOtherFlag ( $"-install_name {StringUtils.Quote ($" @rpath / { App . ExecutableName } . framework / { App . ExecutableName } ")}" ) ;
2016-04-21 15:58:45 +03:00
} else {
2017-04-12 19:34:53 +03:00
string mainlib ;
if ( App . IsWatchExtension ) {
mainlib = "libwatchextension.a" ;
linker_flags . AddOtherFlag ( " -e _xamarin_watchextension_main" ) ;
} else if ( App . IsTVExtension ) {
mainlib = "libtvextension.a" ;
} else if ( App . IsExtension ) {
mainlib = "libextension.a" ;
} else {
mainlib = "libapp.a" ;
}
var libmain = Path . Combine ( libdir , mainlib ) ;
linker_flags . AddLinkWith ( libmain , true ) ;
2016-04-21 15:58:45 +03:00
}
if ( App . EnableProfiling ) {
string libprofiler ;
2017-01-24 22:11:38 +03:00
switch ( App . LibProfilerLinkMode ) {
case AssemblyBuildTarget . DynamicLibrary :
2016-04-21 15:58:45 +03:00
libprofiler = Path . Combine ( libdir , "libmono-profiler-log.dylib" ) ;
2017-01-24 17:41:03 +03:00
linker_flags . AddLinkWith ( libprofiler ) ;
2017-03-15 20:34:32 +03:00
if ( App . HasFrameworksDirectory )
2017-01-24 13:10:20 +03:00
AddToBundle ( libprofiler ) ;
2017-01-24 22:11:38 +03:00
break ;
case AssemblyBuildTarget . StaticObject :
2016-04-21 15:58:45 +03:00
libprofiler = Path . Combine ( libdir , "libmono-profiler-log.a" ) ;
2017-01-24 17:41:03 +03:00
linker_flags . AddLinkWith ( libprofiler ) ;
2017-01-24 22:11:38 +03:00
break ;
case AssemblyBuildTarget . Framework : // We don't ship the profiler as a framework, so this should be impossible.
default :
2018-10-04 16:43:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." , App . LibProfilerLinkMode ) ;
2016-04-21 15:58:45 +03:00
}
}
2018-05-05 00:59:31 +03:00
if ( App . UseInterpreter ) {
string libinterp = Path . Combine ( libdir , "libmono-ee-interp.a" ) ;
linker_flags . AddLinkWith ( libinterp ) ;
string libicalltable = Path . Combine ( libdir , "libmono-icall-table.a" ) ;
linker_flags . AddLinkWith ( libicalltable ) ;
string libilgen = Path . Combine ( libdir , "libmono-ilgen.a" ) ;
linker_flags . AddLinkWith ( libilgen ) ;
}
2016-04-21 15:58:45 +03:00
if ( ! string . IsNullOrEmpty ( App . UserGccFlags ) )
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( App . UserGccFlags ) ;
2016-04-21 15:58:45 +03:00
if ( App . DeadStrip )
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-dead_strip" ) ;
2016-04-21 15:58:45 +03:00
if ( App . IsExtension ) {
if ( App . Platform = = ApplePlatform . iOS & & Driver . XcodeVersion . Major < 7 ) {
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-lpkstart" ) ;
2017-06-06 23:32:25 +03:00
linker_flags . AddOtherFlag ( $"-F {StringUtils.Quote (Path.Combine (Driver.GetFrameworkDirectory (App), " System / Library / PrivateFrameworks "))} -framework PlugInKit" ) ;
2016-04-21 15:58:45 +03:00
}
2017-01-24 17:41:03 +03:00
linker_flags . AddOtherFlag ( "-fapplication-extension" ) ;
2016-04-21 15:58:45 +03:00
}
2018-10-15 22:39:29 +03:00
HandleMonoNative ( App , linker_flags ) ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
link_task = new NativeLinkTask
2017-01-25 19:39:48 +03:00
{
Target = this ,
OutputFile = Executable ,
CompilerFlags = linker_flags ,
} ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
link_task . AddDependency ( link_with_task_output ) ;
link_task . AddDependency ( aot_dependencies ) ;
build_tasks . Add ( link_task ) ;
2016-04-21 15:58:45 +03:00
}
2019-02-05 13:37:02 +03:00
public class MonoNativeInfo
{
public bool RequireMonoNative { get ; set ; }
public bool RequireGss { get ; set ; }
public void Load ( string filename )
{
using ( var reader = new StreamReader ( filename ) ) {
string line ;
while ( ( line = reader . ReadLine ( ) ) ! = null ) {
if ( line . Length = = 0 )
continue ;
var eq = line . IndexOf ( '=' ) ;
var typestr = line . Substring ( 0 , eq ) ;
var valstr = line . Substring ( eq + 1 ) ;
bool value = Convert . ToBoolean ( valstr ) ;
switch ( typestr ) {
case "RequireMonoNative" :
RequireMonoNative = value ;
break ;
case "RequireGss" :
RequireGss = value ;
break ;
default :
throw ErrorHelper . CreateError ( 99 , $"Internal error: invalid type string while loading cached Mono.Native info: {typestr}. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new)." ) ;
}
}
}
}
public void Save ( string filename )
{
using ( var writer = new StreamWriter ( filename ) ) {
writer . WriteLine ( "RequireMonoNative={0}" , RequireMonoNative ) ;
writer . WriteLine ( "RequireGss={0}" , RequireGss ) ;
}
}
}
MonoNativeInfo mono_native_info ;
public MonoNativeInfo MonoNative
{
get {
if ( mono_native_info ! = null )
return mono_native_info ;
mono_native_info = new MonoNativeInfo ( ) ;
var cache_location = Path . Combine ( App . Cache . Location , "mono-native-info.txt" ) ;
if ( cached_link ) {
mono_native_info . Load ( cache_location ) ;
} else {
mono_native_info . RequireMonoNative = LinkContext ? . RequireMonoNative ? ? true ;
mono_native_info . RequireGss = LinkContext ? . RequireGss ? ? true ;
mono_native_info . Save ( cache_location ) ;
}
return mono_native_info ;
}
}
2018-11-14 21:20:48 +03:00
void HandleMonoNative ( Application app , CompilerFlags compiler_flags )
2018-10-15 22:39:29 +03:00
{
if ( app . MonoNativeMode = = MonoNativeMode . None )
return ;
2019-02-05 13:37:02 +03:00
if ( ! MonoNative . RequireMonoNative )
2018-11-14 21:20:48 +03:00
return ;
2018-10-15 22:39:29 +03:00
var libnative = app . GetLibNativeName ( ) ;
var libdir = Driver . GetMonoTouchLibDirectory ( app ) ;
2018-11-14 21:20:48 +03:00
Driver . Log ( 3 , "Adding mono-native library {0} for {1}." , libnative , app ) ;
2018-10-15 22:39:29 +03:00
switch ( app . LibMonoNativeLinkMode ) {
case AssemblyBuildTarget . DynamicLibrary :
libnative = Path . Combine ( libdir , libnative + ".dylib" ) ;
compiler_flags . AddLinkWith ( libnative ) ;
break ;
case AssemblyBuildTarget . StaticObject :
libnative = Path . Combine ( libdir , libnative + ".a" ) ;
compiler_flags . AddLinkWith ( libnative ) ;
2018-11-14 21:20:48 +03:00
switch ( app . Platform ) {
case ApplePlatform . iOS :
2019-02-05 13:37:02 +03:00
if ( MonoNative . RequireGss ) {
2018-11-14 21:20:48 +03:00
Driver . Log ( 3 , "Adding GSS framework reference." ) ;
compiler_flags . AddFramework ( "GSS" ) ;
}
break ;
}
2018-10-15 22:39:29 +03:00
break ;
default :
2019-02-01 21:47:55 +03:00
throw ErrorHelper . CreateError ( 100 , "Invalid assembly build target: '{0}'. Please file a bug report with a test case (https://github.com/xamarin/xamarin-macios/issues/new." , app . LibMonoLinkMode ) ;
2018-10-15 22:39:29 +03:00
}
}
2017-12-22 18:28:29 +03:00
public static void AdjustDylibs ( string output )
2016-04-21 15:58:45 +03:00
{
var sb = new StringBuilder ( ) ;
2017-12-22 18:28:29 +03:00
foreach ( var dependency in Xamarin . MachO . GetNativeDependencies ( output ) ) {
2016-04-21 15:58:45 +03:00
if ( ! dependency . StartsWith ( "/System/Library/PrivateFrameworks/" , StringComparison . Ordinal ) )
continue ;
var fixed_dep = dependency . Replace ( "/PrivateFrameworks/" , "/Frameworks/" ) ;
sb . Append ( " -change " ) . Append ( dependency ) . Append ( ' ' ) . Append ( fixed_dep ) ;
}
if ( sb . Length > 0 ) {
2017-12-22 18:28:29 +03:00
var quoted_name = StringUtils . Quote ( output ) ;
2016-04-21 15:58:45 +03:00
sb . Append ( ' ' ) . Append ( quoted_name ) ;
Driver . XcodeRun ( "install_name_tool" , sb . ToString ( ) ) ;
sb . Clear ( ) ;
}
}
public bool CanWeSymlinkTheApplication ( )
{
2016-12-23 20:50:35 +03:00
if ( ! Driver . CanWeSymlinkTheApplication ( App ) )
2016-04-21 15:58:45 +03:00
return false ;
foreach ( var a in Assemblies )
if ( ! a . CanSymLinkForApplication ( ) )
return false ;
return true ;
}
public void Symlink ( )
{
foreach ( var a in Assemblies )
a . Symlink ( ) ;
var targetExecutable = Executable ;
Application . TryDelete ( targetExecutable ) ;
try {
var launcher = new StringBuilder ( ) ;
launcher . Append ( Path . Combine ( Driver . MonoTouchDirectory , "bin" , "simlauncher" ) ) ;
2016-09-30 22:02:17 +03:00
if ( Is32Build )
launcher . Append ( "32" ) ;
else if ( Is64Build )
launcher . Append ( "64" ) ;
2016-04-21 15:58:45 +03:00
launcher . Append ( "-sgen" ) ;
File . Copy ( launcher . ToString ( ) , Executable ) ;
2016-11-03 16:35:46 +03:00
File . SetLastWriteTime ( Executable , DateTime . Now ) ;
2016-04-21 15:58:45 +03:00
} catch ( MonoTouchException ) {
throw ;
} catch ( Exception ex ) {
throw new MonoTouchException ( 1015 , true , ex , "Failed to create the executable '{0}': {1}" , targetExecutable , ex . Message ) ;
}
Symlinked = true ;
2018-11-14 21:20:48 +03:00
if ( App . MonoNativeMode ! = MonoNativeMode . None ) {
var lib_native_target = Path . Combine ( TargetDirectory , "libmono-native.dylib" ) ;
try {
var lib_native_name = App . GetLibNativeName ( ) + ".dylib" ;
var lib_native_path = Path . Combine ( Driver . GetMonoTouchLibDirectory ( App ) , lib_native_name ) ;
2019-02-01 22:10:58 +03:00
Application . UpdateFile ( lib_native_path , lib_native_target ) ;
Driver . Log ( 3 , "Added mono-native library {0} for {1}." , lib_native_name , App . MonoNativeMode ) ;
2018-11-14 21:20:48 +03:00
} catch ( MonoTouchException ) {
throw ;
} catch ( Exception ex ) {
throw new MonoTouchException ( 1015 , true , ex , "Failed to create the Mono.Native library '{0}': {1}" , lib_native_target , ex . Message ) ;
}
}
2016-04-21 15:58:45 +03:00
if ( Driver . Verbosity > 0 )
Console . WriteLine ( "Application ({0}) was built using fast-path for simulator." , string . Join ( ", " , Abis . ToArray ( ) ) ) ;
}
}
}
