2016-04-21 15:57:02 +03:00
/ *
* Copyright 2014 Xamarin Inc . All rights reserved .
*
* Authors :
* Rolf Bjarne Kvinge < rolf @xamarin . com >
*
* /
using System ;
using System.Diagnostics ;
using System.IO ;
using System.Linq ;
using System.Runtime.InteropServices ;
using System.Text ;
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
using System.Threading.Tasks ;
2016-04-21 15:57:02 +03:00
using Xamarin.Utils ;
2016-05-11 13:47:26 +03:00
using XamCore.ObjCRuntime ;
2016-04-21 15:57:02 +03:00
namespace Xamarin.Bundler {
public partial class Driver {
2016-12-23 20:50:35 +03:00
static void AddSharedOptions ( Application app , Mono . Options . OptionSet options )
2016-05-11 13:24:55 +03:00
{
2017-02-14 19:46:26 +03:00
options . Add ( "warnaserror:" , "An optional comma-separated list of warning codes that should be reported as errors (if no warnings are specified all warnings are reported as errors)." , v = >
{
try {
if ( ! string . IsNullOrEmpty ( v ) ) {
foreach ( var code in v . Split ( new char [ ] { ',' } , StringSplitOptions . RemoveEmptyEntries ) )
ErrorHelper . SetWarningLevel ( ErrorHelper . WarningLevel . Error , int . Parse ( code ) ) ;
} else {
ErrorHelper . SetWarningLevel ( ErrorHelper . WarningLevel . Error ) ;
}
} catch ( Exception ex ) {
ErrorHelper . Error ( 26 , ex , "Could not parse the command line argument '{0}': {1}" , "--warnaserror" , ex . Message ) ;
}
} ) ;
options . Add ( "nowarn:" , "An optional comma-separated list of warning codes to ignore (if no warnings are specified all warnings are ignored)." , v = >
{
try {
if ( ! string . IsNullOrEmpty ( v ) ) {
foreach ( var code in v . Split ( new char [ ] { ',' } , StringSplitOptions . RemoveEmptyEntries ) )
ErrorHelper . SetWarningLevel ( ErrorHelper . WarningLevel . Disable , int . Parse ( code ) ) ;
} else {
ErrorHelper . SetWarningLevel ( ErrorHelper . WarningLevel . Disable ) ;
}
} catch ( Exception ex ) {
ErrorHelper . Error ( 26 , ex , "Could not parse the command line argument '{0}': {1}" , "--nowarn" , ex . Message ) ;
}
} ) ;
2016-12-23 20:50:35 +03:00
options . Add ( "coop:" , "If the GC should run in cooperative mode." , v = > { app . EnableCoopGC = ParseBool ( v , "coop" ) ; } , hidden : true ) ;
2017-03-17 01:14:42 +03:00
options . Add ( "sgen-conc" , "Enable the *experimental* concurrent garbage collector." , v = > { app . EnableSGenConc = true ; } ) ;
2016-12-22 22:11:50 +03:00
options . Add ( "marshal-objectivec-exceptions:" , "Specify how Objective-C exceptions should be marshalled. Valid values: default, unwindmanagedcode, throwmanagedexception, abort and disable. The default depends on the target platform (on watchOS the default is 'throwmanagedexception', while on all other platforms it's 'disable')." , v = > {
2016-05-11 13:47:26 +03:00
switch ( v ) {
case "default" :
2016-12-23 20:50:35 +03:00
app . MarshalObjectiveCExceptions = MarshalObjectiveCExceptionMode . Default ;
2016-05-11 13:47:26 +03:00
break ;
case "unwindmanaged" :
case "unwindmanagedcode" :
2016-12-23 20:50:35 +03:00
app . MarshalObjectiveCExceptions = MarshalObjectiveCExceptionMode . UnwindManagedCode ;
2016-05-11 13:47:26 +03:00
break ;
case "throwmanaged" :
case "throwmanagedexception" :
2016-12-23 20:50:35 +03:00
app . MarshalObjectiveCExceptions = MarshalObjectiveCExceptionMode . ThrowManagedException ;
2016-05-11 13:47:26 +03:00
break ;
case "abort" :
2016-12-23 20:50:35 +03:00
app . MarshalObjectiveCExceptions = MarshalObjectiveCExceptionMode . Abort ;
2016-05-11 13:47:26 +03:00
break ;
case "disable" :
2016-12-23 20:50:35 +03:00
app . MarshalObjectiveCExceptions = MarshalObjectiveCExceptionMode . Disable ;
2016-05-11 13:47:26 +03:00
break ;
default :
2016-12-22 22:11:50 +03:00
throw ErrorHelper . CreateError ( 26 , "Could not parse the command line argument '{0}': {1}" , "--marshal-objective-exceptions" , $"Invalid value: {v}. Valid values are: default, unwindmanagedcode, throwmanagedexception, abort and disable." ) ;
2016-05-11 13:47:26 +03:00
}
} ) ;
2016-12-22 22:11:50 +03:00
options . Add ( "marshal-managed-exceptions:" , "Specify how managed exceptions should be marshalled. Valid values: default, unwindnativecode, throwobjectivecexception, abort and disable. The default depends on the target platform (on watchOS the default is 'throwobjectivecexception', while on all other platform it's 'disable')." , v = > {
2016-05-11 13:47:26 +03:00
switch ( v ) {
case "default" :
2016-12-23 20:50:35 +03:00
app . MarshalManagedExceptions = MarshalManagedExceptionMode . Default ;
2016-05-11 13:47:26 +03:00
break ;
case "unwindnative" :
case "unwindnativecode" :
2016-12-23 20:50:35 +03:00
app . MarshalManagedExceptions = MarshalManagedExceptionMode . UnwindNativeCode ;
2016-05-11 13:47:26 +03:00
break ;
case "throwobjectivec" :
case "throwobjectivecexception" :
2016-12-23 20:50:35 +03:00
app . MarshalManagedExceptions = MarshalManagedExceptionMode . ThrowObjectiveCException ;
2016-05-11 13:47:26 +03:00
break ;
case "abort" :
2016-12-23 20:50:35 +03:00
app . MarshalManagedExceptions = MarshalManagedExceptionMode . Abort ;
2016-05-11 13:47:26 +03:00
break ;
case "disable" :
2016-12-23 20:50:35 +03:00
app . MarshalManagedExceptions = MarshalManagedExceptionMode . Disable ;
2016-05-11 13:47:26 +03:00
break ;
default :
2016-12-22 22:11:50 +03:00
throw ErrorHelper . CreateError ( 26 , "Could not parse the command line argument '{0}': {1}" , "--marshal-managed-exceptions" , $"Invalid value: {v}. Valid values are: default, unwindnativecode, throwobjectivecexception, abort and disable." ) ;
2016-05-11 13:47:26 +03:00
}
} ) ;
2017-01-11 23:10:39 +03:00
options . Add ( "j|jobs=" , "The level of concurrency. Default is the number of processors." , v = > {
Jobs = int . Parse ( v ) ;
} ) ;
2017-04-19 14:17:05 +03:00
options . Add ( "embeddinator" , "Enables Embeddinator targetting mode." , v = > {
app . Embeddinator = true ;
} , true ) ;
[mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417. (#2162)
* [mtouch] Improve how we make sure native symbols aren't stripped away. Fixes #51710 and #54417.
* Refactor required symbol collection to store more information about each
symbol (field, function, Objective-C class), and in general make the code
more straight forward.
* Implement support for generating source code that references these symbols,
and do this whenever we can't ask the native linker to keep these symbols
(when using bitcode). Additionally make it possible to do this manually, so
that the source code can be generated for non-bitcode platforms too (which
is useful if the number of symbols is enormous, in which case we might
surpass the maximum command-line length).
* Also make it possible to completely ignore native symbols, or ignore them on
a per-symbol basis. This provides a fallback for users if we get something
right and we try to preserve something that shouldn't be preserved (for
instance if it doesn't exist), and the user ends up with unfixable linker
errors.
* Don't collect Objective-C classes unless they're in an assembly with
LinkWith attributes. We don't need to preserve Objective-C classes in any
other circumstances.
* Implement everything for both Xamarin.iOS and Xamarin.Mac, and share the
code between them.
* Remove previous workaround for bug #51710, since it's no longer needed.
* Add tests.
https://bugzilla.xamarin.com/show_bug.cgi?id=54417
https://bugzilla.xamarin.com/show_bug.cgi?id=51710
* [mtouch] Make sure to only keep symbols from the current app when code sharing.
This fixes a build problem with the interdependent-binding-projects test when
testing in Today Extension mode.
2017-06-02 19:29:19 +03:00
options . Add ( "dynamic-symbol-mode:" , "Specify how dynamic symbols are treated so that they're not linked away by the native linker. Valid values: linker (pass \"-u symbol\" to the native linker), code (generate native code that uses the dynamic symbol), ignore (do nothing and hope for the best). The default is 'code' when using bitcode, and 'linker' otherwise." , ( v ) = > {
switch ( v . ToLowerInvariant ( ) ) {
case "default" :
app . SymbolMode = SymbolMode . Default ;
break ;
case "linker" :
app . SymbolMode = SymbolMode . Linker ;
break ;
case "code" :
app . SymbolMode = SymbolMode . Code ;
break ;
case "ignore" :
app . SymbolMode = SymbolMode . Ignore ;
break ;
default :
throw ErrorHelper . CreateError ( 26 , "Could not parse the command line argument '{0}': {1}" , "--dynamic-symbol-mode" , $"Invalid value: {v}. Valid values are: default, linker, code and ignore." ) ;
}
} ) ;
options . Add ( "ignore-dynamic-symbol:" , "Specify that Xamarin.iOS/Xamarin.Mac should not try to prevent the linker from removing the specified symbol." , ( v ) = > {
app . IgnoredSymbols . Add ( v ) ;
} ) ;
2017-01-11 23:10:39 +03:00
}
static int Jobs ;
public static int Concurrency {
get {
return Jobs = = 0 ? Environment . ProcessorCount : Jobs ;
}
2016-05-11 13:24:55 +03:00
}
[mtouch/mmp] Print assembly references in verbose mode. (#2139)
* [mtouch/mmp] Print assembly references in verbose mode.
Sample output:
Loaded assembly 'unifiedtestapp, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null' from /Users/rolf/Projects/TestApp/bin/iPhoneSimulator/Debug/unifiedtestapp.exe
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Xamarin.iOS, Version=0.0.0.0, Culture=neutral, PublicKeyToken=84e04ff9cfb79065
Loaded assembly 'mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/mscorlib.dll
Loaded assembly 'Xamarin.iOS, Version=0.0.0.0, Culture=neutral, PublicKeyToken=84e04ff9cfb79065' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/../../64bits/Xamarin.iOS.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System.Core, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756
References: System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756
References: System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'Mono.Security, Version=2.0.5.0, Culture=neutral, PublicKeyToken=0738eb9f132ed756' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/Mono.Security.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System.Xml, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.Xml.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
Loaded assembly 'System.Core, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e' from /work/maccore/master/xamarin-macios/_ios-build/Library/Frameworks/Xamarin.iOS.framework/Versions/git/lib/mono/Xamarin.iOS/System.Core.dll
References: mscorlib, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
References: System, Version=2.0.5.0, Culture=neutral, PublicKeyToken=7cec85d7bea7798e
I believe this will make it easier to diagnose cases where something references a desktop assembly.
* [mtouch/mmp] Share more code between mmp and mtouch.
2017-05-29 17:15:22 +03:00
public static int Verbosity {
get { return verbose ; }
}
2016-04-21 15:57:02 +03:00
#if MONOMAC
#pragma warning disable 0414
static string userTargetFramework = TargetFramework . Default . ToString ( ) ;
#pragma warning restore 0414
#endif
static TargetFramework ? targetFramework ;
public static bool HasTargetFramework {
get { return targetFramework . HasValue ; }
}
public static TargetFramework TargetFramework {
get { return targetFramework . Value ; }
set { targetFramework = value ; }
}
static void SetTargetFramework ( string fx )
{
#if MONOMAC
userTargetFramework = fx ;
#endif
switch ( fx . Trim ( ) . ToLowerInvariant ( ) ) {
#if MONOMAC
case "xammac" :
case "mobile" :
case "xamarin.mac" :
targetFramework = TargetFramework . Xamarin_Mac_2_0 ;
break ;
#endif
default :
TargetFramework parsedFramework ;
if ( ! Xamarin . Utils . TargetFramework . TryParse ( fx , out parsedFramework ) )
throw ErrorHelper . CreateError ( 68 , "Invalid value for target framework: {0}." , fx ) ;
#if MONOMAC
if ( parsedFramework = = TargetFramework . Net_3_0 | | parsedFramework = = TargetFramework . Net_3_5 )
parsedFramework = TargetFramework . Net_2_0 ;
#endif
targetFramework = parsedFramework ;
break ;
}
#if MTOUCH
if ( Array . IndexOf ( TargetFramework . ValidFrameworks , targetFramework . Value ) = = - 1 )
throw ErrorHelper . CreateError ( 70 , "Invalid target framework: {0}. Valid target frameworks are: {1}." , targetFramework . Value , string . Join ( " " , TargetFramework . ValidFrameworks . Select ( ( v ) = > v . ToString ( ) ) . ToArray ( ) ) ) ;
#endif
}
public static int RunCommand ( string path , string args , string [ ] env = null , StringBuilder output = null , bool suppressPrintOnErrors = false )
{
Exception stdin_exc = null ;
var info = new ProcessStartInfo ( path , args ) ;
info . UseShellExecute = false ;
info . RedirectStandardInput = false ;
info . RedirectStandardOutput = true ;
info . RedirectStandardError = true ;
System . Threading . ManualResetEvent stdout_completed = new System . Threading . ManualResetEvent ( false ) ;
System . Threading . ManualResetEvent stderr_completed = new System . Threading . ManualResetEvent ( false ) ;
if ( output = = null )
output = new StringBuilder ( ) ;
if ( env ! = null ) {
if ( env . Length % 2 ! = 0 )
throw new Exception ( "You passed an environment key without a value" ) ;
for ( int i = 0 ; i < env . Length ; i + = 2 )
info . EnvironmentVariables [ env [ i ] ] = env [ i + 1 ] ;
}
if ( verbose > 0 )
Console . WriteLine ( "{0} {1}" , path , args ) ;
using ( var p = Process . Start ( info ) ) {
p . OutputDataReceived + = ( s , e ) = > {
if ( e . Data ! = null ) {
lock ( output )
output . AppendLine ( e . Data ) ;
} else {
stdout_completed . Set ( ) ;
}
} ;
p . ErrorDataReceived + = ( s , e ) = > {
if ( e . Data ! = null ) {
lock ( output )
output . AppendLine ( e . Data ) ;
} else {
stderr_completed . Set ( ) ;
}
} ;
p . BeginOutputReadLine ( ) ;
p . BeginErrorReadLine ( ) ;
p . WaitForExit ( ) ;
stderr_completed . WaitOne ( TimeSpan . FromSeconds ( 1 ) ) ;
stdout_completed . WaitOne ( TimeSpan . FromSeconds ( 1 ) ) ;
if ( p . ExitCode ! = 0 ) {
// note: this repeat the failing command line. However we can't avoid this since we're often
// running commands in parallel (so the last one printed might not be the one failing)
if ( ! suppressPrintOnErrors )
Console . Error . WriteLine ( "Process exited with code {0}, command:\n{1} {2}{3}" , p . ExitCode , path , args , output . Length > 0 ? "\n" + output . ToString ( ) : string . Empty ) ;
return p . ExitCode ;
} else if ( verbose > 0 & & output . Length > 0 & & ! suppressPrintOnErrors ) {
Console . WriteLine ( output . ToString ( ) ) ;
}
if ( stdin_exc ! = null )
throw stdin_exc ;
}
return 0 ;
}
[mtouch] Rework how tasks are built.
The previous build system kept a forward-pointing single linked list of tasks
to execute: task X had a list of subsequent tasks to execute. If task X was
up-to-date, it was not created (and the next tasks were directly added to the
list of tasks to execute).
In this world it became complicated to merge output from tasks (for instance
if the output of task X and task Y should be a consumed by a single task
producing a single output, since the corresponding task would end up in both
X's and Y's list of subsequent tasks).
Example: creating a single framework from the aot-compiled output of multiple
assemblies.
So I've reversed the logic: now we keep track of the final output, and then
each task has a list of dependencies that must be built.
This makes it trivial to create merging tasks (for the previous example, there
could for instance be a CreateFrameworkTask, where its dependencies would be
all the corresponding AotTasks).
We also always create every task, and then each task decides when its executed
whether it should do anything or not. This makes it unnecessary to 'forward-
delete' files when creating tasks (say you have three tasks, A, B, C; B
depends on A, and C depends on B; if A's output isn't up-to-date, it has to
delete its own output if it exists, otherwise B would not detect that it would
have to re-execute, because at task *creation* time, B's input hadn't
changed).
Additionally make it based on async/await, since much of the work happens in
externel processes (and we don't need to spin up additional threads just to
run external processes). This makes us have less code run on background
threads, which makes any issues with thread-safety less likely.
2017-01-26 12:56:55 +03:00
public static Task < int > RunCommandAsync ( string path , string args , string [ ] env = null , StringBuilder output = null , bool suppressPrintOnErrors = false )
{
return Task . Run ( ( ) = > RunCommand ( path , args , env , output , suppressPrintOnErrors ) ) ;
}
2016-04-21 15:57:02 +03:00
#if ! MMP_TEST
static void FileMove ( string source , string target )
{
Application . TryDelete ( target ) ;
File . Move ( source , target ) ;
}
2016-05-11 14:00:20 +03:00
static void MoveIfDifferent ( string path , string tmp )
{
// Don't read the entire file into memory, it can be quite big in certain cases.
bool move = false ;
using ( var fs1 = new FileStream ( path , FileMode . Open , FileAccess . Read ) ) {
using ( var fs2 = new FileStream ( tmp , FileMode . Open , FileAccess . Read ) ) {
if ( fs1 . Length ! = fs2 . Length ) {
Log ( 3 , "New file '{0}' has different length, writing new file." , path ) ;
move = true ;
} else {
move = ! Cache . CompareStreams ( fs1 , fs2 ) ;
}
}
}
if ( move ) {
FileMove ( tmp , path ) ;
} else {
Log ( 3 , "Target {0} is up-to-date." , path ) ;
}
}
2016-04-21 15:57:02 +03:00
public static void WriteIfDifferent ( string path , string contents )
{
var tmp = path + ".tmp" ;
try {
if ( ! File . Exists ( path ) ) {
2016-11-21 15:30:46 +03:00
Directory . CreateDirectory ( Path . GetDirectoryName ( path ) ) ;
2016-04-21 15:57:02 +03:00
File . WriteAllText ( path , contents ) ;
Log ( 3 , "File '{0}' does not exist, creating it." , path ) ;
return ;
}
File . WriteAllText ( tmp , contents ) ;
2016-05-11 14:00:20 +03:00
MoveIfDifferent ( path , tmp ) ;
} catch ( Exception e ) {
File . WriteAllText ( path , contents ) ;
ErrorHelper . Warning ( 1014 , e , "Failed to re-use cached version of '{0}': {1}." , path , e . Message ) ;
} finally {
Application . TryDelete ( tmp ) ;
}
}
2016-04-21 15:57:02 +03:00
2016-05-11 14:00:20 +03:00
public static void WriteIfDifferent ( string path , byte [ ] contents )
{
var tmp = path + ".tmp" ;
2016-04-21 15:57:02 +03:00
2016-05-11 14:00:20 +03:00
try {
if ( ! File . Exists ( path ) ) {
File . WriteAllBytes ( path , contents ) ;
Log ( 3 , "File '{0}' does not exist, creating it." , path ) ;
return ;
2016-04-21 15:57:02 +03:00
}
2016-05-11 14:00:20 +03:00
File . WriteAllBytes ( tmp , contents ) ;
MoveIfDifferent ( path , tmp ) ;
2016-04-21 15:57:02 +03:00
} catch ( Exception e ) {
2016-05-11 14:00:20 +03:00
File . WriteAllBytes ( path , contents ) ;
2016-04-21 15:57:02 +03:00
ErrorHelper . Warning ( 1014 , e , "Failed to re-use cached version of '{0}': {1}." , path , e . Message ) ;
} finally {
Application . TryDelete ( tmp ) ;
}
}
#endif
internal static string GetFullPath ( )
{
return System . Reflection . Assembly . GetExecutingAssembly ( ) . Location ;
}
static Version xcode_version ;
public static Version XcodeVersion {
get {
return xcode_version ;
}
}
}
}