Update module-deps meta files with the latest module data.

Remove the hardcoded module2dir mapping and use the results from echo-requires-recursive instead.
Add README which briefly explains how module-deps is supposed to work.
Add XSLT hacks to modules.mk.footer.
Add cygdrive hacks to modules.mk.stub.
Force string & embedstring to be built after xpcom since they now live in the xpcom hierarchy.

config/module2dir.pl

@@ -13,190 +13,47 @@ use Getopt::Long;
 Getopt::Long::Configure("bundling_override");
 Getopt::Long::Configure("auto_abbrev");
 
+# Globals
+my $list_only_mode = 0;
+my $opt_list_only;
+my $mapfile = "";
+my %map;
 
 sub PrintUsage {
   die <<END_USAGE
   Prints out directories needed for a given list of components.
-  usage: module2dir.pl [--list-only] <component-name1> <component-name2> ...
+  usage: module2dir.pl [--list-only] [--mapfile mapfile] <component-name1> <component-name2> ...
 END_USAGE
 }
 
-my %map = (
+sub parse_map_file($) {
-  'access-proxy',       'extensions/access-builtin/accessproxy',
+    my ($mapfile) = @_;
-  'accessibility',      'accessible',
+    my (%mod_map, $tmp, $dir, $mod, @mod_list);
-  'addrbook',           'mailnews/addrbook',
-  'appcomps',           'xpfe/components',
-  'appshell',           'xpfe/appshell',
-  'appstartup',         'embedding/components/appstartup',
-  'autoconfig',         'extensions/pref/autoconfig',
-  'boehm',              'gc/boehm',
-  'browser',            'xpfe/browser',
-  'calendar',           'mailnews/mime/cthandlers/calendar',
-  'caps',               'caps',
-  'chardet',            'intl/chardet',
-  'chardetc',           'intl/chardet/src/classic',
-  'chrome',             'rdf/chrome',
-  'commandhandler',     'embedding/components/commandhandler',
-  'composer',           'editor/composer',
-  'content',            'content',
-  'cookie',             'extensions/cookie',
-  'ctl',                'extensions/ctl',
-  'dbm',                'dbm',
-  'docshell',           'docshell',
-  'dom',                'dom',
-  'downloadmanager',    'xpfe/components/download-manager',
-  'editor',             'editor',
-  'embed_base',         'embedding/base',
-  'embed_lite',         'embedding/lite',
-  'embedcomponents',    'embedding/components/appstartup embedding/components/windowwatcher embedding/components/printingui embedding/components/build',
-  'expat',              'expat',
-  'exthandler',         'uriloader/exthandler',
-  'filepicker',         'xpfe/components/filepicker',
-  'find',               'xpfe/components/find embedding/components/find',
-  'gfx',                'gfx',
-  'gfx2',               'gfx2',
-  'gtkembedmoz',        'embedding/browser/gtk/src embedding/browser/gtk/tests',
-  'gtkxtbin',           'widget/src/gtkxtbin',
-  'helperAppDlg',       'embedding/components/ui/helperAppDlg',
-  'htmlparser',         'htmlparser',
-  'iiextras',           'extensions/interfaceinfo',
-  'imgbmp',             'modules/libpr0n/decoders/bmp',
-  'imggif',             'modules/libpr0n/decoders/gif',
-  'imgicon',            'modules/libpr0n/decoders/icon',
-  'imgjpeg',            'modules/libpr0n/decoders/jpeg',
-  'imglib2',            'modules/libpr0n/public  modules/libpr0n/src',
-  'imgpng',             'modules/libpr0n/decoders/png',
-  'imgxbm',             'modules/libpr0n/decoders/xbm',
-  'impComm4xMail',      'mailnews/import/comm4x',
-  'impEudra',           'mailnews/import/eudora',
-  'impOutlk',           'mailnews/import/outlook',
-  'impText',            'mailnews/import/text',
-  'import',             'mailnews/import',
-  'importOE',           'mailnews/import/oexpress',
-  'inspector',          'extensions/inspector',
-  'intl',               'intl',
-  'intlcmpt',           'intl/compatibility',
-  'jar',                'modules/libjar',
-  'java',               'sun-java/stubs',
-  'jpeg',               'jpeg',
-  'jprof',              'tools/jprof',
-  'js',                 'js/src/fdlibm js/src',
-  'jsconsole',          'embedding/components/jsconsole',
-  'jsdebug',            'js/jsd',
-  'jsd_refl',           'js/jsd/jsdb',
-  'jsloader',           'js/src/xpconnect/loader',
-  'layout',             'layout',
-  'libreg',             'modules/libreg',
-  'liveconnect',        'js/src/liveconnect',
-  'locale',             'intl/locale',
-  'lwbrk',              'intl/lwbrk',
-  'mailnews',           'mailnews',
-  'MapiProxy',          'mailnews/mapi/mapihook/build',
-  'mfcEmbed',           'embedding/tests/winEmbed',
-  'mkdepend',           'config/mkdepend',
-  'mime',               'mailnews/mime',
-  'mimeemitter',        'mailnews/mime/emitters',
-  'mimetype',           'netwerk/mime',
-  'mng',                'modules/libimg/mng',
-  'mork',               'db/mork db/mdb',
-  'mozcomps',           'xpfe/components',
-  'mozldap',            'directory/xpcom/base',
-  'mozMapi32',          'mailnews/mapi',
-  'mozpango',           'intl/ctl/src/pangoLite',
-  'mozpango-thaix',     'intl/ctl/src/thaiShaper',
-  'mozpango-dvngx',     'intl/ctl/src/hindiShaper',
-  'mozucth',            'xpfe/components/ucth',
-  'mozxfer',            'xpfe/components/xfer',
-  'msgbase',            'mailnews/base',
-  'msgbaseutil',        'mailnews/base/util',
-  'msgcompose',         'mailnews/compose',
-  'msgdb',              'mailnews/db/msgdb',
-  'msgimap',            'mailnews/imap',
-  'msglocal',           'mailnews/local',
-  'msgMapi',            'mailnews/mapi/mapihook',
-  'msgmdn',             'mailnews/extensions/mdn',
-  'msgnews',            'mailnews/news',
-  'msgsmime',           'mailnews/extensions/smime',
-  'necko',              'netwerk/base netwerk/protocol/about netwerk/protocol/data netwerk/protocol/file netwerk/protocol/http netwerk/protocol/jar netwerk/protocol/keyword netwerk/protocol/res netwerk/dns netwerk/socket netwerk/streamconv netwerk/cookie netwerk/build',
-  'necko2',             'netwerk/protocol/ftp netwerk/protocol/gopher netwerk/protocol/viewsource netwerk/build2',
-  'nkcache',            'netwerk/cache',
-  'npevents',           'modules/plugin/samples/testevents',
-  'npsimple',           'modules/plugin/samples/simple',
-  'oji',                'modules/oji',
-  'p3p',                'extensions/p3p',
-  'phembedmoz',         'embedding/browser/photon',
-  'pics',               'extensions/pics',
-  'pipboot',            'security/manager/boot',
-  'pippki',             'security/manager/pki',
-  'pipnss',             'security/manager/ssl',
-  'plugin',             'modules/plugin',
-  'png',                'modules/libimg/png',
-  'pref',               'modules/libpref',
-  'prefmigr',           'profile/pref-migrator',
-  'profile',            'profile',
-  'progressDlg',        'embedding/components/ui/progressDlg',
-  'pyxpcom',            'extensions/python/xpcom',
-  'rdf',                'rdf',
-  'rdfutil',            'rdf/util',
-  'rdfldapds',          'directory/xpcom/tests',
-  'SanePlugin',         'modules/plugin/samples/SanePlugin',
-  'shistory',           'xpfe/components/shistory',
-  'sidebar',            'xpfe/components/sidebar',
-  'smime',              'mailnews/mime/cthandlers/smimestub',
-  'string',             'string',
-  'svg_doc',            'content/svg/document',
-  'test_necko',         'netwerk/test',
-  'TestStreamConv',     'netwerk/streamconv/test',
-  'timebombgen',        'xpfe/components/timebomb/tools',
-  'transformiix',       'extensions/transformiix',
-  'txmgr',              'editor/txmgr',
-  'txtsvc',             'editor/txtsvc',
-  'ucgendat',           'intl/unicharutil/tools/',
-  'uconv',              'intl/uconv',
-  'ucvcn',              'intl/uconv/ucvcn',
-  'ucvibm',             'intl/uconv/ucvibm',
-  'ucvja',              'intl/uconv/ucvja',
-  'ucvko',              'intl/uconv/ucvko',
-  'ucvlatin',           'intl/uconv/ucvlatin',
-  'ucvmath',            'intl/uconv/ucvmath',
-  'ucvtw',              'intl/uconv/ucvtw',
-  'ucvtw2',             'intl/uconv/ucvtw2',
-  'unicharutil',        'intl/unicharutil',
-  'universalchardet',   'extensions/universalchardet',
-  'uriloader',          'uriloader',
-  'util',               'modules/libutil',
-  'vcard',              'mailnews/mime/cthandlers/vcard',
-  'view',               'view',
-  'wallet',             'extensions/wallet',
-  'walletviewers',      'extensions/wallet/build extensions/wallet/cookieviewer extensions/wallet/editor extensions/wallet/signonviewer extensions/wallet/walletpreview',
-  'webbrwsr',           'embedding/browser/webBrowser embedding/browser/build',
-  'webbrowserpersist',  'embedding/components/webbrowserpersist',
-  'webshell',           'webshell',
-  'webshell_tests',     'webshell/tests',
-  'widget',             'widget',
-  'windowwatcher',      'embedding/components/windowwatcher',
-  'wsproxytest',        'extensions/xmlextras/proxy/src/tests',
-  'xlibrgb',            'gfx/src/xlibrgb',
-  'xml-rpc',            'extensions/xml-rpc',
-  'xmlextras',          'extensions/xmlextras',
-  'xmlterm',            'extensions/xmlterm',
-  'xpcom',              'xpcom',
-  'xpcomsample',        'xpcom/sample',
-  'xpconnect',          'js/src/xpconnect',
-  'xpcshell',           'js/src/xpconnect/shell',
-  'xpctools',           'js/src/xpconnect/tools',
-  'xpconnect_tests',    'js/src/xpconnect/tests',
-  'xpctest',            'js/src/xpconnect/tests/components',
-  'xpinstall',          'xpinstall',
-  'xpistub',            'xpinstall/stub',
-  'xpnet',              'xpinstall/wizard/libxpnet',
-  'xremoteservice',     'xpfe/components/xremote',
-  'xul',                'content/xul/content',
-  'xuldoc',             'content/xul/document',
-  'xultmpl',            'content/xul/templates',
-  'zlib',               'modules/zlib',
-);
 
+    undef %mod_map;
+    open (MAPFILE, "$mapfile") || die ("$mapfile: $!\n");
+    while ($tmp=<MAPFILE>) {
+	chomp ($tmp);
+	($dir, $mod, @mod_list) = split(/:/, $tmp, 3);
+	$mod =~ s/[\s]*(\S+)[\s]*/$1/;
+	$mod_map{$mod} .= "$dir ";
+    }
+    close(MAPFILE);
+    foreach $mod (sort(keys %mod_map)) {
+	my (@dirlist, @trimlist, $found, $tdir);
+	@dirlist = split(/\s+/, $mod_map{$mod});
+	$mod_map{$mod} = "";
+	foreach $dir (@dirlist) {
+	    $found = 0; 
+	    foreach $tdir (@trimlist) {
+		$found++, last if ($dir =~ m/^$tdir\// || $dir eq $tdir);
+	    }
+	    push @trimlist, $dir if (!$found);
+        }
+	$map{$mod} = join(" ", @trimlist);
+	#print "$mod: $map{$mod}\n";
+    }
+}
 
 sub dir_for_required_component {
   my ($component) = @_;
@@ -214,21 +71,22 @@ sub dir_for_required_component {
   return $rv;
 }
 
-my $list_only_mode = 0;
-my $opt_list_only;
 {
 
   # Add stdin to the commandline.  This makes commandline-only mode hang,
   # call it a bug.  Not sure how to get around this.
   push (@ARGV, split(' ',<STDIN>));
 
-  PrintUsage() if !GetOptions('list-only' => \$opt_list_only);
+  PrintUsage() if !GetOptions('list-only' => \$opt_list_only,
+			      'mapfile=s' => \$mapfile);
 
   # Pick up arguments, if any.
   if($opt_list_only) {
   	$list_only_mode = 1;
   }
 
+  &parse_map_file($mapfile);
+
   my $arg;
   my $dir;
   while ($arg = shift @ARGV) {

config/rules.mk

@@ -1664,7 +1664,11 @@ echo-requires:
 	@echo $(REQUIRES)
 
 echo-requires-recursive::
+ifdef _REPORT_ALL_DIRS
+	@echo $(subst $(topsrcdir)/,,$(srcdir)): $(MODULE): $(REQUIRES)
+else
 	@$(if $(REQUIRES),echo $(subst $(topsrcdir)/,,$(srcdir)): $(MODULE): $(REQUIRES))
+endif
 	+$(LOOP_OVER_DIRS)
 
 echo-depth-path:

tools/module-deps/.cvsignore

@@ -0,0 +1 @@
+meta.dot

tools/module-deps/README

@@ -0,0 +1,50 @@
+Here's a brief "how this works" doc to refresh my memory when I look at this in another 8 months.
+
+The files:
+
+all.dot - A dependency list of all of the modules in the current tree.
+	- Generated by running module-graph.pl on an existing build tree. 
+	- Only contains the dependencies for the platform it was run on
+	  (Bug 185188)
+
+allmakefiles.stub -  This is the stub portion of allmakefiles.sh which is used
+	when running bootstrap.pl so that only specific Makefiles are generated
+
+bootstrap.pl - This script was originally designed to use the results from
+	module-graph.pl to create a cvs checkout list & build order and to
+	build the tree using those lists.   It has been modified to be able to
+	update the standard allmakefiles.sh & modules.mk files to track module
+	changes as well.
+
+extra.dot - This file contains a mapping of additional module dependencies 
+	which could not be found by module-graph.pl.
+
+force_order.txt - This file contains hard ordering rules for use with 
+	module-graph.pl's --force-order command line option.
+
+meta.dot - This generated file is the combination of all.dot & extra.dot and
+	is used when creating allmakefiles.sh & modules.mk.
+
+module2dir.map - This file contains the mapping of the modules and their
+	dependencies based upon the directory.  This file is created by
+	running 'make -s echo-requires-recursively _REPORT_ALL_DIRS=1 > file'
+	on an existing tree.
+
+module-graph.pl - Outputs a Graphviz-compatible graph description file.
+	It can output a dependency graph based upon an existing description
+	file or it can generate a new one based upon the values of MODULE &
+	REQUIRES in Makefiles in an existing tree.
+
+modules.mk.footer - Footer for generated modules.mk
+
+modules.mk.stub - Header for generated mdoules.mk
+
+
+How to regenerate allmakefiles.sh & modules.mk when modules change:
+
+cd $MOZ_OBJDIR
+perl ../mozilla/tools/module-deps/module-graph.pl > ../mozilla/tools/module-deps/all.dot
+make -s echo-requires-recursive _REPORT_ALL_DIRS=1 > ../mozilla/tools/module-deps/module2dir.map
+cd ../mozilla
+perl tools/module-deps/bootstrap.pl --topsrcdir `pwd` --module-file-only
+

tools/module-deps/all.dot

@@ -56,6 +56,7 @@ digraph G {
         ipcd_test [style=filled];
         jar [style=filled];
         java [style=filled];
+        jpeg [style=filled];
         js [style=filled];
         jsconsole [style=filled];
         jsdebug [style=filled];
@@ -72,6 +73,7 @@ digraph G {
         mailview [style=filled];
         mime [style=filled];
         mimeemitter [style=filled];
+        mimemitter [style=filled];
         mimetype [style=filled];
         mork [style=filled];
         mozldap [style=filled];
@@ -94,6 +96,7 @@ digraph G {
         oji [style=filled];
         p3p [style=filled];
         plugin [style=filled];
+        png [style=filled];
         pref [style=filled];
         prefetch [style=filled];
         prefmigr [style=filled];
@@ -167,6 +170,7 @@ digraph G {
     string -> xpcom;
     embedstring -> xpcom;
     mork -> xpcom;
+    png -> zlib;
     icalss -> ical;
     profilesharingsetup -> string;
     profilesharingsetup -> xpcom;
@@ -184,9 +188,6 @@ digraph G {
     ucvtw -> xpcom;
     ucvtw2 -> uconv;
     ucvtw2 -> xpcom;
-    imgpng -> imglib2;
-    imgpng -> gfx;
-    imgpng -> xpcom;
     xpcshell -> js;
     xpcshell -> xpconnect;
     xpcshell -> xpcom;
@@ -229,9 +230,6 @@ digraph G {
     ucvko -> uconv;
     ucvko -> string;
     ucvko -> xpcom;
-    jar -> caps;
-    jar -> string;
-    jar -> xpcom;
     TestStreamConv -> string;
     TestStreamConv -> necko;
     TestStreamConv -> xpcom;
@@ -271,10 +269,6 @@ digraph G {
     xpcom_compat_c -> xpcom_obsolete;
     xpcom_compat_c -> string;
     xpcom_compat_c -> xpcom;
-    imgjpeg -> imglib2;
-    imgjpeg -> gfx;
-    imgjpeg -> string;
-    imgjpeg -> xpcom;
     nkdatetime -> necko;
     nkdatetime -> string;
     nkdatetime -> mimetype;
@@ -303,6 +297,15 @@ digraph G {
     xpconnect -> caps;
     xpconnect -> string;
     xpconnect -> xpcom;
+    jar -> caps;
+    jar -> zlib;
+    jar -> string;
+    jar -> xpcom;
+    imgpng -> png;
+    imgpng -> imglib2;
+    imgpng -> gfx;
+    imgpng -> zlib;
+    imgpng -> xpcom;
     TestXPC -> xpconnect_tests;
     TestXPC -> js;
     TestXPC -> string;
@@ -318,16 +321,16 @@ digraph G {
     xpctest -> string;
     xpctest -> xpconnect;
     xpctest -> xpcom;
-    imglib2 -> gfx;
-    imglib2 -> nkcache;
-    imglib2 -> necko;
-    imglib2 -> string;
-    imglib2 -> xpcom;
     wsproxytest -> dom;
     wsproxytest -> websrvcs;
     wsproxytest -> xpconnect;
     wsproxytest -> string;
     wsproxytest -> xpcom;
+    imgjpeg -> imglib2;
+    imgjpeg -> gfx;
+    imgjpeg -> jpeg;
+    imgjpeg -> string;
+    imgjpeg -> xpcom;
     necko2 -> necko;
     necko2 -> string;
     necko2 -> xpcom;
@@ -408,6 +411,14 @@ digraph G {
     uconv -> chardet;
     uconv -> locale;
     uconv -> unicharutil;
+    imglib2 -> gfx;
+    imglib2 -> nkcache;
+    imglib2 -> necko;
+    imglib2 -> string;
+    imglib2 -> xpcom;
+    imglib2 -> png;
+    imglib2 -> jpeg;
+    imglib2 -> zlib;
     impText -> intl;
     impText -> pref;
     impText -> necko;
@@ -892,23 +903,6 @@ digraph G {
     wallet -> widget;
     wallet -> xpcom;
     wallet -> unicharutil;
-    necko -> string;
-    necko -> xpcom;
-    necko -> pref;
-    necko -> mimetype;
-    necko -> uconv;
-    necko -> nkcache;
-    necko -> intl;
-    necko -> caps;
-    necko -> xpconnect;
-    necko -> locale;
-    necko -> js;
-    necko -> unicharutil;
-    necko -> exthandler;
-    necko -> util;
-    necko -> jar;
-    necko -> imglib2;
-    necko -> necko2;
     xmlterm -> dom;
     xmlterm -> webshell;
     xmlterm -> content;
@@ -962,6 +956,24 @@ digraph G {
     gfx -> xlibrgb;
     gfx -> windowwatcher;
     gfx -> util;
+    necko -> string;
+    necko -> xpcom;
+    necko -> pref;
+    necko -> mimetype;
+    necko -> uconv;
+    necko -> caps;
+    necko -> nkcache;
+    necko -> intl;
+    necko -> locale;
+    necko -> xpconnect;
+    necko -> js;
+    necko -> unicharutil;
+    necko -> exthandler;
+    necko -> zlib;
+    necko -> jar;
+    necko -> util;
+    necko -> imglib2;
+    necko -> necko2;
     browser -> content;
     browser -> string;
     browser -> gfx;
@@ -1018,25 +1030,6 @@ digraph G {
     transformiix -> intl;
     transformiix -> windowwatcher;
     transformiix -> mimetype;
-    xpinstall -> libreg;
-    xpinstall -> chrome;
-    xpinstall -> caps;
-    xpinstall -> locale;
-    xpinstall -> string;
-    xpinstall -> plugin;
-    xpinstall -> dom;
-    xpinstall -> intl;
-    xpinstall -> pref;
-    xpinstall -> necko;
-    xpinstall -> xpconnect;
-    xpinstall -> uriloader;
-    xpinstall -> windowwatcher;
-    xpinstall -> js;
-    xpinstall -> jar;
-    xpinstall -> xpcom_obsolete;
-    xpinstall -> widget;
-    xpinstall -> xpcom;
-    xpinstall -> unicharutil;
     gtkembedmoz -> string;
     gtkembedmoz -> dom;
     gtkembedmoz -> xpcom;
@@ -1151,6 +1144,26 @@ digraph G {
     oji -> liveconnect;
     oji -> widget;
     oji -> xpcom;
+    xpinstall -> libreg;
+    xpinstall -> chrome;
+    xpinstall -> caps;
+    xpinstall -> locale;
+    xpinstall -> string;
+    xpinstall -> zlib;
+    xpinstall -> plugin;
+    xpinstall -> dom;
+    xpinstall -> intl;
+    xpinstall -> pref;
+    xpinstall -> necko;
+    xpinstall -> xpconnect;
+    xpinstall -> uriloader;
+    xpinstall -> windowwatcher;
+    xpinstall -> js;
+    xpinstall -> jar;
+    xpinstall -> xpcom_obsolete;
+    xpinstall -> widget;
+    xpinstall -> xpcom;
+    xpinstall -> unicharutil;
     apprunner -> profile;
     apprunner -> jprof;
     apprunner -> chrome;

tools/module-deps/bootstrap.pl

@@ -178,7 +178,7 @@ sub get_dirs_for_module($) {
   my $dirs_string = "";
   my $dirs_string_no_mozilla = "";  # dirs_string, stripping off mozilla/
   
-  my $dirs_cmd = "echo $modules_string | $topsrcdir/config/module2dir\.pl --list-only";
+  my $dirs_cmd = "echo $modules_string | $topsrcdir/config/module2dir\.pl --list-only --mapfile $toolsdir/module2dir\.map";
 
   print "\nGenerating directories list for $root_modules\n" if (!$silence);
   $dirs_string = run_shell_command($dirs_cmd, 0);

tools/module-deps/force_order.txt

@@ -17,6 +17,8 @@
 #               you can figure it out.
 #
 
+xpcom                       embedstring
+xpcom                       string
 png                         imglib2
 jpeg                        imglib2
 find                        embedcomponents

tools/module-deps/modules.mk.footer

@@ -1,4 +1,9 @@
 
+# There's always an exception to the rule...
+# XSLT has a special standalone mode that uses ifdefs to maintain a reduced dependency list.
+BM_DIRS_transformiix = xpcom expat extensions/transformiix
+BM_CVS_transformiix = xpcom expat extensions/transformiix
+
 #
 # Tally
 #

tools/module-deps/modules.mk.stub

@@ -16,12 +16,15 @@
 # Copyright (C) 2000-2002 Christopher Seawood. All Rights Reserved.
 #
 #
-BUILD_MODULE_DIRS := config build include
+BUILD_MODULE_DIRS := config build
-BUILD_MODULE_CVS = config build include
+BUILD_MODULE_CVS = config build
 
 # client.mk does not have topsrcdir set
 ifndef topsrcdir
 topsrcdir=$(TOPSRCDIR)
+ifdef MOZ_TOOLS
+topsrcdir := $(shell cygpath -w $(TOPSRCDIR) | sed -e 's|\\|/|g')
+endif
 endif
 
 ifndef MOZ_NATIVE_NSPR