bug 391680, don't hardcode browser/mail in too many places, make compare locales work better for suite and calendar, r=KaiRo

testing/tests/l10n/lib/Mozilla/CompareLocales.py

@@ -116,16 +116,37 @@ class FileCollector:
   def iterateFiles(self, mod, locale):
     return FileCollector.Iter(Paths.get_base_path(mod, locale))
 
-def collectFiles(aComparer):
-  'returns new files, files to compare, files to remove'
+def collectFiles(aComparer, apps = None, locales = None):
+  '''
+  returns new files, files to compare, files to remove
+  apps or locales need to be given, apps is a list, locales is a
+  hash mapping applications to languages.
+  If apps is given, it will look up all-locales for all apps for the
+  languages to test.
+  'toolkit' is added to the list of modules, too.
+  '''
+  if not apps and not locales:
+    raise RuntimeError, "collectFiles needs either apps or locales"
+  if apps and locales:
+    raise RuntimeError, "You don't want to give both apps or locales"
+  if locales:
+    apps = locales.keys()
+    # add toolkit, with all of the languages of all apps
+    all = set()
+    for locs in locales.values():
+      all.update(locs)
+    locales['toolkit'] = list(all)
+  else:
+    locales = Paths.allLocales(apps)
+  modules = Paths.Modules(apps)
   en = FileCollector()
   l10n = FileCollector()
-  for cat in Paths.locales.keys():
-    logging.debug(" testing " + cat+ " on " + str(Paths.modules))
-    aComparer.notifyLocales(cat, Paths.locales[cat])
-    for mod in Paths.modules[cat]:
+  for cat in modules.keys():
+    logging.debug(" testing " + cat+ " on " + str(modules))
+    aComparer.notifyLocales(cat, locales[cat])
+    for mod in modules[cat]:
       en_fls = en.getFiles(mod, 'en-US')
-      for loc in Paths.locales[cat]:
+      for loc in locales[cat]:
         fls = dict(en_fls) # create copy for modification
         for l_fl, l_path in l10n.iterateFiles(mod, loc):
           if fls.has_key(l_fl):
@@ -180,10 +201,10 @@ class CompareCollector:
       self.files[aLocale]['obsoleteFiles'].append((aModule, aLeaf))
     pass
 
-def compare(testLocales=[]):
+def compare(apps=None, testLocales=None):
   result = {}
   c = CompareCollector()
-  collectFiles(c)
+  collectFiles(c, apps=apps, locales=testLocales)
   key = re.compile('[kK]ey')
   for fl, locales in c.cl.iteritems():
     (mod,path) = fl

testing/tests/l10n/lib/Mozilla/Paths.py

@@ -35,28 +35,59 @@
 #
 # ***** END LICENSE BLOCK *****
 
-modules = {'toolkit': ['netwerk','dom','toolkit','security/manager'],
-           'browser': ['browser','extensions/reporter',
-                       'other-licenses/branding/firefox'],
-           'mail': ['mail','other-licenses/branding/thunderbird',
-                    'editor/ui']}
+import os.path
+from subprocess import *
 
-components = {}
-for mod, lst in modules.iteritems():
-  for c in lst:
-    components[c] = mod
 
-locales = {}
-all = {}
-for app in ['browser', 'mail']:
-  path = 'mozilla/%s/locales/all-locales' % app
-  locales[app] = [l.strip() for l in open(path)]
-  for loc in locales[app]: all[loc] = 1
-all = sorted(all.keys())
-locales['toolkit'] = all
-#modules = {'browser': ['browser']} # XXX debug
-#locales = {'browser': ['fr', 'pl']} # locales['mail']} # XXX debug
-pass
+class Modules(dict):
+  '''
+  Subclass of dict to hold information on which directories belong to a
+  particular app.
+  It expects to have mozilla/client.mk right there from the working dir,
+  and asks that for the LOCALES_foo variables.
+  This only works for toolkit applications, as it's assuming that the
+  apps include toolkit.
+  '''
+  def __init__(self, apps):
+    super(dict, self).__init__()
+    lapps = apps[:]
+    lapps.insert(0, 'toolkit')
+    of  = os.popen('make -f mozilla/client.mk ' + \
+                   ' '.join(['echo-variable-LOCALES_' + app for app in lapps]))
+    
+    for val in of.readlines():
+      self[lapps.pop(0)] = val.strip().split()
+    for k,v in self.iteritems():
+      if k == 'toolkit':
+        continue
+      self[k] = [d for d in v if d not in self['toolkit']]
+
+class Components(dict):
+  '''
+  Subclass of dict to map module dirs to applications. This reverses the
+  mapping you'd get from a Modules class, and it in fact uses one to do
+  its job.
+  '''
+  def __init__(self, apps):
+    modules = Modules(apps)
+    for mod, lst in modules.iteritems():
+      for c in lst:
+        self[c] = mod
+
+def allLocales(apps):
+  '''
+  Get a locales hash for the given list of applications, mapping
+  applications to the list of languages given by all-locales.
+  Adds a module 'toolkit' holding all languages for all applications, too.
+  '''
+  locales = {}
+  all = set()
+  for app in apps:
+    path = 'mozilla/%s/locales/all-locales' % app
+    locales[app] = [l.strip() for l in open(path)]
+    all.update(locales[app])
+  locales['toolkit'] = list(all)
+  return locales
 
 def get_base_path(mod, loc):
   'statics for path patterns and conversion'

testing/tests/l10n/lib/Mozilla/Tests.py

@@ -65,7 +65,7 @@ class CompareTest(Base):
     pass
   def run(self):
     '''Runs CompareLocales.compare()'''
-    return CompareLocales.compare()
+    return CompareLocales.compare(apps=['browser','mail'])
   def serialize(self, result, saveHandler):
     '''Serialize the CompareLocales result by locale into
       cmp-details-ab-CD
@@ -76,6 +76,7 @@ class CompareTest(Base):
     class Separator:
       def __init__(self):
         self.leafBase = 'cmp-details-'
+        self.components = Paths.Components(['browser','mail'])
       def getDetails(self, res, locale):
         dic = {}
         res[locale]['tested'].sort()
@@ -89,7 +90,7 @@ class CompareTest(Base):
         counts = dict([(mod,0) for mod in res['tested']])
         counts['total'] = len(res[name])
         for mod, path, key in res[name]:
-          counts[Paths.components[mod]] +=1
+          counts[self.components[mod]] +=1
           if not dic[name].has_key(mod):
             dic[name][mod] = {path:[key]}
             continue
@@ -105,7 +106,7 @@ class CompareTest(Base):
         counts = dict([(mod,0) for mod in res['tested']])
         counts['total'] = len(res[name])
         for mod, path in res[name]:
-          counts[Paths.components[mod]] +=1
+          counts[self.components[mod]] +=1
           if not dic[name].has_key(mod):
             dic[name][mod] = [path]
           else:

testing/tests/l10n/scripts/compare-locales

@@ -63,15 +63,16 @@ logging.basicConfig(level=(logging.WARNING - (options.v - options.q)*10))
 
 # import Paths loaded all-locales for both browser and mail, we overwrite
 # that with our settings before calling into CompareLocales
-Paths.locales = {options.application: args, 'toolkit': args}
+locales = {options.application: args}
 
 # actually compare the localizations
-res = CompareLocales.compare()
+res = CompareLocales.compare(testLocales=locales)
 
 # helper class to merge all the lists into more consice
 # dicts
 class Separator:
-  def __init__(self):
+  def __init__(self, apps):
+    self.components = Paths.Components(apps)
     pass
   def getDetails(self, res, locale):
     dic = {}
@@ -86,7 +87,7 @@ class Separator:
     counts = dict([(mod,0) for mod in res['tested']])
     counts['total'] = len(res[name])
     for mod, path, key in res[name]:
-      counts[Paths.components[mod]] +=1
+      counts[self.components[mod]] +=1
       if mod not in  dic[name]:
         dic[name][mod] = {path:[key]}
         continue
@@ -102,14 +103,14 @@ class Separator:
     counts = dict([(mod,0) for mod in res['tested']])
     counts['total'] = len(res[name])
     for mod, path in res[name]:
-      counts[Paths.components[mod]] +=1
+      counts[self.components[mod]] +=1
       if mod not in dic[name]:
         dic[name][mod] = [path]
       else:
         dic[name][mod].append(path)
     res[name] = counts
 
-s = Separator()
+s = Separator([options.application])
 
 # pretty print results for all localizations
 for loc in args:

testing/tests/l10n/scripts/test-locales

@@ -147,7 +147,7 @@ opts, optlist = cp.parse_args(sys.argv[1:])
 #
 # Set up Logging
 #
-logging.basicConfig(level=(logging.INFO + 10*(opts.q - opts.v)))
+logging.basicConfig(level=(logging.WARNING + 10*(opts.q - opts.v)))
 # Add a handler to store the output
 h = LogHandler()
 logging.getLogger('').addHandler(h)