Bug 1515746 - [flake8] Unsupport subdir .flake8 files and use new 'per-file-ignores' config instead, r=egao

This removes all .flake8 files except for the one at the root of the repo.
Instead we use the new 'per-file-ignores' config introduced in 3.7. To ignore
specific errors in a subdirectory, add a line like this to the root .flake8:

[per-file-ignores]
    path/to/subdir/*: E100, F200, ...

The reasons for this change are:

1. Unblock flake8 blacklist (bug 1367092).
2. Simplify configuration and code.
3. Encourage more consistent styling.
4. Improve performance.
5. Greater editor consistency.

Differential Revision: https://phabricator.services.mozilla.com/D18354

--HG--
extra : moz-landing-system : lando

.flake8

@@ -18,7 +18,12 @@ exclude =
     mobile/android/*.configure,
     node_modules,
     security/nss/,
+    testing/firefox-ui/**/__init__.py,
+    testing/marionette/**/__init__.py,
+    testing/marionette/harness/marionette_harness/runner/mixins,
+    testing/marionette/harness/marionette_harness/tests,
     testing/mochitest/pywebsocket,
+    testing/mozharness/configs/test/test_malformed.py,
     tools/lint/test/files,
     tools/infer/test/*.configure,
     tools/crashreporter/*.configure,
@@ -33,3 +38,7 @@ ignore =
     #   F723: syntax error in type comment
     #       text contains quotes which breaks our custom JSON formatter
     F723, E121, E123, E126, E129, E133, E226, E241, E242, E402, E704, E741, W503,
+
+per-file-ignores =
+    ipc/ipdl/*: F403, F405
+    testing/mozharness/configs/*: E124, E127, E128, E131, E231, E261, E265, E266, E501, W391

build/moz.configure/.flake8

@@ -1,5 +0,0 @@
-[flake8]
-# F821: undefined name
-#       This is the moz.configure style
-ignore = F821
-

ipc/ipdl/.flake8

@@ -1,5 +0,0 @@
-[flake8]
-# See http://pep8.readthedocs.io/en/latest/intro.html#configuration
-ignore = E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402, E741, F405, F403, W504
-max-line-length = 99
-

security/manager/.flake8

@@ -1,14 +0,0 @@
-[flake8]
-# E128: continuation line under-indented for visual indent.
-#       Best visual indent typically determined based on context.
-# E261: at least two spaces before inline comment.
-#       Not PSM style.
-# E302: expected 2 blank lines, found 1.
-#       Not really PSM style.
-ignore = E128,E261,E302,W605,W504
-
-# PSM style is to stick close to 80 chars, but 4 space indentation means we
-# sometimes need more space.
-max-line-length = 100
-
-filename = *.py

testing/firefox-ui/.flake8

@@ -1,3 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,

testing/marionette/client/.flake8

@@ -1,5 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
-    disti/*,
-    build/*,

testing/marionette/harness/.flake8

@@ -1,7 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,
-    disti/*,
-    build/*,
-    marionette_harness/runner/mixins/*,
-    marionette_harness/tests/*,

testing/marionette/puppeteer/.flake8

@@ -1,3 +0,0 @@
-[flake8]
-exclude =
-    __init__.py,

testing/mozharness/configs/.flake8

@@ -1,33 +0,0 @@
-[flake8]
-ignore =
-    # From root
-    E121, E123, E126, E129, E133, E226, E241, E242, E704, W503, E402, W605, W504,
-    # The following errors should be fixed eventually
-    # line too long
-    E501,
-    # at least two spaces before inline comment
-    E261,
-    # continuation line under-indented for visual indent
-    E128,
-    # whitespace before ':'
-    E203,
-    # blank line at end of file
-    W391,
-    # multiple statements on one line
-    E702,
-    # closing bracket does not match visual indentation
-    E124,
-    # missing whitespace after ':'
-    E231,
-    # continuation line over-indented for visual indent
-    E127,
-    # too many leading '#' for block comment
-    E266,
-    # comment should start with '# '
-    E262, E265,
-    # continuation line unaligned for hanging indent
-    E131,
-    # continuation line missing indentation or outdented
-    E122,
-exclude =
-    test/test_malformed.py

tools/lint/python/flake8.py

@@ -8,13 +8,11 @@ import platform
 import signal
 import subprocess
 import sys
-from collections import defaultdict
 
 from mozprocess import ProcessHandlerMixin
 
 from mozlint import result
 from mozlint.util import pip
-from mozlint.pathutils import get_ancestors_by_name
 
 
 here = os.path.abspath(os.path.dirname(__file__))
@@ -95,16 +93,6 @@ class Flake8Process(ProcessHandlerMixin):
         signal.signal(signal.SIGINT, orig)
 
 
-def run_process(config, cmd):
-    proc = Flake8Process(config, cmd)
-    proc.run()
-    try:
-        proc.wait()
-    except KeyboardInterrupt:
-        proc.kill()
-        return 1
-
-
 def setup(root):
     if not pip.reinstall_program(FLAKE8_REQUIREMENTS_PATH):
         print(FLAKE8_INSTALL_ERROR)
@@ -116,42 +104,33 @@ def lint(paths, config, **lintargs):
     global results
     results = []
 
+    config_path = os.path.join(lintargs['root'], '.flake8')
     cmdargs = [
         os.path.join(bindir, 'flake8'),
+        '--config', config_path,
         '--format', '{"path":"%(path)s","lineno":%(row)s,'
                     '"column":%(col)s,"rule":"%(code)s","message":"%(text)s"}',
         '--filename', ','.join(['*.{}'.format(e) for e in config['extensions']]),
     ]
 
-    fix_cmdargs = [
+    if lintargs.get('fix'):
-        os.path.join(bindir, 'autopep8'),
+        fix_cmdargs = [
-        '--global-config', os.path.join(lintargs['root'], '.flake8'),
+            os.path.join(bindir, 'autopep8'),
-        '--in-place', '--recursive',
+            '--global-config', config_path,
-    ]
+            '--in-place', '--recursive',
+        ]
 
-    if config.get('exclude'):
+        if config.get('exclude'):
-        fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
+            fix_cmdargs.extend(['--exclude', ','.join(config['exclude'])])
 
-    # Run any paths with a .flake8 file in the directory separately so
+        subprocess.call(fix_cmdargs + paths)
-    # it gets picked up. This means only .flake8 files that live in
-    # directories that are explicitly included will be considered.
-    # See bug 1277851
-    paths_by_config = defaultdict(list)
-    for path in paths:
-        configs = get_ancestors_by_name('.flake8', path, lintargs['root'])
-        paths_by_config[os.pathsep.join(configs) if configs else 'default'].append(path)
 
-    for configs, paths in paths_by_config.items():
+    proc = Flake8Process(config, cmdargs + paths)
-        if lintargs.get('fix'):
+    proc.run()
-            subprocess.call(fix_cmdargs + paths)
+    try:
-
+        proc.wait()
-        cmd = cmdargs[:]
+    except KeyboardInterrupt:
-        if configs != 'default':
+        proc.kill()
-            configs = reversed(configs.split(os.pathsep))
+        return 1
-            cmd.extend(['--append-config={}'.format(c) for c in configs])
-
-        cmd.extend(paths)
-        if run_process(config, cmd):
-            break
 
     return results

tools/lint/test/test_flake8.py

@@ -18,14 +18,11 @@ def test_lint_single_file(lint, paths):
     assert len(results) == 2
 
 
-def test_lint_custom_config(lint, paths):
+def test_lint_custom_config_ignored(lint, paths):
     results = lint(paths('custom'))
-    assert len(results) == 0
+    assert len(results) == 2
 
     results = lint(paths('custom/good.py'))
-    assert len(results) == 0
-
-    results = lint(paths('custom', 'bad.py'))
     assert len(results) == 2