Bug 1857516 - Get rid of deprecated imp python module in favor of importlib r=saschanaz CLOSED TREE

Following the official migration guide from
https://docs.python.org/3/whatsnew/3.12.html#imp

Differential Revision: https://phabricator.services.mozilla.com/D190465
This commit is contained in:
serge-sans-paille 2023-10-17 09:17:47 +00:00
Родитель ea9336a2ca
Коммит 65ae75234f
29 изменённых файлов: 130 добавлений и 96 удалений

Просмотреть файл

@ -2,7 +2,6 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import imp
import io
import json
import os
@ -12,7 +11,7 @@ import tempfile
import unittest
import mozpack.path as mozpath
from mozfile import NamedTemporaryFile
from mozfile import NamedTemporaryFile, load_source
from mozunit import MockedOpen, main
from mozwebidlcodegen import WebIDLCodegenManager, WebIDLCodegenManagerState
@ -242,7 +241,7 @@ class TestWebIDLCodegenManager(unittest.TestCase):
with NamedTemporaryFile("wt") as fh:
fh.write("# Original content")
fh.flush()
mod = imp.load_source("mozwebidlcodegen.fakemodule", fh.name)
mod = load_source("mozwebidlcodegen.fakemodule", fh.name)
mod.__file__ = fake_path
args = self._get_manager_args()

Просмотреть файл

@ -4,11 +4,10 @@
import codecs
import encodings.idna
import imp
import os
import re
import sys
from make_dafsa import words_to_cxx, words_to_bin
from make_dafsa import words_to_bin, words_to_cxx
"""
Processes a file containing effective TLD data. See the following URL for a
@ -22,7 +21,6 @@ http://wiki.mozilla.org/Gecko:Effective_TLD_Service
def getEffectiveTLDs(path):
file = codecs.open(path, "r", "UTF-8")
entries = []
domains = set()
for line in file:
# line always contains a line terminator unless the file is empty

Просмотреть файл

@ -5,13 +5,15 @@
import argparse
import ast
import errno
import imp
import sys
import types
import uuid
from collections.abc import Iterable
from pathlib import Path
from typing import Dict, Optional, Union
from mozfile import load_source
from .base import MissingFileError
INVALID_ENTRY_POINT = r"""
@ -411,13 +413,13 @@ def load_commands_from_file(path: Union[str, Path], module_name=None):
# Ensure parent module is present otherwise we'll (likely) get
# an error due to unknown parent.
if "mach.commands" not in sys.modules:
mod = imp.new_module("mach.commands")
mod = types.ModuleType("mach.commands")
sys.modules["mach.commands"] = mod
module_name = f"mach.commands.{uuid.uuid4().hex}"
try:
imp.load_source(module_name, str(path))
load_source(module_name, str(path))
except IOError as e:
if e.errno != errno.ENOENT:
raise

Просмотреть файл

@ -1,8 +1,8 @@
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import imp
import sys
import types
from pathlib import Path
from unittest.mock import patch
@ -38,7 +38,7 @@ class TestEntryPoints(TestBase):
# Ensure parent module is present otherwise we'll (likely) get
# an error due to unknown parent.
if "mach.commands" not in sys.modules:
mod = imp.new_module("mach.commands")
mod = types.ModuleType("mach.commands")
sys.modules["mach.commands"] = mod
mock.return_value = [Entry([self.provider_dir])]

Просмотреть файл

@ -28,6 +28,7 @@ from mach.decorators import (
SettingsProvider,
SubCommand,
)
from mozfile import load_source
import mozbuild.settings # noqa need @SettingsProvider hook to execute
from mozbuild.base import (
@ -1115,11 +1116,10 @@ def android_gtest(
# run gtest via remotegtests.py
exit_code = 0
import imp
path = os.path.join("testing", "gtest", "remotegtests.py")
with open(path, "r") as fh:
imp.load_module("remotegtests", fh, path, (".py", "r", imp.PY_SOURCE))
load_source("remotegtests", path)
import remotegtests
tester = remotegtests.RemoteGTests()

Просмотреть файл

@ -2,7 +2,7 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import imp
import os
import re
import sys
@ -14,6 +14,7 @@ from unittest.case import SkipTest
import six
from marionette_driver.errors import TimeoutException, UnresponsiveInstanceException
from mozfile import load_source
from mozlog import get_default_logger
@ -363,21 +364,20 @@ class MarionetteTestCase(CommonTestCase):
testvars,
**kwargs
):
# since we use imp.load_source to load test modules, if a module
# is loaded with the same name as another one the module would just be
# reloaded.
# since load_source caches modules, if a module is loaded with the same
# name as another one the module would just be reloaded.
#
# We may end up by finding too many test in a module then since
# reload() only update the module dict (so old keys are still there!)
# see https://docs.python.org/2/library/functions.html#reload
# We may end up by finding too many test in a module then since reload()
# only update the module dict (so old keys are still there!) see
# https://docs.python.org/2/library/functions.html#reload
#
# we get rid of that by removing the module from sys.modules,
# so we ensure that it will be fully loaded by the
# imp.load_source call.
# we get rid of that by removing the module from sys.modules, so we
# ensure that it will be fully loaded by the imp.load_source call.
if mod_name in sys.modules:
del sys.modules[mod_name]
test_mod = imp.load_source(mod_name, filepath)
test_mod = load_source(mod_name, filepath)
for name in dir(test_mod):
obj = getattr(test_mod, name)

Просмотреть файл

@ -14,6 +14,7 @@ import six
from mach.decorators import Command, CommandArgument
from mozbuild.base import MachCommandConditions as conditions
from mozbuild.base import MozbuildObject
from mozfile import load_source
here = os.path.abspath(os.path.dirname(__file__))
@ -90,11 +91,8 @@ class MochitestRunner(MozbuildObject):
"""Runs a mochitest."""
# runtests.py is ambiguous, so we load the file/module manually.
if "mochitest" not in sys.modules:
import imp
path = os.path.join(self.mochitest_dir, "runtests.py")
with open(path, "r") as fh:
imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
load_source("mochitest", path)
import mochitest
@ -145,11 +143,9 @@ class MochitestRunner(MozbuildObject):
if host_ret != 0:
return host_ret
import imp
path = os.path.join(self.mochitest_dir, "runtestsremote.py")
with open(path, "r") as fh:
imp.load_module("runtestsremote", fh, path, (".py", "r", imp.PY_SOURCE))
load_source("runtestsremote", path)
import runtestsremote
options = Namespace(**kwargs)
@ -190,14 +186,11 @@ def setup_argument_parser():
with warnings.catch_warnings():
warnings.simplefilter("ignore")
import imp
path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
if not os.path.exists(path):
path = os.path.join(here, "runtests.py")
with open(path, "r") as fh:
imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
load_source("mochitest", path)
from mochitest_options import MochitestArgumentParser
@ -232,14 +225,11 @@ def setup_junit_argument_parser():
warnings.simplefilter("ignore")
# runtests.py contains MochitestDesktop, required by runjunit
import imp
path = os.path.join(build_obj.topobjdir, mochitest_dir, "runtests.py")
if not os.path.exists(path):
path = os.path.join(here, "runtests.py")
with open(path, "r") as fh:
imp.load_module("mochitest", fh, path, (".py", "r", imp.PY_SOURCE))
load_source("mochitest", path)
import runjunit
from mozrunner.devices.android_device import (

Просмотреть файл

@ -22,6 +22,7 @@ __all__ = [
"extract",
"is_url",
"load",
"load_source",
"copy_contents",
"match",
"move",
@ -632,6 +633,19 @@ def load(resource):
return urllib.request.urlopen(resource)
# see https://docs.python.org/3/whatsnew/3.12.html#imp
def load_source(modname, filename):
import importlib.machinery
import importlib.util
loader = importlib.machinery.SourceFileLoader(modname, filename)
spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
module = importlib.util.module_from_spec(spec)
sys.modules[module.__name__] = module
loader.exec_module(module)
return module
# We can't depend on mozpack.path here, so copy the 'match' function over.
re_cache = {}

Просмотреть файл

@ -12,7 +12,6 @@ author: Jordan Lund
import copy
import glob
import imp
import json
import multiprocessing
import os
@ -25,6 +24,7 @@ from datetime import datetime, timedelta
here = os.path.abspath(os.path.dirname(__file__))
sys.path.insert(1, os.path.dirname(here))
from mozfile import load_source
from mozharness.base.errors import BaseErrorList
from mozharness.base.log import INFO, WARNING
from mozharness.base.script import PreScriptAction
@ -1198,7 +1198,7 @@ class DesktopUnittest(TestingMixin, MercurialScript, MozbaseMixin, CodeCoverageM
)
if suite_category == "reftest":
ref_formatter = imp.load_source(
ref_formatter = load_source(
"ReftestFormatter",
os.path.abspath(
os.path.join(dirs["abs_reftest_dir"], "output.py")

Просмотреть файл

@ -5,13 +5,13 @@
import argparse
import errno
import hashlib
import imp
import os
import sys
import manifestdownload
import six
from mach.util import get_state_dir
from mozfile import load_source
from mozlog.structured import commandline
from six.moves import configparser
from wptrunner import wptcommandline
@ -21,9 +21,7 @@ manifest = None
def do_delayed_imports(wpt_dir):
global manifest
imp.load_source(
"localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py")
)
load_source("localpaths", os.path.join(wpt_dir, "tests", "tools", "localpaths.py"))
sys.path.insert(0, os.path.join(wpt_dir, "tools", "manifest"))
import manifest

Просмотреть файл

@ -1,6 +1,8 @@
import os
import imp
from tools.wpt.utils import load_source
# Use the file from the parent directory.
mod = imp.load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
mod = load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
os.path.basename(__file__)))
main = mod.main

Просмотреть файл

@ -1,6 +1,8 @@
import os
import imp
from tools.wpt.utils import load_source
# Use the file from the parent directory.
mod = imp.load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
mod = load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
os.path.basename(__file__)))
main = mod.main

Просмотреть файл

@ -1,6 +1,8 @@
import os
import imp
from tools.wpt.utils import load_source
# Use the file from the parent directory.
mod = imp.load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
mod = load_source("_parent", os.path.join(os.path.dirname(os.path.dirname(__file__)),
os.path.basename(__file__)))
main = mod.main

Просмотреть файл

@ -1,16 +1,18 @@
# mypy: ignore-errors
import imp
import json
import os
from tools.wpt.utils import load_source
here = os.path.dirname(__file__)
localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, "localpaths.py")))
localpaths = load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, "localpaths.py")))
root = localpaths.repo_root
from manifest import manifest
def main(request, response):
path = os.path.join(root, "MANIFEST.json")

Просмотреть файл

@ -3,10 +3,10 @@
import errno
import logging
import os
import sys
import shutil
import stat
import subprocess
import sys
import tarfile
import time
import zipfile
@ -166,3 +166,16 @@ def sha256sum(file_path):
for chunk in iter(lambda: f.read(4096), b''):
hash.update(chunk)
return hash.hexdigest()
# see https://docs.python.org/3/whatsnew/3.12.html#imp
def load_source(modname, filename):
import importlib.machinery
import importlib.util
loader = importlib.machinery.SourceFileLoader(modname, filename)
spec = importlib.util.spec_from_file_location(modname, filename, loader=loader)
module = importlib.util.module_from_spec(spec)
sys.modules[module.__name__] = module
loader.exec_module(module)
return module

Просмотреть файл

@ -2,7 +2,6 @@
import copy
import functools
import imp
import io
import os
from collections import OrderedDict, defaultdict
@ -10,16 +9,17 @@ from datetime import datetime
from mozlog import reader
from mozlog.formatters import JSONFormatter
from mozlog.handlers import BaseHandler, StreamHandler, LogLevelFilter
from mozlog.handlers import BaseHandler, LogLevelFilter, StreamHandler
from tools.wpt.utils import load_source
from . import wptrunner
here = os.path.dirname(__file__)
localpaths = imp.load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py")))
localpaths = load_source("localpaths", os.path.abspath(os.path.join(here, os.pardir, os.pardir, "localpaths.py")))
from ci.tc.github_checks_output import get_gh_checks_outputter # type: ignore
from wpt.markdown import markdown_adjust, table # type: ignore
# If a test takes more than (FLAKY_THRESHOLD*timeout) and does not consistently
# time out, it is considered slow (potentially flaky).
FLAKY_THRESHOLD = 0.8

Просмотреть файл

@ -1,12 +1,13 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,12 +1,14 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,12 +1,13 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,12 +1,14 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,8 +1,9 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(isomorphic_decode(__file__))
def main(request, response):
@ -11,7 +12,7 @@ def main(request, response):
response.headers.set(b'Access-Control-Allow-Methods', b'GET')
response.headers.set(b'Access-Control-Allow-Headers', b'authorization, x-user, x-pass')
response.headers.set(b'Access-Control-Expose-Headers', b'x-challenge, xhr-user, ses-user')
auth = imp.load_source(u"", os.path.abspath(os.path.join(here, os.pardir, u"authentication.py")))
auth = load_source(u"", os.path.abspath(os.path.join(here, os.pardir, u"authentication.py")))
if request.method == u"OPTIONS":
return b""
else:

Просмотреть файл

@ -1,12 +1,13 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,12 +1,13 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -1,8 +1,9 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(isomorphic_decode(__file__))
def main(request, response):
@ -11,7 +12,7 @@ def main(request, response):
response.headers.set(b'Access-Control-Allow-Methods', b'GET')
response.headers.set(b'Access-Control-Allow-Headers', b'authorization, x-user, x-pass')
response.headers.set(b'Access-Control-Expose-Headers', b'x-challenge, xhr-user, ses-user')
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
os.pardir,
u"authentication.py"))
if request.method == u"OPTIONS":

Просмотреть файл

@ -1,8 +1,9 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(isomorphic_decode(__file__))
def main(request, response):
@ -11,7 +12,7 @@ def main(request, response):
response.headers.set(b'Access-Control-Allow-Methods', b'GET')
response.headers.set(b'Access-Control-Allow-Headers', b'x-user, x-pass')
response.headers.set(b'Access-Control-Expose-Headers', b'x-challenge, xhr-user, ses-user')
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
os.pardir,
u"authentication.py"))
if request.method == u"OPTIONS":

Просмотреть файл

@ -1,12 +1,13 @@
import imp
import os
from wptserve.utils import isomorphic_decode
from tools.wpt.utils import load_source
here = os.path.dirname(os.path.abspath(isomorphic_decode(__file__)))
def main(request, response):
auth = imp.load_source(u"", os.path.join(here,
auth = load_source(u"", os.path.join(here,
u"..",
u"authentication.py"))
return auth.main(request, response)

Просмотреть файл

@ -4,15 +4,15 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import imp
import os
import sys
from mozfile import load_source
from mozlog import structuredlog
here = os.path.split(__file__)[0]
imp.load_source(
load_source(
"localpaths", os.path.join(here, os.pardir, "tests", "tools", "localpaths.py")
)

Просмотреть файл

@ -21,6 +21,7 @@ import yaml
from mach.decorators import Command, CommandArgument, SubCommand
from mach.registrar import Registrar
from mozbuild.util import memoize
from mozfile import load_source
here = os.path.abspath(os.path.dirname(__file__))
topsrcdir = os.path.abspath(os.path.dirname(os.path.dirname(here)))
@ -363,11 +364,8 @@ def toggle_no_autodoc():
@memoize
def _read_project_properties():
import imp
path = os.path.normpath(manager().conf_py_path)
with open(path, "r") as fh:
conf = imp.load_module("doc_conf", fh, path, (".py", "r", imp.PY_SOURCE))
conf = load_source("doc_conf", path)
# Prefer the Mozilla project name, falling back to Sphinx's
# default variable if it isn't defined.

Просмотреть файл

@ -2,12 +2,12 @@
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
import imp
import json
import os
import mozunit
import pytest
from mozfile import load_source
from tryselect.tasks import build, resolve_tests_by_suite
MOZHARNESS_SCRIPTS = {
@ -55,10 +55,10 @@ uses).
def get_mozharness_test_paths(name):
scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness", "scripts")
files = imp.find_module(name, [scriptdir])
mod = imp.load_module("scripts.{}".format(name), *files)
scriptdir = os.path.join(build.topsrcdir, "testing", "mozharness")
mod = load_source(
"scripts." + name, os.path.join(scriptdir, "scripts", name + ".py")
)
class_name = MOZHARNESS_SCRIPTS[name]["class_name"]
cls = getattr(mod, class_name)
@ -82,13 +82,15 @@ def all_suites():
def generate_suites_from_config(path):
configdir = os.path.join(build.topsrcdir, "testing", "mozharness", "configs")
parent, name = os.path.split(path)
name = os.path.splitext(name)[0]
files = imp.find_module("{}".format(name), [os.path.join(configdir, parent)])
mod = imp.load_module("config.{}".format(name), *files)
configdir = os.path.join(
build.topsrcdir, "testing", "mozharness", "configs", parent
)
mod = load_source(name, os.path.join(configdir, name + ".py"))
config = mod.config
for category in sorted(config["suite_definitions"]):