Swift: make `codegen` a bit more language-agnostic

This commit is contained in:
Paolo Tranquilli 2023-02-21 14:25:59 +01:00
Родитель f8f926ad50
Коммит e4627cb702
8 изменённых файлов: 114 добавлений и 98 удалений

Просмотреть файл

@ -24,31 +24,44 @@ def _parse_args() -> argparse.Namespace:
"and cpp")
p.add_argument("--verbose", "-v", action="store_true", help="print more information")
p.add_argument("--quiet", "-q", action="store_true", help="only print errors")
p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir,
help="the directory that should be regarded as the root of the swift codebase. Used to compute QL "
"imports and in some comments (default %(default)s)")
p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "schema.py",
help="input schema file (default %(default)s)")
p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme",
help="output file for dbscheme generation, input file for trap generation (default %(default)s)")
p.add_argument("--ql-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated",
help="output directory for generated QL files (default %(default)s)")
p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements",
help="output directory for QL stub/customization files (default %(default)s). Defines also the "
"generated qll file importing every class file")
p.add_argument("--ql-test-output", type=_abspath, default=paths.swift_dir / "ql/test/extractor-tests/generated",
help="output directory for QL generated extractor test files (default %(default)s)")
p.add_argument("--root-dir", type=_abspath, default=paths.root_dir,
help="the directory that should be regarded as the root of the language pack codebase. Used to"
"compute QL imports and in some comments and as root for relative paths provided as options "
"(default %(default)s)")
p.add_argument("--language", default=paths.root_dir.name,
help="string that should replace {language} in other provided options")
path_arguments = [
p.add_argument("--schema", default="schema.py",
help="input schema file (default %(default)s)"),
p.add_argument("--dbscheme", default="ql/lib/{language}.dbscheme",
help="output file for dbscheme generation, input file for trap generation (default "
"%(default)s)"),
p.add_argument("--ql-output", default="ql/lib/codeql/{language}/generated",
help="output directory for generated QL files (default %(default)s)"),
p.add_argument("--ql-stub-output", default="ql/lib/codeql/{language}/elements",
help="output directory for QL stub/customization files (default %(default)s). Defines also the "
"generated qll file importing every class file"),
p.add_argument("--ql-test-output", default="ql/test/extractor-tests/generated",
help="output directory for QL generated extractor test files (default %(default)s)"),
p.add_argument("--cpp-output",
help="output directory for generated C++ files, required if trap or cpp is provided to "
"--generate"),
p.add_argument("--generated-registry", default="ql/.generated.list",
help="registry file containing information about checked-in generated code"),
]
p.add_argument("--ql-format", action="store_true", default=True,
help="use codeql to autoformat QL files (which is the default)")
p.add_argument("--no-ql-format", action="store_false", dest="ql_format", help="do not format QL files")
p.add_argument("--codeql-binary", default="codeql", help="command to use for QL formatting (default %(default)s)")
p.add_argument("--cpp-output", type=_abspath,
help="output directory for generated C++ files, required if trap or cpp is provided to --generate")
p.add_argument("--generated-registry", type=_abspath, default=paths.swift_dir / "ql/.generated.list",
help="registry file containing information about checked-in generated code")
p.add_argument("--force", "-f", action="store_true",
help="generate all files without skipping unchanged files and overwriting modified ones")
return p.parse_args()
help="generate all files without skipping unchanged files and overwriting modified ones"),
opts = p.parse_args()
# absolutize all paths relative to --root-dir
for arg in path_arguments:
path = getattr(opts, arg.dest)
if path is not None:
setattr(opts, arg.dest, opts.root_dir / path.format(language=opts.language))
return opts
def _abspath(x: str) -> typing.Optional[pathlib.Path]:
@ -65,7 +78,7 @@ def run():
log_level = logging.INFO
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
for target in opts.generate:
generate(target, opts, render.Renderer(opts.swift_dir))
generate(target, opts, render.Renderer(opts.root_dir))
if __name__ == "__main__":

Просмотреть файл

@ -110,12 +110,12 @@ def get_declarations(data: schema.Schema):
return declarations
def get_includes(data: schema.Schema, include_dir: pathlib.Path, swift_dir: pathlib.Path):
def get_includes(data: schema.Schema, include_dir: pathlib.Path, root_dir: pathlib.Path):
includes = []
for inc in data.includes:
inc = include_dir / inc
with open(inc) as inclusion:
includes.append(SchemeInclude(src=inc.relative_to(swift_dir), data=inclusion.read()))
includes.append(SchemeInclude(src=inc.relative_to(root_dir), data=inclusion.read()))
return includes
@ -125,8 +125,8 @@ def generate(opts, renderer):
data = schemaloader.load_file(input)
dbscheme = Scheme(src=input.relative_to(opts.swift_dir),
includes=get_includes(data, include_dir=input.parent, swift_dir=opts.swift_dir),
dbscheme = Scheme(src=input.relative_to(opts.root_dir),
includes=get_includes(data, include_dir=input.parent, root_dir=opts.root_dir),
declarations=get_declarations(data))
renderer.render(dbscheme, out)

Просмотреть файл

@ -198,8 +198,8 @@ def get_ql_ipa_class(cls: schema.Class):
return get_ql_ipa_class_db(cls.name)
def get_import(file: pathlib.Path, swift_dir: pathlib.Path):
stem = file.relative_to(swift_dir / "ql/lib").with_suffix("")
def get_import(file: pathlib.Path, root_dir: pathlib.Path):
stem = file.relative_to(root_dir / "ql/lib").with_suffix("")
return str(stem).replace("/", ".")
@ -344,7 +344,7 @@ def generate(opts, renderer):
classes_by_dir_and_name = sorted(classes.values(), key=lambda cls: (cls.dir, cls.name))
for c in classes_by_dir_and_name:
imports[c.name] = get_import(stub_out / c.path, opts.swift_dir)
imports[c.name] = get_import(stub_out / c.path, opts.root_dir)
for c in classes.values():
qll = out / c.path.with_suffix(".qll")
@ -355,7 +355,7 @@ def generate(opts, renderer):
path = _get_path(c)
stub_file = stub_out / path
if not renderer.is_customized_stub(stub_file):
base_import = get_import(out / path, opts.swift_dir)
base_import = get_import(out / path, opts.root_dir)
renderer.render(_get_stub(c, base_import), stub_file)
# for example path/to/elements -> path/to/elements.qll
@ -404,7 +404,7 @@ def generate(opts, renderer):
if not renderer.is_customized_stub(stub_file):
# stub rendering must be postponed as we might not have yet all subtracted ipa types in `ipa_type`
stubs[stub_file] = ql.Synth.ConstructorStub(ipa_type)
constructor_import = get_import(stub_file, opts.swift_dir)
constructor_import = get_import(stub_file, opts.root_dir)
constructor_imports.append(constructor_import)
if ipa_type.is_ipa:
ipa_constructor_imports.append(constructor_import)

Просмотреть файл

@ -6,13 +6,13 @@ import os
try:
workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']).resolve() # <- means we are using bazel run
swift_dir = workspace_dir / 'swift'
root_dir = workspace_dir / 'swift'
except KeyError:
_this_file = pathlib.Path(__file__).resolve()
swift_dir = _this_file.parents[2]
workspace_dir = swift_dir.parent
root_dir = _this_file.parents[2]
workspace_dir = root_dir.parent
lib_dir = swift_dir / 'codegen' / 'lib'
templates_dir = swift_dir / 'codegen' / 'templates'
lib_dir = root_dir / 'codegen' / 'lib'
templates_dir = root_dir / 'codegen' / 'templates'
exe_file = pathlib.Path(sys.argv[0]).resolve()

Просмотреть файл

@ -25,13 +25,16 @@ class Error(Exception):
class Renderer:
""" Template renderer using mustache templates in the `templates` directory """
def __init__(self, swift_dir: pathlib.Path):
def __init__(self, root_dir: pathlib.Path):
self._r = pystache.Renderer(search_dirs=str(paths.templates_dir), escape=lambda u: u)
self._swift_dir = swift_dir
self._generator = self._get_path(paths.exe_file)
self._root_dir = root_dir
try:
self._generator = self._get_path(paths.exe_file)
except ValueError:
self._generator = paths.exe_file.name
def _get_path(self, file: pathlib.Path):
return file.relative_to(self._swift_dir)
return file.relative_to(self._root_dir)
def render(self, data: object, output: pathlib.Path):
""" Render `data` to `output`.
@ -60,7 +63,7 @@ class Renderer:
def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path, force: bool = False) -> "RenderManager":
return RenderManager(self._swift_dir, generated, stubs, registry, force)
return RenderManager(self._root_dir, generated, stubs, registry, force)
class RenderManager(Renderer):
@ -85,10 +88,10 @@ class RenderManager(Renderer):
pre: str
post: typing.Optional[str] = None
def __init__(self, swift_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
def __init__(self, root_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path, force: bool = False):
super().__init__(swift_dir)
super().__init__(root_dir)
self._registry_path = registry
self._force = force
self._hashes = {}
@ -117,7 +120,7 @@ class RenderManager(Renderer):
self._hashes.pop(self._get_path(f), None)
# clean up the registry from files that do not exist any more
for f in list(self._hashes):
if not (self._swift_dir / f).exists():
if not (self._root_dir / f).exists():
self._hashes.pop(f)
self._dump_registry()

Просмотреть файл

@ -17,16 +17,16 @@ def run_mock():
# these are lambdas so that they will use patched paths when called
def stub_path(): return paths.swift_dir / "ql/lib/stub/path"
def stub_path(): return paths.root_dir / "ql/lib/stub/path"
def ql_output_path(): return paths.swift_dir / "ql/lib/other/path"
def ql_output_path(): return paths.root_dir / "ql/lib/other/path"
def ql_test_output_path(): return paths.swift_dir / "ql/test/path"
def ql_test_output_path(): return paths.root_dir / "ql/test/path"
def generated_registry_path(): return paths.swift_dir / "registry.list"
def generated_registry_path(): return paths.root_dir / "registry.list"
def import_file(): return stub_path().with_suffix(".qll")
@ -47,7 +47,7 @@ def qlgen_opts(opts):
opts.ql_test_output = ql_test_output_path()
opts.generated_registry = generated_registry_path()
opts.ql_format = True
opts.swift_dir = paths.swift_dir
opts.root_dir = paths.root_dir
opts.force = False
return opts

Просмотреть файл

@ -22,7 +22,7 @@ def pystache_renderer(pystache_renderer_cls):
@pytest.fixture
def sut(pystache_renderer):
return render.Renderer(paths.swift_dir)
return render.Renderer(paths.root_dir)
def assert_file(file, text):
@ -48,12 +48,12 @@ def test_render(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.swift_dir / "some/output.txt"
output = paths.root_dir / "some/output.txt"
sut.render(data, output)
assert_file(output, text)
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
@ -61,8 +61,8 @@ def test_managed_render(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.swift_dir / "some/output.txt"
registry = paths.swift_dir / "a/registry.list"
output = paths.root_dir / "some/output.txt"
registry = paths.root_dir / "a/registry.list"
write(registry)
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
@ -72,7 +72,7 @@ def test_managed_render(pystache_renderer, sut):
assert_file(registry, f"some/output.txt {hash(text)} {hash(text)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
@ -80,8 +80,8 @@ def test_managed_render_with_no_registry(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.swift_dir / "some/output.txt"
registry = paths.swift_dir / "a/registry.list"
output = paths.root_dir / "some/output.txt"
registry = paths.root_dir / "a/registry.list"
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
renderer.render(data, output)
@ -90,7 +90,7 @@ def test_managed_render_with_no_registry(pystache_renderer, sut):
assert_file(registry, f"some/output.txt {hash(text)} {hash(text)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
@ -99,8 +99,8 @@ def test_managed_render_with_post_processing(pystache_renderer, sut):
text = "some text"
postprocessed_text = "some other text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.swift_dir / "some/output.txt"
registry = paths.swift_dir / "a/registry.list"
output = paths.root_dir / "some/output.txt"
registry = paths.root_dir / "a/registry.list"
write(registry)
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
@ -111,14 +111,14 @@ def test_managed_render_with_post_processing(pystache_renderer, sut):
assert_file(registry, f"some/output.txt {hash(text)} {hash(postprocessed_text)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
def test_managed_render_with_erasing(pystache_renderer, sut):
output = paths.swift_dir / "some/output.txt"
stub = paths.swift_dir / "some/stub.txt"
registry = paths.swift_dir / "a/registry.list"
output = paths.root_dir / "some/output.txt"
stub = paths.root_dir / "some/stub.txt"
registry = paths.root_dir / "a/registry.list"
write(output)
write(stub, "// generated bla bla")
write(registry)
@ -134,9 +134,9 @@ def test_managed_render_with_erasing(pystache_renderer, sut):
def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
output = paths.swift_dir / "some/output.txt"
output = paths.root_dir / "some/output.txt"
some_output = "some output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(output, some_output)
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
@ -149,16 +149,16 @@ def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
def test_managed_render_with_skipping_of_stub_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
stub = paths.swift_dir / "some/stub.txt"
stub = paths.root_dir / "some/stub.txt"
some_output = "// generated some output"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(stub, some_processed_output)
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
@ -171,14 +171,14 @@ def test_managed_render_with_skipping_of_stub_file(pystache_renderer, sut):
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
def test_managed_render_with_modified_generated_file(pystache_renderer, sut):
output = paths.swift_dir / "some/output.txt"
output = paths.root_dir / "some/output.txt"
some_processed_output = "// some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(output, "// something else")
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
@ -187,9 +187,9 @@ def test_managed_render_with_modified_generated_file(pystache_renderer, sut):
def test_managed_render_with_modified_stub_file_still_marked_as_generated(pystache_renderer, sut):
stub = paths.swift_dir / "some/stub.txt"
stub = paths.root_dir / "some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(stub, "// generated something else")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
@ -198,9 +198,9 @@ def test_managed_render_with_modified_stub_file_still_marked_as_generated(pystac
def test_managed_render_with_modified_stub_file_not_marked_as_generated(pystache_renderer, sut):
stub = paths.swift_dir / "some/stub.txt"
stub = paths.root_dir / "some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(stub, "// no more generated")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
@ -218,11 +218,11 @@ def test_managed_render_exception_drops_written_and_inexsistent_from_registry(py
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.swift_dir / "some/output.txt"
registry = paths.swift_dir / "x/registry.list"
output = paths.root_dir / "some/output.txt"
registry = paths.root_dir / "x/registry.list"
write(output, text)
write(paths.swift_dir / "a")
write(paths.swift_dir / "c")
write(paths.root_dir / "a")
write(paths.root_dir / "c")
write(registry, "a a a\n"
f"some/output.txt whatever {hash(text)}\n"
"b b b\n"
@ -237,9 +237,9 @@ def test_managed_render_exception_drops_written_and_inexsistent_from_registry(py
def test_managed_render_drops_inexsistent_from_registry(pystache_renderer, sut):
registry = paths.swift_dir / "x/registry.list"
write(paths.swift_dir / "a")
write(paths.swift_dir / "c")
registry = paths.root_dir / "x/registry.list"
write(paths.root_dir / "a")
write(paths.root_dir / "c")
write(registry, f"a {hash('')} {hash('')}\n"
"b b b\n"
f"c {hash('')} {hash('')}")
@ -251,9 +251,9 @@ def test_managed_render_drops_inexsistent_from_registry(pystache_renderer, sut):
def test_managed_render_exception_does_not_erase(pystache_renderer, sut):
output = paths.swift_dir / "some/output.txt"
stub = paths.swift_dir / "some/stub.txt"
registry = paths.swift_dir / "a/registry.list"
output = paths.root_dir / "some/output.txt"
stub = paths.root_dir / "some/stub.txt"
registry = paths.root_dir / "a/registry.list"
write(output)
write(stub, "// generated bla bla")
write(registry)
@ -277,7 +277,7 @@ def test_render_with_extensions(pystache_renderer, sut):
sut.render(data, output)
expected_templates = ["test_template_foo", "test_template_bar", "test_template_baz"]
assert pystache_renderer.mock_calls == [
mock.call.render_name(t, data, generator=paths.exe_file.relative_to(paths.swift_dir))
mock.call.render_name(t, data, generator=paths.exe_file.relative_to(paths.root_dir))
for t in expected_templates
]
for expected_output, expected_contents in zip(expected_outputs, rendered):
@ -286,9 +286,9 @@ def test_render_with_extensions(pystache_renderer, sut):
def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
output = paths.swift_dir / "some/output.txt"
output = paths.root_dir / "some/output.txt"
some_output = "some output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(output, some_output)
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
@ -301,16 +301,16 @@ def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
stub = paths.swift_dir / "some/stub.txt"
stub = paths.root_dir / "some/stub.txt"
some_output = "// generated some output"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(stub, some_processed_output)
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
@ -323,14 +323,14 @@ def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.root_dir)),
]
def test_managed_render_with_force_ignores_modified_generated_file(sut):
output = paths.swift_dir / "some/output.txt"
output = paths.root_dir / "some/output.txt"
some_processed_output = "// some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(output, "// something else")
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
@ -339,9 +339,9 @@ def test_managed_render_with_force_ignores_modified_generated_file(sut):
def test_managed_render_with_force_ignores_modified_stub_file_still_marked_as_generated(sut):
stub = paths.swift_dir / "some/stub.txt"
stub = paths.root_dir / "some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
registry = paths.root_dir / "a/registry.list"
write(stub, "// generated something else")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")

Просмотреть файл

@ -33,13 +33,13 @@ def render_manager(renderer):
@pytest.fixture
def opts():
ret = mock.MagicMock()
ret.swift_dir = paths.swift_dir
ret.root_dir = paths.root_dir
return ret
@pytest.fixture(autouse=True)
def override_paths(tmp_path):
with mock.patch("swift.codegen.lib.paths.swift_dir", tmp_path), \
with mock.patch("swift.codegen.lib.paths.root_dir", tmp_path), \
mock.patch("swift.codegen.lib.paths.exe_file", tmp_path / "exe"):
yield