Swift: merge `codegen` and `cppcodegen`

Python code was simplified, and now a `--generate` option can be used
to drive what can be generated.

The extractor pack creation now will use an internally generated
dbscheme. This should be the same as the checked in one, but doing so
allows `bazel run create-extractor-pack` and `bazel run codegen` to be
run independently from one another, while previously the former had to
follow the latter in case of a schema change. This is the change that
triggered the above simplification, as in order for the two dbscheme
files to be identical, the first `// generated` line had to state the
same generator script.
This commit is contained in:
Paolo Tranquilli 2022-06-01 09:47:45 +02:00
Родитель 4b2b6fae88
Коммит 77f7fe8dbc
13 изменённых файлов: 70 добавлений и 149 удалений

2
.github/workflows/swift-codegen.yml поставляемый
Просмотреть файл

@ -25,7 +25,7 @@ jobs:
git diff --exit-code --stat HEAD
- name: Generate C++ files
run: |
bazel run //swift/codegen:cppcodegen -- --cpp-output=$PWD/swift-generated-headers
bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers
- uses: actions/upload-artifact@v3
with:
name: swift-generated-headers

Просмотреть файл

@ -3,17 +3,11 @@ load("@rules_pkg//:install.bzl", "pkg_install")
load("//:defs.bzl", "codeql_platform")
load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
filegroup(
name = "dbscheme",
srcs = ["ql/lib/swift.dbscheme"],
visibility = ["//visibility:public"],
)
pkg_files(
name = "dbscheme_files",
srcs = [
"ql/lib/swift.dbscheme.stats",
":dbscheme",
"//swift/extractor/trap:generated_dbscheme",
],
)

Просмотреть файл

@ -15,15 +15,6 @@ filegroup(
py_binary(
name = "codegen",
srcs = ["codegen.py"],
visibility = ["//swift/codegen/test:__pkg__"],
deps = ["//swift/codegen/generators"],
)
# as opposed to the above, that is meant to only be run with bazel run,
# we need to be precise with data dependencies of this which is meant be run during build
py_binary(
name = "cppcodegen",
srcs = ["cppcodegen.py"],
data = [
":schema",
":schema_includes",

Просмотреть файл

@ -1,7 +1,49 @@
#!/usr/bin/env python3
""" Driver script to run all checked in code generation """
""" Driver script to run all code generation """
import argparse
import logging
import pathlib
import sys
import importlib
import types
import typing
from swift.codegen.lib import render, paths
from swift.codegen.generators import generate
def _parse_args() -> argparse.Namespace:
p = argparse.ArgumentParser()
p.add_argument("--generate", type=lambda x: x.split(","), default=["dbscheme", "ql"])
p.add_argument("--verbose", "-v", action="store_true")
p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir)
p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
p.add_argument("--ql-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
p.add_argument("--ql-format", action="store_true", default=True)
p.add_argument("--no-ql-format", action="store_false", dest="ql_format")
p.add_argument("--codeql-binary", default="codeql")
p.add_argument("--cpp-output", type=_abspath)
p.add_argument("--cpp-namespace", default="codeql")
p.add_argument("--trap-affix", default="Trap")
p.add_argument("--cpp-include-dir", default="swift/extractor/trap")
return p.parse_args()
def _abspath(x: str) -> pathlib.Path:
return pathlib.Path(x).resolve()
def run():
opts = _parse_args()
log_level = logging.DEBUG if opts.verbose else logging.INFO
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
exe_path = paths.exe_file.relative_to(opts.swift_dir)
for target in opts.generate:
generate(target, opts, render.Renderer(exe_path))
from swift.codegen.generators import generator, dbschemegen, qlgen
if __name__ == "__main__":
generator.run(dbschemegen, qlgen)
run()

Просмотреть файл

@ -1,7 +0,0 @@
#!/usr/bin/env python3
""" Driver script to run all cpp code generation """
from swift.codegen.generators import generator, dbschemegen, trapgen, cppgen
if __name__ == "__main__":
generator.run(dbschemegen, trapgen, cppgen)

Просмотреть файл

@ -0,0 +1,6 @@
from . import dbschemegen, qlgen, trapgen, cppgen
def generate(target, opts, renderer):
module = globals()[f"{target}gen"]
module.generate(opts, renderer)

Просмотреть файл

@ -5,7 +5,6 @@ import inflection
from toposort import toposort_flatten
from swift.codegen.lib import cpp, schema
from swift.codegen.generators import generator
def _get_type(t: str, trap_affix: str) -> str:
@ -64,13 +63,8 @@ class Processor:
def generate(opts, renderer):
assert opts.cpp_output
processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes}, opts.trap_affix)
out = opts.cpp_output
renderer.render(cpp.ClassList(processor.get_classes(), opts.cpp_namespace, opts.trap_affix,
opts.cpp_include_dir, opts.schema), out / f"{opts.trap_affix}Classes.h")
tags = ("cpp", "schema")
if __name__ == "__main__":
generator.run()

Просмотреть файл

@ -4,7 +4,6 @@ import pathlib
import inflection
from swift.codegen.lib import schema
from swift.codegen.generators import generator
from swift.codegen.lib.dbscheme import *
log = logging.getLogger(__name__)
@ -93,9 +92,3 @@ def generate(opts, renderer):
declarations=get_declarations(data))
renderer.render(dbscheme, out)
tags = ("schema", "dbscheme")
if __name__ == "__main__":
generator.run()

Просмотреть файл

@ -1,33 +0,0 @@
""" generator script scaffolding """
import argparse
import logging
import sys
from typing import Set
from swift.codegen.lib import render, paths
from swift.codegen.generators import options
def _parse(tags: Set[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser()
for opt in options.get(tags):
opt.add_to(parser)
return parser.parse_args()
def run(*modules, **kwargs):
""" run generation functions in specified in `modules`, or in current module by default
"""
if modules:
if kwargs:
opts = argparse.Namespace(**kwargs)
else:
opts = _parse({t for m in modules for t in m.tags})
log_level = logging.DEBUG if opts.verbose else logging.INFO
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
exe_path = paths.exe_file.relative_to(opts.swift_dir)
for m in modules:
m.generate(opts, render.Renderer(exe_path))
else:
run(sys.modules["__main__"], **kwargs)

Просмотреть файл

@ -1,58 +0,0 @@
""" generator options, categorized by tags """
import argparse
import collections
import pathlib
from typing import Set
from swift.codegen.lib import paths
def _init_options():
Option("--verbose", "-v", action="store_true")
Option("--swift-dir", type=_abspath, default=paths.swift_dir)
Option("--schema", tags=["schema"], type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
Option("--dbscheme", tags=["dbscheme"], type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
Option("--ql-format", tags=["ql"], action="store_true", default=True)
Option("--no-ql-format", tags=["ql"], action="store_false", dest="ql_format")
Option("--codeql-binary", tags=["ql"], default="codeql")
Option("--cpp-output", tags=["cpp"], type=_abspath, required=True)
Option("--cpp-namespace", tags=["cpp"], default="codeql")
Option("--trap-affix", tags=["cpp"], default="Trap")
Option("--cpp-include-dir", tags=["cpp"], default="swift/extractor/trap")
def _abspath(x):
return pathlib.Path(x).resolve()
_options = collections.defaultdict(list)
class Option:
def __init__(self, *args, tags=None, **kwargs):
tags = tags or []
self.args = args
self.kwargs = kwargs
if tags:
for t in tags:
_options[t].append(self)
else:
_options["*"].append(self)
def add_to(self, parser: argparse.ArgumentParser):
parser.add_argument(*self.args, **self.kwargs)
_init_options()
def get(tags: Set[str]):
""" get options marked by `tags`
Options tagged by wildcard '*' are always returned
"""
# use specifically tagged options + those tagged with wildcard *
return (o for tag in ('*',) + tuple(tags) for o in _options[tag])

Просмотреть файл

@ -7,7 +7,6 @@ import subprocess
import inflection
from swift.codegen.lib import schema, ql
from swift.codegen.generators import generator
log = logging.getLogger(__name__)
@ -124,9 +123,3 @@ def generate(opts, renderer):
renderer.cleanup(existing)
if opts.ql_format:
format(opts.codeql_binary, renderer.written)
tags = ("schema", "ql")
if __name__ == "__main__":
generator.run()

Просмотреть файл

@ -6,7 +6,6 @@ import inflection
from toposort import toposort_flatten
from swift.codegen.lib import dbscheme, cpp
from swift.codegen.generators import generator
log = logging.getLogger(__name__)
@ -57,6 +56,7 @@ def get_trap(t: dbscheme.Table, trap_affix: str):
def generate(opts, renderer):
assert opts.cpp_output
tag_graph = {}
out = opts.cpp_output
@ -81,9 +81,3 @@ def generate(opts, renderer):
id=tag,
))
renderer.render(cpp.TagList(tags, opts.cpp_namespace, opts.dbscheme), out / f"{opts.trap_affix}Tags.h")
tags = ("cpp", "dbscheme")
if __name__ == "__main__":
generator.run()

Просмотреть файл

@ -8,19 +8,31 @@ genrule(
"generated/TrapEntries.h",
"generated/TrapTags.h",
"generated/TrapClasses.h",
"generated/swift.dbscheme",
],
cmd = " ".join([
"$(location //swift/codegen:cppcodegen)",
"$(location //swift/codegen)",
"--generate=dbscheme,trap,cpp",
"--schema $(location //swift/codegen:schema)",
"--dbscheme $(RULEDIR)/generated/swift.dbscheme",
"--cpp-include-dir " + package_name(),
"--cpp-output $(RULEDIR)/generated",
]),
exec_tools = ["//swift/codegen:cppcodegen"],
exec_tools = ["//swift/codegen"],
)
filegroup(
name = "generated_dbscheme",
srcs = [":generated/swift.dbscheme"],
visibility = ["//visibility:public"],
)
cc_library(
name = "trap",
hdrs = glob(["*.h"]) + [":cppgen"],
hdrs = glob(["*.h"]) + [
"generated/TrapEntries.h",
"generated/TrapTags.h",
"generated/TrapClasses.h",
],
visibility = ["//visibility:public"],
)