зеркало из https://github.com/github/codeql.git
Swift: merge `codegen` and `cppcodegen`
Python code was simplified, and now a `--generate` option can be used to drive what can be generated. The extractor pack creation now will use an internally generated dbscheme. This should be the same as the checked in one, but doing so allows `bazel run create-extractor-pack` and `bazel run codegen` to be run independently from one another, while previously the former had to follow the latter in case of a schema change. This is the change that triggered the above simplification, as in order for the two dbscheme files to be identical, the first `// generated` line had to state the same generator script.
This commit is contained in:
Родитель
4b2b6fae88
Коммит
77f7fe8dbc
|
@ -25,7 +25,7 @@ jobs:
|
|||
git diff --exit-code --stat HEAD
|
||||
- name: Generate C++ files
|
||||
run: |
|
||||
bazel run //swift/codegen:cppcodegen -- --cpp-output=$PWD/swift-generated-headers
|
||||
bazel run //swift/codegen:codegen -- --generate=trap,cpp --cpp-output=$PWD/swift-generated-headers
|
||||
- uses: actions/upload-artifact@v3
|
||||
with:
|
||||
name: swift-generated-headers
|
||||
|
|
|
@ -3,17 +3,11 @@ load("@rules_pkg//:install.bzl", "pkg_install")
|
|||
load("//:defs.bzl", "codeql_platform")
|
||||
load("//misc/bazel:pkg_runfiles.bzl", "pkg_runfiles")
|
||||
|
||||
filegroup(
|
||||
name = "dbscheme",
|
||||
srcs = ["ql/lib/swift.dbscheme"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
pkg_files(
|
||||
name = "dbscheme_files",
|
||||
srcs = [
|
||||
"ql/lib/swift.dbscheme.stats",
|
||||
":dbscheme",
|
||||
"//swift/extractor/trap:generated_dbscheme",
|
||||
],
|
||||
)
|
||||
|
||||
|
|
|
@ -15,15 +15,6 @@ filegroup(
|
|||
py_binary(
|
||||
name = "codegen",
|
||||
srcs = ["codegen.py"],
|
||||
visibility = ["//swift/codegen/test:__pkg__"],
|
||||
deps = ["//swift/codegen/generators"],
|
||||
)
|
||||
|
||||
# as opposed to the above, that is meant to only be run with bazel run,
|
||||
# we need to be precise with data dependencies of this which is meant be run during build
|
||||
py_binary(
|
||||
name = "cppcodegen",
|
||||
srcs = ["cppcodegen.py"],
|
||||
data = [
|
||||
":schema",
|
||||
":schema_includes",
|
||||
|
|
|
@ -1,7 +1,49 @@
|
|||
#!/usr/bin/env python3
|
||||
""" Driver script to run all checked in code generation """
|
||||
""" Driver script to run all code generation """
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import pathlib
|
||||
import sys
|
||||
import importlib
|
||||
import types
|
||||
import typing
|
||||
|
||||
from swift.codegen.lib import render, paths
|
||||
from swift.codegen.generators import generate
|
||||
|
||||
|
||||
def _parse_args() -> argparse.Namespace:
|
||||
p = argparse.ArgumentParser()
|
||||
p.add_argument("--generate", type=lambda x: x.split(","), default=["dbscheme", "ql"])
|
||||
p.add_argument("--verbose", "-v", action="store_true")
|
||||
p.add_argument("--swift-dir", type=_abspath, default=paths.swift_dir)
|
||||
p.add_argument("--schema", type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
|
||||
p.add_argument("--dbscheme", type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
|
||||
p.add_argument("--ql-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
|
||||
p.add_argument("--ql-stub-output", type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
|
||||
p.add_argument("--ql-format", action="store_true", default=True)
|
||||
p.add_argument("--no-ql-format", action="store_false", dest="ql_format")
|
||||
p.add_argument("--codeql-binary", default="codeql")
|
||||
p.add_argument("--cpp-output", type=_abspath)
|
||||
p.add_argument("--cpp-namespace", default="codeql")
|
||||
p.add_argument("--trap-affix", default="Trap")
|
||||
p.add_argument("--cpp-include-dir", default="swift/extractor/trap")
|
||||
return p.parse_args()
|
||||
|
||||
|
||||
def _abspath(x: str) -> pathlib.Path:
|
||||
return pathlib.Path(x).resolve()
|
||||
|
||||
|
||||
def run():
|
||||
opts = _parse_args()
|
||||
log_level = logging.DEBUG if opts.verbose else logging.INFO
|
||||
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
|
||||
exe_path = paths.exe_file.relative_to(opts.swift_dir)
|
||||
for target in opts.generate:
|
||||
generate(target, opts, render.Renderer(exe_path))
|
||||
|
||||
from swift.codegen.generators import generator, dbschemegen, qlgen
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run(dbschemegen, qlgen)
|
||||
run()
|
||||
|
|
|
@ -1,7 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
""" Driver script to run all cpp code generation """
|
||||
|
||||
from swift.codegen.generators import generator, dbschemegen, trapgen, cppgen
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run(dbschemegen, trapgen, cppgen)
|
|
@ -0,0 +1,6 @@
|
|||
from . import dbschemegen, qlgen, trapgen, cppgen
|
||||
|
||||
|
||||
def generate(target, opts, renderer):
|
||||
module = globals()[f"{target}gen"]
|
||||
module.generate(opts, renderer)
|
|
@ -5,7 +5,6 @@ import inflection
|
|||
from toposort import toposort_flatten
|
||||
|
||||
from swift.codegen.lib import cpp, schema
|
||||
from swift.codegen.generators import generator
|
||||
|
||||
|
||||
def _get_type(t: str, trap_affix: str) -> str:
|
||||
|
@ -64,13 +63,8 @@ class Processor:
|
|||
|
||||
|
||||
def generate(opts, renderer):
|
||||
assert opts.cpp_output
|
||||
processor = Processor({cls.name: cls for cls in schema.load(opts.schema).classes}, opts.trap_affix)
|
||||
out = opts.cpp_output
|
||||
renderer.render(cpp.ClassList(processor.get_classes(), opts.cpp_namespace, opts.trap_affix,
|
||||
opts.cpp_include_dir, opts.schema), out / f"{opts.trap_affix}Classes.h")
|
||||
|
||||
|
||||
tags = ("cpp", "schema")
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run()
|
||||
|
|
|
@ -4,7 +4,6 @@ import pathlib
|
|||
import inflection
|
||||
|
||||
from swift.codegen.lib import schema
|
||||
from swift.codegen.generators import generator
|
||||
from swift.codegen.lib.dbscheme import *
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
@ -93,9 +92,3 @@ def generate(opts, renderer):
|
|||
declarations=get_declarations(data))
|
||||
|
||||
renderer.render(dbscheme, out)
|
||||
|
||||
|
||||
tags = ("schema", "dbscheme")
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run()
|
||||
|
|
|
@ -1,33 +0,0 @@
|
|||
""" generator script scaffolding """
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import sys
|
||||
from typing import Set
|
||||
|
||||
from swift.codegen.lib import render, paths
|
||||
from swift.codegen.generators import options
|
||||
|
||||
|
||||
def _parse(tags: Set[str]) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser()
|
||||
for opt in options.get(tags):
|
||||
opt.add_to(parser)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def run(*modules, **kwargs):
|
||||
""" run generation functions in specified in `modules`, or in current module by default
|
||||
"""
|
||||
if modules:
|
||||
if kwargs:
|
||||
opts = argparse.Namespace(**kwargs)
|
||||
else:
|
||||
opts = _parse({t for m in modules for t in m.tags})
|
||||
log_level = logging.DEBUG if opts.verbose else logging.INFO
|
||||
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
|
||||
exe_path = paths.exe_file.relative_to(opts.swift_dir)
|
||||
for m in modules:
|
||||
m.generate(opts, render.Renderer(exe_path))
|
||||
else:
|
||||
run(sys.modules["__main__"], **kwargs)
|
|
@ -1,58 +0,0 @@
|
|||
""" generator options, categorized by tags """
|
||||
|
||||
import argparse
|
||||
import collections
|
||||
import pathlib
|
||||
from typing import Set
|
||||
|
||||
from swift.codegen.lib import paths
|
||||
|
||||
|
||||
def _init_options():
|
||||
Option("--verbose", "-v", action="store_true")
|
||||
Option("--swift-dir", type=_abspath, default=paths.swift_dir)
|
||||
Option("--schema", tags=["schema"], type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
|
||||
Option("--dbscheme", tags=["dbscheme"], type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
|
||||
Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
|
||||
Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
|
||||
Option("--ql-format", tags=["ql"], action="store_true", default=True)
|
||||
Option("--no-ql-format", tags=["ql"], action="store_false", dest="ql_format")
|
||||
Option("--codeql-binary", tags=["ql"], default="codeql")
|
||||
Option("--cpp-output", tags=["cpp"], type=_abspath, required=True)
|
||||
Option("--cpp-namespace", tags=["cpp"], default="codeql")
|
||||
Option("--trap-affix", tags=["cpp"], default="Trap")
|
||||
Option("--cpp-include-dir", tags=["cpp"], default="swift/extractor/trap")
|
||||
|
||||
|
||||
def _abspath(x):
|
||||
return pathlib.Path(x).resolve()
|
||||
|
||||
|
||||
_options = collections.defaultdict(list)
|
||||
|
||||
|
||||
class Option:
|
||||
def __init__(self, *args, tags=None, **kwargs):
|
||||
tags = tags or []
|
||||
self.args = args
|
||||
self.kwargs = kwargs
|
||||
if tags:
|
||||
for t in tags:
|
||||
_options[t].append(self)
|
||||
else:
|
||||
_options["*"].append(self)
|
||||
|
||||
def add_to(self, parser: argparse.ArgumentParser):
|
||||
parser.add_argument(*self.args, **self.kwargs)
|
||||
|
||||
|
||||
_init_options()
|
||||
|
||||
|
||||
def get(tags: Set[str]):
|
||||
""" get options marked by `tags`
|
||||
|
||||
Options tagged by wildcard '*' are always returned
|
||||
"""
|
||||
# use specifically tagged options + those tagged with wildcard *
|
||||
return (o for tag in ('*',) + tuple(tags) for o in _options[tag])
|
|
@ -7,7 +7,6 @@ import subprocess
|
|||
import inflection
|
||||
|
||||
from swift.codegen.lib import schema, ql
|
||||
from swift.codegen.generators import generator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
@ -124,9 +123,3 @@ def generate(opts, renderer):
|
|||
renderer.cleanup(existing)
|
||||
if opts.ql_format:
|
||||
format(opts.codeql_binary, renderer.written)
|
||||
|
||||
|
||||
tags = ("schema", "ql")
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run()
|
||||
|
|
|
@ -6,7 +6,6 @@ import inflection
|
|||
from toposort import toposort_flatten
|
||||
|
||||
from swift.codegen.lib import dbscheme, cpp
|
||||
from swift.codegen.generators import generator
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
@ -57,6 +56,7 @@ def get_trap(t: dbscheme.Table, trap_affix: str):
|
|||
|
||||
|
||||
def generate(opts, renderer):
|
||||
assert opts.cpp_output
|
||||
tag_graph = {}
|
||||
out = opts.cpp_output
|
||||
|
||||
|
@ -81,9 +81,3 @@ def generate(opts, renderer):
|
|||
id=tag,
|
||||
))
|
||||
renderer.render(cpp.TagList(tags, opts.cpp_namespace, opts.dbscheme), out / f"{opts.trap_affix}Tags.h")
|
||||
|
||||
|
||||
tags = ("cpp", "dbscheme")
|
||||
|
||||
if __name__ == "__main__":
|
||||
generator.run()
|
||||
|
|
|
@ -8,19 +8,31 @@ genrule(
|
|||
"generated/TrapEntries.h",
|
||||
"generated/TrapTags.h",
|
||||
"generated/TrapClasses.h",
|
||||
"generated/swift.dbscheme",
|
||||
],
|
||||
cmd = " ".join([
|
||||
"$(location //swift/codegen:cppcodegen)",
|
||||
"$(location //swift/codegen)",
|
||||
"--generate=dbscheme,trap,cpp",
|
||||
"--schema $(location //swift/codegen:schema)",
|
||||
"--dbscheme $(RULEDIR)/generated/swift.dbscheme",
|
||||
"--cpp-include-dir " + package_name(),
|
||||
"--cpp-output $(RULEDIR)/generated",
|
||||
]),
|
||||
exec_tools = ["//swift/codegen:cppcodegen"],
|
||||
exec_tools = ["//swift/codegen"],
|
||||
)
|
||||
|
||||
filegroup(
|
||||
name = "generated_dbscheme",
|
||||
srcs = [":generated/swift.dbscheme"],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
||||
cc_library(
|
||||
name = "trap",
|
||||
hdrs = glob(["*.h"]) + [":cppgen"],
|
||||
hdrs = glob(["*.h"]) + [
|
||||
"generated/TrapEntries.h",
|
||||
"generated/TrapTags.h",
|
||||
"generated/TrapClasses.h",
|
||||
],
|
||||
visibility = ["//visibility:public"],
|
||||
)
|
||||
|
|
Загрузка…
Ссылка в новой задаче