Swift: make C++ code generation more self-contained

This is solving a papercut, where the C++ build was relying on the
local dbscheme file to be up-to-date, even if all the information for
building is actually in `schema.yml`. This made a pure C++ development
cycle with changes to `schema.yml` clumsy, as it required a further
dbscheme generation step.

Now for C++ the dbscheme is generated internally in the build files, and
thus a change in `schema.yml` is reflected immediately in the C++ build.

A `swift/codegen` step for checked in generated code (including the
dbscheme) is still required, but a developer can do it just before
running QL tests or committing, instead of during each C++
recompilation.

Some directory reorganization was also carried out, moving specific
generator modules to a new `generators` python package, and only leaving
the two drivers at the top level.

.github/workflows/swift-codegen.yml

@@ -25,8 +25,7 @@ jobs:
           git diff --exit-code --stat HEAD
       - name: Generate C++ files
         run: |
-          bazel run //swift/codegen:trapgen -- --cpp-output=$PWD/swift-generated-headers
-          bazel run //swift/codegen:cppgen -- --cpp-output=$PWD/swift-generated-headers
+          bazel run //swift/codegen:cppcodegen -- --cpp-output=$PWD/swift-generated-headers
       - uses: actions/upload-artifact@v3
         with:
           name: swift-generated-headers

swift/codegen/BUILD.bazel

@@ -14,41 +14,22 @@ filegroup(
 
 py_binary(
     name = "codegen",
-    srcs = glob(
-        ["*.py"],
-        exclude = ["trapgen.py"],
-    ),
+    srcs = ["codegen.py"],
     visibility = ["//swift/codegen/test:__pkg__"],
-    deps = ["//swift/codegen/lib"],
+    deps = ["//swift/codegen/generators"],
 )
 
 # as opposed to the above, that is meant to only be run with bazel run,
 # we need to be precise with data dependencies of this which is meant be run during build
 py_binary(
-    name = "trapgen",
-    srcs = ["trapgen.py"],
-    data = [
-        "//swift:dbscheme",
-        "//swift/codegen/templates:trap",
-    ],
-    visibility = ["//swift:__subpackages__"],
-    deps = [
-        "//swift/codegen/lib",
-        requirement("toposort"),
-    ],
-)
-
-py_binary(
-    name = "cppgen",
-    srcs = ["cppgen.py"],
+    name = "cppcodegen",
+    srcs = ["cppcodegen.py"],
     data = [
         ":schema",
         ":schema_includes",
         "//swift/codegen/templates:cpp",
+        "//swift/codegen/templates:trap",
     ],
     visibility = ["//swift:__subpackages__"],
-    deps = [
-        "//swift/codegen/lib",
-        requirement("toposort"),
-    ],
+    deps = ["//swift/codegen/generators"],
 )

swift/codegen/codegen.py

@@ -1,9 +1,7 @@
 #!/usr/bin/env python3
 """ Driver script to run all checked in code generation """
 
-from lib import generator
-import dbschemegen
-import qlgen
+from swift.codegen.generators import generator, dbschemegen, qlgen
 
 if __name__ == "__main__":
     generator.run(dbschemegen, qlgen)

swift/codegen/cppcodegen.py

@@ -0,0 +1,7 @@
+#!/usr/bin/env python3
+""" Driver script to run all cpp code generation """
+
+from swift.codegen.generators import generator, dbschemegen, trapgen, cppgen
+
+if __name__ == "__main__":
+    generator.run(dbschemegen, trapgen, cppgen)

swift/codegen/generators/BUILD.bazel

@@ -0,0 +1,11 @@
+load("@swift_codegen_deps//:requirements.bzl", "requirement")
+
+py_library(
+    name = "generators",
+    srcs = glob(["*.py"]),
+    visibility = ["//swift/codegen:__subpackages__"],
+    deps = [
+        "//swift/codegen/lib",
+        requirement("toposort"),
+    ],
+)

swift/codegen/generators/cppgen.py

@@ -4,7 +4,8 @@ from typing import Dict
 import inflection
 from toposort import toposort_flatten
 
-from swift.codegen.lib import cpp, generator, schema
+from swift.codegen.lib import cpp, schema
+from swift.codegen.generators import generator
 
 
 def _get_type(t: str, trap_affix: str) -> str:

swift/codegen/generators/dbschemegen.py

@@ -3,7 +3,8 @@ import pathlib
 
 import inflection
 
-from swift.codegen.lib import paths, schema, generator
+from swift.codegen.lib import schema
+from swift.codegen.generators import generator
 from swift.codegen.lib.dbscheme import *
 
 log = logging.getLogger(__name__)

swift/codegen/generators/generator.py

@@ -5,7 +5,8 @@ import logging
 import sys
 from typing import Set
 
-from . import options, render, paths
+from swift.codegen.lib import render, paths
+from swift.codegen.generators import options
 
 
 def _parse(tags: Set[str]) -> argparse.Namespace:

swift/codegen/generators/options.py

@@ -5,7 +5,7 @@ import collections
 import pathlib
 from typing import Set
 
-from . import paths
+from swift.codegen.lib import paths
 
 
 def _init_options():

swift/codegen/generators/qlgen.py

@@ -6,7 +6,8 @@ import subprocess
 
 import inflection
 
-from swift.codegen.lib import schema, paths, generator, ql
+from swift.codegen.lib import schema, ql
+from swift.codegen.generators import generator
 
 log = logging.getLogger(__name__)
 

swift/codegen/generators/trapgen.py

@@ -5,7 +5,8 @@ import logging
 import inflection
 from toposort import toposort_flatten
 
-from swift.codegen.lib import dbscheme, generator, cpp
+from swift.codegen.lib import dbscheme, cpp
+from swift.codegen.generators import generator
 
 log = logging.getLogger(__name__)
 

swift/codegen/lib/BUILD.bazel

@@ -1,11 +1,12 @@
 load("@swift_codegen_deps//:requirements.bzl", "requirement")
+
 py_library(
     name = "lib",
     srcs = glob(["*.py"]),
+    visibility = ["//swift/codegen:__subpackages__"],
     deps = [
         requirement("pystache"),
         requirement("pyyaml"),
         requirement("inflection"),
     ],
-    visibility = ["//swift/codegen:__subpackages__"],
 )

swift/codegen/test/BUILD.bazel

@@ -17,8 +17,7 @@ py_library(
         srcs = [src],
         deps = [
             ":utils",
-            "//swift/codegen",
-            "//swift/codegen:trapgen",
+            "//swift/codegen/generators",
         ],
     )
     for src in glob(["test_*.py"])

swift/codegen/test/test_cppgen.py

@@ -1,6 +1,6 @@
 import sys
 
-from swift.codegen import cppgen
+from swift.codegen.generators import cppgen
 from swift.codegen.lib import cpp
 from swift.codegen.test.utils import *
 

swift/codegen/test/test_dbschemegen.py

@@ -1,6 +1,6 @@
 import sys
 
-from swift.codegen import dbschemegen
+from swift.codegen.generators import dbschemegen
 from swift.codegen.lib import dbscheme
 from swift.codegen.test.utils import *
 

swift/codegen/test/test_qlgen.py

@@ -1,7 +1,7 @@
 import subprocess
 import sys
 
-from swift.codegen import qlgen
+from swift.codegen.generators import qlgen
 from swift.codegen.lib import ql
 from swift.codegen.test.utils import *
 

swift/codegen/test/test_trapgen.py

@@ -1,6 +1,6 @@
 import sys
 
-from swift.codegen import trapgen
+from swift.codegen.generators import trapgen
 from swift.codegen.lib import cpp, dbscheme
 from swift.codegen.test.utils import *
 

swift/extractor/trap/BUILD.bazel

@@ -1,19 +1,3 @@
-genrule(
-    name = "trapgen",
-    srcs = ["//swift:dbscheme"],
-    outs = [
-        "generated/TrapEntries.h",
-        "generated/TrapTags.h",
-    ],
-    cmd = " ".join([
-        "$(location //swift/codegen:trapgen)",
-        "--dbscheme $<",
-        "--cpp-include-dir " + package_name(),
-        "--cpp-output $(RULEDIR)/generated",
-    ]),
-    exec_tools = ["//swift/codegen:trapgen"],
-)
-
 genrule(
     name = "cppgen",
     srcs = [
@@ -21,22 +5,22 @@ genrule(
         "//swift/codegen:schema_includes",
     ],
     outs = [
+        "generated/TrapEntries.h",
+        "generated/TrapTags.h",
         "generated/TrapClasses.h",
     ],
     cmd = " ".join([
-        "$(location //swift/codegen:cppgen)",
+        "$(location //swift/codegen:cppcodegen)",
         "--schema $(location //swift/codegen:schema)",
+        "--dbscheme $(RULEDIR)/generated/swift.dbscheme",
         "--cpp-include-dir " + package_name(),
         "--cpp-output $(RULEDIR)/generated",
     ]),
-    exec_tools = ["//swift/codegen:cppgen"],
+    exec_tools = ["//swift/codegen:cppcodegen"],
 )
 
 cc_library(
     name = "trap",
-    hdrs = glob(["*.h"]) + [
-        ":trapgen",
-        ":cppgen",
-    ],
+    hdrs = glob(["*.h"]) + [":cppgen"],
     visibility = ["//visibility:public"],
 )