Kotlin: first support for Kotlin extractor build

.bazelrc

@@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20
 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64
 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor
 
+# emitting jdeps does not work when building the 2.0.0+ kotlin extractor
+build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false
+
 try-import %workspace%/local.bazelrc

MODULE.bazel

@@ -21,6 +21,14 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0")
 bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl")
 bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json")
 bazel_dep(name = "fmt", version = "10.0.0")
+bazel_dep(name = "rules_kotlin", version = "1.9.4")
+
+# we patch `rules_kotlin` to allow passing `-language-version` at a jvm_kt_library level
+single_version_override(
+    module_name = "rules_kotlin",
+    patch_strip = 1,
+    patches = ["//java/kotlin-extractor:rules_kotlin.patch"],
+)
 
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
 pip.parse(
@@ -48,11 +56,23 @@ node.toolchain(
 )
 use_repo(node, "nodejs", "nodejs_toolchains")
 
-lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files")
-
-lfs_files(
-    name = "kotlin_deps",
-    dir = "//java/kotlin-extractor:deps",
+kotlin_extractor_deps = use_extension("//java/kotlin-extractor:extension.bzl", "kotlin_extractor_deps")
+use_repo(
+    kotlin_extractor_deps,
+    "kotlin_extractor_dep_1.4.32",
+    "kotlin_extractor_dep_1.5.0",
+    "kotlin_extractor_dep_1.5.10",
+    "kotlin_extractor_dep_1.5.20",
+    "kotlin_extractor_dep_1.5.30",
+    "kotlin_extractor_dep_1.6.0",
+    "kotlin_extractor_dep_1.6.20",
+    "kotlin_extractor_dep_1.7.0",
+    "kotlin_extractor_dep_1.7.20",
+    "kotlin_extractor_dep_1.8.0",
+    "kotlin_extractor_dep_1.9.0-Beta",
+    "kotlin_extractor_dep_1.9.20-Beta",
+    "kotlin_extractor_dep_2.0.0-Beta4",
+    "kotlin_extractor_dep_2.0.255-SNAPSHOT",
 )
 
 register_toolchains(

java/kotlin-extractor/BUILD.bazel

@@ -0,0 +1,64 @@
+load(
+    "//java/kotlin-extractor:versions.bzl",
+    "VERSIONS",
+    "get_compatilibity_sources",
+    "version_less",
+)
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
+load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options")
+
+py_binary(
+    name = "generate_dbscheme",
+    srcs = ["generate_dbscheme.py"],
+)
+
+genrule(
+    name = "generated-dbscheme",
+    srcs = ["//java:dbscheme"],
+    outs = ["KotlinExtractorDbScheme.kt"],
+    cmd = "$(execpath :generate_dbscheme) $< $@",
+    tools = [":generate_dbscheme"],
+)
+
+kt_javac_options(
+    name = "javac-options",
+    warn = "off",
+)
+
+[
+    (
+        kt_kotlinc_options(
+            name = "kotlinc-options-%s" % v,
+            include_stdlibs = "none",
+            jvm_target = "1.8",
+            language_version = v[:3],
+            warn = "error",
+            x_optin = [
+                "kotlin.RequiresOptIn",
+                "org.jetbrains.kotlin.ir.symbols.%s" %
+                ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"),
+            ],
+            x_suppress_version_warnings = True,
+        ),
+        kt_jvm_library(
+            name = "kotlin-extractor-%s" % v,
+            srcs =
+                [":generated-dbscheme"] +
+                glob(
+                    [
+                        "src/**/*.kt",
+                        "src/**/*.java",
+                    ],
+                    exclude = ["src/main/kotlin/utils/versions/**"],
+                ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"),
+            javac_opts = ":javac-options",
+            kotlinc_opts = ":kotlinc-options-%s" % v,
+            module_name = "codeql-kotlin-extractor",
+            deps = [
+                "@kotlin_extractor_dep_%s//:kotlin-compiler" % v,
+                "@kotlin_extractor_dep_%s//:kotlin-stdlib" % v,
+            ],
+        ),
+    )
+    for v in VERSIONS
+]

java/kotlin-extractor/deps/empty.jar

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85
+size 22

java/kotlin-extractor/extension.bzl

@@ -0,0 +1,63 @@
+load("//java/kotlin-extractor:versions.bzl", "VERSIONS")
+load("//misc/bazel:lfs.bzl", "lfs_smudge")
+
+_kotlin_dep_build = """
+load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import")
+
+package(default_visibility = ["//visibility:public"])
+
+kt_jvm_import(
+    name = "kotlin-compiler",
+    jar = "kotlin-compiler-{version}.jar",
+)
+
+kt_jvm_import(
+    name = "kotlin-compiler-embeddable",
+    jar = "kotlin-compiler-embeddable-{version}.jar",
+)
+
+kt_jvm_import(
+    name = "kotlin-stdlib",
+    jar = "kotlin-stdlib-{version}.jar",
+)
+"""
+
+def _kotlin_dep_impl(repository_ctx):
+    _, sep, version = repository_ctx.name.rpartition("_")
+    if not sep:
+        fail("rule @%s malformed, name should be <prefix>_<kotlin version>")
+
+    sources = [
+        #        "empty.jar",
+        "kotlin-compiler-%s.jar" % version,
+        "kotlin-compiler-embeddable-%s.jar" % version,
+        "kotlin-stdlib-%s.jar" % version,
+    ]
+    sources = [repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % p)) for p in sources]
+    lfs_smudge(repository_ctx, sources)
+
+    # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings
+    for jar in (
+        "annotations-13.0.jar",
+        "kotlin-stdlib.jar",
+        "kotlin-reflect.jar",
+        "kotlin-script-runtime.jar",
+        "trove4j.jar",
+    ):
+        repository_ctx.symlink("empty.jar", jar)
+    repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(version = version))
+
+_kotlin_dep = repository_rule(implementation = _kotlin_dep_impl)
+
+def _kotlin_deps_impl(module_ctx):
+    deps = []
+    for v in VERSIONS:
+        dep = "kotlin_extractor_dep_%s" % v
+        _kotlin_dep(name = dep)
+        deps.append(dep)
+    return module_ctx.extension_metadata(
+        root_module_direct_deps = deps,
+        root_module_direct_dev_deps = [],
+    )
+
+kotlin_extractor_deps = module_extension(implementation = _kotlin_deps_impl)

java/kotlin-extractor/generate_dbscheme.py

@@ -8,6 +8,7 @@ unions = {}
 tables = {}
 
 dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme'
+output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt'
 
 def parse_dbscheme(filename):
     with open(filename, 'r') as f:
@@ -152,7 +153,7 @@ def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = N
     kt.write(')\\n")\n')
     kt.write('}\n')
 
-with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt:
+with open(output, 'w') as kt:
     kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n')
     kt.write('package com.github.codeql\n')
     kt.write('import java.util.Date\n')

java/kotlin-extractor/rules_kotlin.patch

@@ -0,0 +1,32 @@
+diff --git a/src/main/starlark/core/options/opts.kotlinc.bzl b/src/main/starlark/core/options/opts.kotlinc.bzl
+index 9b15fb8..c0ac2cd 100644
+--- a/src/main/starlark/core/options/opts.kotlinc.bzl
++++ b/src/main/starlark/core/options/opts.kotlinc.bzl
+@@ -28,6 +28,11 @@ def _map_jvm_target_to_flag(version):
+         return None
+     return ["-jvm-target=%s" % version]
+ 
++def _map_language_version_to_flag(version):
++    if not version:
++        return None
++    return ["-language-version=%s" % version, "-api-version=%s" % version]
++
+ _KOPTS_ALL = {
+     "warn": struct(
+         args = dict(
+@@ -349,6 +354,15 @@ _KOPTS_ALL = {
+         value_to_flag = None,
+         map_value_to_flag = _map_jvm_target_to_flag,
+     ),
++    "language_version": struct(
++        args = dict(
++            default = "1.9",
++            doc = "-language-version",
++        ),
++        type = attr.string,
++        value_to_flag = None,
++        map_value_to_flag = _map_language_version_to_flag,
++    ),
+ }
+ 
+ # Filters out options that are not available in current compiler release

java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java

@@ -547,6 +547,16 @@ public class OdasaOutput {
 			}
 		}
 
+        @Override
+        public int hashCode() {
+            int hash = 7;
+            hash = 31 * hash + majorVersion;
+            hash = 31 * hash + minorVersion;
+            hash = 31 * hash + (int)lastModified;
+            hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode());
+            return hash;
+        }
+
 		private boolean newerThan(TrapClassVersion tcv) {
 			// Classes being compiled from source have major version 0 but should take precedence
 			// over any classes with the same qualified name loaded from the classpath

java/kotlin-extractor/versions.bzl

@@ -0,0 +1,49 @@
+VERSIONS = [
+    "1.4.32",
+    "1.5.0",
+    "1.5.10",
+    "1.5.20",
+    "1.5.30",
+    "1.6.0",
+    "1.6.20",
+    "1.7.0",
+    "1.7.20",
+    "1.8.0",
+    "1.9.0-Beta",
+    "1.9.20-Beta",
+    "2.0.0-Beta4",
+    "2.0.255-SNAPSHOT",
+]
+
+def _version_to_tuple(v):
+    v, _, tail = v.partition("-")
+    v = tuple([int(x) for x in v.split(".")])
+    return v + (tail,)
+
+def _tuple_to_version(t):
+    ret = ".".join([str(x) for x in t[:3]])
+    if t[3]:
+        ret += "-" + t[3]
+    return ret
+
+def version_less(lhs, rhs):
+    return _version_to_tuple(lhs) < _version_to_tuple(rhs)
+
+def _basename(path):
+    if "/" not in path:
+        return path
+    return path[path.rindex("/") + 1:]
+
+def get_compatilibity_sources(version, dir):
+    prefix = "%s/v_" % dir
+    available = native.glob(["%s*" % prefix], exclude_directories = 0)
+
+    # we want files with the same base name to replace ones for previous versions, hence the map
+    srcs = {}
+    for d in available:
+        compat_version = d[len(prefix):].replace("_", ".")
+        if version_less(version, compat_version):
+            break
+        files = native.glob(["%s/*.kt" % d])
+        srcs |= {_basename(f): f for f in files}
+    return srcs.values()

misc/bazel/lfs.bzl

@@ -1,4 +1,4 @@
-def _smudge(repository_ctx, srcs):
+def lfs_smudge(repository_ctx, srcs):
     for src in srcs:
         repository_ctx.watch(src)
     script = Label("//misc/bazel/internal:git_lfs_smudge.py")
@@ -14,7 +14,7 @@ def _download_and_extract_lfs(repository_ctx):
     src = repository_ctx.path(attr.src)
     if attr.build_file_content and attr.build_file:
         fail("You should specify only one among build_file_content and build_file for rule @%s" % repository_ctx.name)
-    _smudge(repository_ctx, [src])
+    lfs_smudge(repository_ctx, [src])
     repository_ctx.extract(src.basename, stripPrefix = attr.strip_prefix)
     repository_ctx.delete(src.basename)
     if attr.build_file_content:
@@ -33,7 +33,7 @@ def _download_lfs(repository_ctx):
         if not dir.is_dir:
             fail("`dir` not a directory in @%s" % repository_ctx.name)
         srcs = [f for f in dir.readdir() if not f.is_dir]
-    _smudge(repository_ctx, srcs)
+    lfs_smudge(repository_ctx, srcs)
 
     # with bzlmod the name is qualified with `~` separators, and we want the base name here
     name = repository_ctx.name.split("~")[-1]