From c242466d316c8e029420dea01d1179e975b19b14 Mon Sep 17 00:00:00 2001 From: Paolo Tranquilli Date: Wed, 3 Apr 2024 16:29:16 +0200 Subject: [PATCH] Kotlin: first support for Kotlin extractor build --- .bazelrc | 3 + MODULE.bazel | 30 +++++++-- java/kotlin-extractor/BUILD.bazel | 64 +++++++++++++++++++ java/kotlin-extractor/deps/BUILD.bazel | 0 java/kotlin-extractor/deps/empty.jar | 3 + java/kotlin-extractor/extension.bzl | 63 ++++++++++++++++++ java/kotlin-extractor/generate_dbscheme.py | 3 +- java/kotlin-extractor/rules_kotlin.patch | 32 ++++++++++ .../semmle/extractor/java/OdasaOutput.java | 10 +++ java/kotlin-extractor/versions.bzl | 49 ++++++++++++++ misc/bazel/lfs.bzl | 6 +- 11 files changed, 254 insertions(+), 9 deletions(-) create mode 100644 java/kotlin-extractor/deps/BUILD.bazel create mode 100755 java/kotlin-extractor/deps/empty.jar create mode 100644 java/kotlin-extractor/extension.bzl create mode 100644 java/kotlin-extractor/rules_kotlin.patch create mode 100644 java/kotlin-extractor/versions.bzl diff --git a/.bazelrc b/.bazelrc index 12232b4bbd6..6592f65f83a 100644 --- a/.bazelrc +++ b/.bazelrc @@ -14,4 +14,7 @@ build:linux --cxxopt=-std=c++20 build:macos --cxxopt=-std=c++20 --cpu=darwin_x86_64 build:windows --cxxopt=/std:c++20 --cxxopt=/Zc:preprocessor +# emitting jdeps does not work when building the 2.0.0+ kotlin extractor +build --@rules_kotlin//kotlin/settings:jvm_emit_jdeps=false + try-import %workspace%/local.bazelrc diff --git a/MODULE.bazel b/MODULE.bazel index ad47c834c17..c8c2804c0da 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -21,6 +21,14 @@ bazel_dep(name = "bazel_skylib", version = "1.5.0") bazel_dep(name = "abseil-cpp", version = "20240116.0", repo_name = "absl") bazel_dep(name = "nlohmann_json", version = "3.11.3", repo_name = "json") bazel_dep(name = "fmt", version = "10.0.0") +bazel_dep(name = "rules_kotlin", version = "1.9.4") + +# we patch `rules_kotlin` to allow passing `-language-version` at a jvm_kt_library level +single_version_override( + module_name = "rules_kotlin", + patch_strip = 1, + patches = ["//java/kotlin-extractor:rules_kotlin.patch"], +) pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( @@ -48,11 +56,23 @@ node.toolchain( ) use_repo(node, "nodejs", "nodejs_toolchains") -lfs_files = use_repo_rule("//misc/bazel:lfs.bzl", "lfs_files") - -lfs_files( - name = "kotlin_deps", - dir = "//java/kotlin-extractor:deps", +kotlin_extractor_deps = use_extension("//java/kotlin-extractor:extension.bzl", "kotlin_extractor_deps") +use_repo( + kotlin_extractor_deps, + "kotlin_extractor_dep_1.4.32", + "kotlin_extractor_dep_1.5.0", + "kotlin_extractor_dep_1.5.10", + "kotlin_extractor_dep_1.5.20", + "kotlin_extractor_dep_1.5.30", + "kotlin_extractor_dep_1.6.0", + "kotlin_extractor_dep_1.6.20", + "kotlin_extractor_dep_1.7.0", + "kotlin_extractor_dep_1.7.20", + "kotlin_extractor_dep_1.8.0", + "kotlin_extractor_dep_1.9.0-Beta", + "kotlin_extractor_dep_1.9.20-Beta", + "kotlin_extractor_dep_2.0.0-Beta4", + "kotlin_extractor_dep_2.0.255-SNAPSHOT", ) register_toolchains( diff --git a/java/kotlin-extractor/BUILD.bazel b/java/kotlin-extractor/BUILD.bazel index e69de29bb2d..b5162410499 100644 --- a/java/kotlin-extractor/BUILD.bazel +++ b/java/kotlin-extractor/BUILD.bazel @@ -0,0 +1,64 @@ +load( + "//java/kotlin-extractor:versions.bzl", + "VERSIONS", + "get_compatilibity_sources", + "version_less", +) +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") +load("@rules_kotlin//kotlin:core.bzl", "kt_javac_options", "kt_kotlinc_options") + +py_binary( + name = "generate_dbscheme", + srcs = ["generate_dbscheme.py"], +) + +genrule( + name = "generated-dbscheme", + srcs = ["//java:dbscheme"], + outs = ["KotlinExtractorDbScheme.kt"], + cmd = "$(execpath :generate_dbscheme) $< $@", + tools = [":generate_dbscheme"], +) + +kt_javac_options( + name = "javac-options", + warn = "off", +) + +[ + ( + kt_kotlinc_options( + name = "kotlinc-options-%s" % v, + include_stdlibs = "none", + jvm_target = "1.8", + language_version = v[:3], + warn = "error", + x_optin = [ + "kotlin.RequiresOptIn", + "org.jetbrains.kotlin.ir.symbols.%s" % + ("IrSymbolInternals" if version_less(v, "2.0.0") else "UnsafeDuringIrConstructionAPI"), + ], + x_suppress_version_warnings = True, + ), + kt_jvm_library( + name = "kotlin-extractor-%s" % v, + srcs = + [":generated-dbscheme"] + + glob( + [ + "src/**/*.kt", + "src/**/*.java", + ], + exclude = ["src/main/kotlin/utils/versions/**"], + ) + get_compatilibity_sources(v, "src/main/kotlin/utils/versions"), + javac_opts = ":javac-options", + kotlinc_opts = ":kotlinc-options-%s" % v, + module_name = "codeql-kotlin-extractor", + deps = [ + "@kotlin_extractor_dep_%s//:kotlin-compiler" % v, + "@kotlin_extractor_dep_%s//:kotlin-stdlib" % v, + ], + ), + ) + for v in VERSIONS +] diff --git a/java/kotlin-extractor/deps/BUILD.bazel b/java/kotlin-extractor/deps/BUILD.bazel new file mode 100644 index 00000000000..e69de29bb2d diff --git a/java/kotlin-extractor/deps/empty.jar b/java/kotlin-extractor/deps/empty.jar new file mode 100755 index 00000000000..fe3604a050f --- /dev/null +++ b/java/kotlin-extractor/deps/empty.jar @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8739c76e681f900923b900c9df0ef75cf421d39cabb54650c4b9ad19b6a76d85 +size 22 diff --git a/java/kotlin-extractor/extension.bzl b/java/kotlin-extractor/extension.bzl new file mode 100644 index 00000000000..8034a176871 --- /dev/null +++ b/java/kotlin-extractor/extension.bzl @@ -0,0 +1,63 @@ +load("//java/kotlin-extractor:versions.bzl", "VERSIONS") +load("//misc/bazel:lfs.bzl", "lfs_smudge") + +_kotlin_dep_build = """ +load("@rules_kotlin//kotlin:jvm.bzl", "kt_jvm_import") + +package(default_visibility = ["//visibility:public"]) + +kt_jvm_import( + name = "kotlin-compiler", + jar = "kotlin-compiler-{version}.jar", +) + +kt_jvm_import( + name = "kotlin-compiler-embeddable", + jar = "kotlin-compiler-embeddable-{version}.jar", +) + +kt_jvm_import( + name = "kotlin-stdlib", + jar = "kotlin-stdlib-{version}.jar", +) +""" + +def _kotlin_dep_impl(repository_ctx): + _, sep, version = repository_ctx.name.rpartition("_") + if not sep: + fail("rule @%s malformed, name should be _") + + sources = [ + # "empty.jar", + "kotlin-compiler-%s.jar" % version, + "kotlin-compiler-embeddable-%s.jar" % version, + "kotlin-stdlib-%s.jar" % version, + ] + sources = [repository_ctx.path(Label("//java/kotlin-extractor/deps:%s" % p)) for p in sources] + lfs_smudge(repository_ctx, sources) + + # for some reason rules_kotlin warns about these jars missing, this is to silence those warnings + for jar in ( + "annotations-13.0.jar", + "kotlin-stdlib.jar", + "kotlin-reflect.jar", + "kotlin-script-runtime.jar", + "trove4j.jar", + ): + repository_ctx.symlink("empty.jar", jar) + repository_ctx.file("BUILD.bazel", _kotlin_dep_build.format(version = version)) + +_kotlin_dep = repository_rule(implementation = _kotlin_dep_impl) + +def _kotlin_deps_impl(module_ctx): + deps = [] + for v in VERSIONS: + dep = "kotlin_extractor_dep_%s" % v + _kotlin_dep(name = dep) + deps.append(dep) + return module_ctx.extension_metadata( + root_module_direct_deps = deps, + root_module_direct_dev_deps = [], + ) + +kotlin_extractor_deps = module_extension(implementation = _kotlin_deps_impl) diff --git a/java/kotlin-extractor/generate_dbscheme.py b/java/kotlin-extractor/generate_dbscheme.py index fb891fb105c..be0c5622ed1 100755 --- a/java/kotlin-extractor/generate_dbscheme.py +++ b/java/kotlin-extractor/generate_dbscheme.py @@ -8,6 +8,7 @@ unions = {} tables = {} dbscheme = sys.argv[1] if len(sys.argv) >= 2 else '../ql/lib/config/semmlecode.dbscheme' +output = sys.argv[2] if len(sys.argv) >= 3 else 'src/main/kotlin/KotlinExtractorDbScheme.kt' def parse_dbscheme(filename): with open(filename, 'r') as f: @@ -152,7 +153,7 @@ def genTable(kt, relname, columns, enum = None, kind = None, num = None, typ = N kt.write(')\\n")\n') kt.write('}\n') -with open('src/main/kotlin/KotlinExtractorDbScheme.kt', 'w') as kt: +with open(output, 'w') as kt: kt.write('/* Generated by ' + sys.argv[0] + ': Do not edit manually. */\n') kt.write('package com.github.codeql\n') kt.write('import java.util.Date\n') diff --git a/java/kotlin-extractor/rules_kotlin.patch b/java/kotlin-extractor/rules_kotlin.patch new file mode 100644 index 00000000000..3bc9ff4aac2 --- /dev/null +++ b/java/kotlin-extractor/rules_kotlin.patch @@ -0,0 +1,32 @@ +diff --git a/src/main/starlark/core/options/opts.kotlinc.bzl b/src/main/starlark/core/options/opts.kotlinc.bzl +index 9b15fb8..c0ac2cd 100644 +--- a/src/main/starlark/core/options/opts.kotlinc.bzl ++++ b/src/main/starlark/core/options/opts.kotlinc.bzl +@@ -28,6 +28,11 @@ def _map_jvm_target_to_flag(version): + return None + return ["-jvm-target=%s" % version] + ++def _map_language_version_to_flag(version): ++ if not version: ++ return None ++ return ["-language-version=%s" % version, "-api-version=%s" % version] ++ + _KOPTS_ALL = { + "warn": struct( + args = dict( +@@ -349,6 +354,15 @@ _KOPTS_ALL = { + value_to_flag = None, + map_value_to_flag = _map_jvm_target_to_flag, + ), ++ "language_version": struct( ++ args = dict( ++ default = "1.9", ++ doc = "-language-version", ++ ), ++ type = attr.string, ++ value_to_flag = None, ++ map_value_to_flag = _map_language_version_to_flag, ++ ), + } + + # Filters out options that are not available in current compiler release diff --git a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java index 830b2012c98..4bb30527fb8 100644 --- a/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java +++ b/java/kotlin-extractor/src/main/java/com/semmle/extractor/java/OdasaOutput.java @@ -547,6 +547,16 @@ public class OdasaOutput { } } + @Override + public int hashCode() { + int hash = 7; + hash = 31 * hash + majorVersion; + hash = 31 * hash + minorVersion; + hash = 31 * hash + (int)lastModified; + hash = 31 * hash + (extractorName == null ? 0 : extractorName.hashCode()); + return hash; + } + private boolean newerThan(TrapClassVersion tcv) { // Classes being compiled from source have major version 0 but should take precedence // over any classes with the same qualified name loaded from the classpath diff --git a/java/kotlin-extractor/versions.bzl b/java/kotlin-extractor/versions.bzl new file mode 100644 index 00000000000..473825682c1 --- /dev/null +++ b/java/kotlin-extractor/versions.bzl @@ -0,0 +1,49 @@ +VERSIONS = [ + "1.4.32", + "1.5.0", + "1.5.10", + "1.5.20", + "1.5.30", + "1.6.0", + "1.6.20", + "1.7.0", + "1.7.20", + "1.8.0", + "1.9.0-Beta", + "1.9.20-Beta", + "2.0.0-Beta4", + "2.0.255-SNAPSHOT", +] + +def _version_to_tuple(v): + v, _, tail = v.partition("-") + v = tuple([int(x) for x in v.split(".")]) + return v + (tail,) + +def _tuple_to_version(t): + ret = ".".join([str(x) for x in t[:3]]) + if t[3]: + ret += "-" + t[3] + return ret + +def version_less(lhs, rhs): + return _version_to_tuple(lhs) < _version_to_tuple(rhs) + +def _basename(path): + if "/" not in path: + return path + return path[path.rindex("/") + 1:] + +def get_compatilibity_sources(version, dir): + prefix = "%s/v_" % dir + available = native.glob(["%s*" % prefix], exclude_directories = 0) + + # we want files with the same base name to replace ones for previous versions, hence the map + srcs = {} + for d in available: + compat_version = d[len(prefix):].replace("_", ".") + if version_less(version, compat_version): + break + files = native.glob(["%s/*.kt" % d]) + srcs |= {_basename(f): f for f in files} + return srcs.values() diff --git a/misc/bazel/lfs.bzl b/misc/bazel/lfs.bzl index a5559d55a74..b58831f72ea 100644 --- a/misc/bazel/lfs.bzl +++ b/misc/bazel/lfs.bzl @@ -1,4 +1,4 @@ -def _smudge(repository_ctx, srcs): +def lfs_smudge(repository_ctx, srcs): for src in srcs: repository_ctx.watch(src) script = Label("//misc/bazel/internal:git_lfs_smudge.py") @@ -14,7 +14,7 @@ def _download_and_extract_lfs(repository_ctx): src = repository_ctx.path(attr.src) if attr.build_file_content and attr.build_file: fail("You should specify only one among build_file_content and build_file for rule @%s" % repository_ctx.name) - _smudge(repository_ctx, [src]) + lfs_smudge(repository_ctx, [src]) repository_ctx.extract(src.basename, stripPrefix = attr.strip_prefix) repository_ctx.delete(src.basename) if attr.build_file_content: @@ -33,7 +33,7 @@ def _download_lfs(repository_ctx): if not dir.is_dir: fail("`dir` not a directory in @%s" % repository_ctx.name) srcs = [f for f in dir.readdir() if not f.is_dir] - _smudge(repository_ctx, srcs) + lfs_smudge(repository_ctx, srcs) # with bzlmod the name is qualified with `~` separators, and we want the base name here name = repository_ctx.name.split("~")[-1]