github
/
codeql
зеркало из https://github.com/github/codeql.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Merge pull request #17350 from hvitved/tree-sitter-0.23 

Bump `tree-sitter` to `0.23.0`
This commit is contained in:
Tom Hvitved 2024-09-04 13:45:14 +02:00 коммит произвёл GitHub
Родитель 05ffb47711 eb1b2a5594
Коммит ce2df04854
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: B5690EEEBB952194
21 изменённых файлов: 283 добавлений и 200 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

56
ql/Cargo.lock сгенерированный
Просмотреть файл

@ -121,9 +121,12 @@ dependencies = [
[[package]]
name = "cc"
version = "1.0.98"
version = "1.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
@ -700,14 +703,14 @@ dependencies = [
[[package]]
name = "regex"
version = "1.10.4"
version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick 1.0.1",
"memchr",
"regex-automata 0.4.4",
"regex-syntax 0.8.2",
"regex-syntax 0.8.4",
]
[[package]]
@ -727,7 +730,7 @@ checksum = "3b7fa1134405e2ec9353fd416b17f8dacd46c473d7d3fd1cf202706a14eb792a"
dependencies = [
"aho-corasick 1.0.1",
"memchr",
"regex-syntax 0.8.2",
"regex-syntax 0.8.4",
]
[[package]]
@ -738,9 +741,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.2"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "rustix"
@ -814,6 +817,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "smallvec"
version = "1.10.0"
@ -930,12 +939,14 @@ dependencies = [
[[package]]
name = "tree-sitter"
version = "0.22.6"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df7cc499ceadd4dcdf7ec6d4cbc34ece92c3fa07821e287aedecd4416c516dca"
checksum = "20f4cd3642c47a85052a887d86704f4eac272969f61b686bdd3f772122aabaff"
dependencies = [
"cc",
"regex",
"regex-syntax 0.8.4",
"tree-sitter-language",
]
[[package]]
@ -944,33 +955,40 @@ version = "0.0.1"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-json"
version = "0.21.0"
source = "git+https://github.com/tree-sitter/tree-sitter-json.git?rev=94f5c527b2965465956c2000ed6134dd24daf2a7#94f5c527b2965465956c2000ed6134dd24daf2a7"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-json.git?rev=8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571#8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-language"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2545046bd1473dac6c626659cc2567c6c0ff302fc8b84a56c4243378276f7f57"
[[package]]
name = "tree-sitter-ql"
version = "0.22.5"
source = "git+https://github.com/tree-sitter/tree-sitter-ql.git?rev=fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6#fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-ql.git?rev=c73c31c89cb0019ef56fe8bc1723e7c36e0be607#c73c31c89cb0019ef56fe8bc1723e7c36e0be607"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-ql-dbscheme"
version = "0.0.1"
source = "git+https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git?rev=5f770f57fa415607ff50e3d237d47c8f11440eb3#5f770f57fa415607ff50e3d237d47c8f11440eb3"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git?rev=1980b4b6998a1138d326f863e6168f0f2c0c544d#1980b4b6998a1138d326f863e6168f0f2c0c544d"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]

2
ql/buramu/Cargo.toml
Просмотреть файл

@ -1,7 +1,7 @@
[package]
name = "buramu"
version = "0.1.0"
edition = "2018"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html

8
ql/buramu/tree-sitter-blame/Cargo.toml
Просмотреть файл

@ -5,7 +5,7 @@ version = "0.0.1"
keywords = ["incremental", "parsing", "blame"]
categories = ["parsing", "text-editors"]
repository = "https://github.com/tree-sitter/tree-sitter-blame"
edition = "2018"
edition = "2021"
license = "MIT"
build = "bindings/rust/build.rs"
@ -20,10 +20,8 @@ include = [
path = "bindings/rust/lib.rs"
[dependencies]
tree-sitter = ">= 0.22.6"
tree-sitter-language = "0.1.0"
tree-sitter = ">= 0.23"
[build-dependencies]
cc = "1.0"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

23
ql/buramu/tree-sitter-blame/binding.gyp
Просмотреть файл

@ -2,18 +2,29 @@
"targets": [
{
"target_name": "tree_sitter_blame_binding",
"dependencies": [
"<!(node -p \"require('node-addon-api').targets\"):node_addon_api_except",
],
"include_dirs": [
"<!(node -e \"require('nan')\")",
"src"
"src",
],
"sources": [
"bindings/node/binding.cc",
"src/parser.c",
# If your language uses an external scanner, add it here.
# NOTE: if your language has an external scanner, add it here.
],
"conditions": [
["OS!='win'", {
"cflags_c": [
"-std=c11",
],
}, { # OS == "win"
"cflags_c": [
"/std:c11",
"/utf-8",
],
}],
],
"cflags_c": [
"-std=c99",
]
}
]
}

36
ql/buramu/tree-sitter-blame/bindings/node/binding.cc
Просмотреть файл

@ -1,28 +1,20 @@
#include "tree_sitter/parser.h"
#include <node.h>
#include "nan.h"
#include <napi.h>
using namespace v8;
typedef struct TSLanguage TSLanguage;
extern "C" TSLanguage * tree_sitter_blame();
extern "C" TSLanguage *tree_sitter_blame();
namespace {
// "tree-sitter", "language" hashed with BLAKE2
const napi_type_tag LANGUAGE_TYPE_TAG = {
0x8AF2E5212AD58ABF, 0xD5006CAD83ABBA16
};
NAN_METHOD(New) {}
void Init(Local<Object> exports, Local<Object> module) {
Local<FunctionTemplate> tpl = Nan::New<FunctionTemplate>(New);
tpl->SetClassName(Nan::New("Language").ToLocalChecked());
tpl->InstanceTemplate()->SetInternalFieldCount(1);
Local<Function> constructor = Nan::GetFunction(tpl).ToLocalChecked();
Local<Object> instance = constructor->NewInstance(Nan::GetCurrentContext()).ToLocalChecked();
Nan::SetInternalFieldPointer(instance, 0, tree_sitter_blame());
Nan::Set(instance, Nan::New("name").ToLocalChecked(), Nan::New("blame").ToLocalChecked());
Nan::Set(module, Nan::New("exports").ToLocalChecked(), instance);
Napi::Object Init(Napi::Env env, Napi::Object exports) {
exports["name"] = Napi::String::New(env, "blame");
auto language = Napi::External<TSLanguage>::New(env, tree_sitter_blame());
language.TypeTag(&LANGUAGE_TYPE_TAG);
exports["language"] = language;
return exports;
}
NODE_MODULE(tree_sitter_blame_binding, Init)
} // namespace
NODE_API_MODULE(tree_sitter_blame_binding, Init)

18
ql/buramu/tree-sitter-blame/bindings/node/index.js
Просмотреть файл

@ -1,18 +1,6 @@
try {
module.exports = require("../../build/Release/tree_sitter_blame_binding");
} catch (error1) {
if (error1.code !== 'MODULE_NOT_FOUND') {
throw error1;
}
try {
module.exports = require("../../build/Debug/tree_sitter_blame_binding");
} catch (error2) {
if (error2.code !== 'MODULE_NOT_FOUND') {
throw error2;
}
throw error1
}
}
const root = require("path").join(__dirname, "..", "..");
module.exports = require("node-gyp-build")(root);
try {
module.exports.nodeTypeInfo = require("../../src/node-types.json");

3
ql/buramu/tree-sitter-blame/bindings/rust/build.rs
Просмотреть файл

@ -7,6 +7,9 @@ fn main() {
.flag_if_supported("-Wno-unused-parameter")
.flag_if_supported("-Wno-unused-but-set-variable")
.flag_if_supported("-Wno-trigraphs");
#[cfg(target_env = "msvc")]
c_config.flag("-utf-8");
let parser_path = src_dir.join("parser.c");
c_config.file(&parser_path);

41
ql/buramu/tree-sitter-blame/bindings/rust/lib.rs
Просмотреть файл

@ -1,13 +1,18 @@
//! This crate provides blame language support for the [tree-sitter][] parsing library.
//! This crate provides Blame language support for the [tree-sitter][] parsing library.
//!
//! Typically, you will use the [language][language func] function to add this language to a
//! tree-sitter [Parser][], and then use the parser to parse some code:
//!
//! ```
//! let code = "";
//! ```ignore
//! let code = r#"
//! "#;
//! let mut parser = tree_sitter::Parser::new();
//! parser.set_language(tree_sitter_blame::language()).expect("Error loading blame grammar");
//! let language = tree_sitter_blame::LANGUAGE;
//! parser
//! .set_language(&language.into())
//! .expect("Error loading Blame parser"); // fails for some reason, so code block is ignored for now
//! let tree = parser.parse(code, None).unwrap();
//! assert!(!tree.root_node().has_error());
//! ```
//!
//! [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
@ -15,30 +20,26 @@
//! [Parser]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Parser.html
//! [tree-sitter]: https://tree-sitter.github.io/
use tree_sitter::Language;
use tree_sitter_language::LanguageFn;
extern "C" {
fn tree_sitter_blame() -> Language;
fn tree_sitter_blame() -> *const ();
}
/// Get the tree-sitter [Language][] for this grammar.
///
/// [Language]: https://docs.rs/tree-sitter/*/tree_sitter/struct.Language.html
pub fn language() -> Language {
unsafe { tree_sitter_blame() }
}
/// The tree-sitter [`LanguageFn`] for this grammar.
pub const LANGUAGE: LanguageFn = unsafe { LanguageFn::from_raw(tree_sitter_blame) };
/// The content of the [`node-types.json`][] file for this grammar.
///
/// [`node-types.json`]: https://tree-sitter.github.io/tree-sitter/using-parsers#static-node-types
pub const NODE_TYPES: &'static str = include_str!("../../src/node-types.json");
pub const NODE_TYPES: &str = include_str!("../../src/node-types.json");
// Uncomment these to include any queries that this grammar contains
// NOTE: uncomment these to include any queries that this grammar contains:
// pub const HIGHLIGHTS_QUERY: &'static str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &'static str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &'static str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &'static str = include_str!("../../queries/tags.scm");
// pub const HIGHLIGHTS_QUERY: &str = include_str!("../../queries/highlights.scm");
// pub const INJECTIONS_QUERY: &str = include_str!("../../queries/injections.scm");
// pub const LOCALS_QUERY: &str = include_str!("../../queries/locals.scm");
// pub const TAGS_QUERY: &str = include_str!("../../queries/tags.scm");
#[cfg(test)]
mod tests {
@ -46,7 +47,7 @@ mod tests {
fn test_can_load_grammar() {
let mut parser = tree_sitter::Parser::new();
parser
.set_language(super::language())
.expect("Error loading blame language");
.set_language(&super::LANGUAGE.into())
.expect("Error loading Blame parser");
}
}

39
ql/buramu/tree-sitter-blame/package.json
Просмотреть файл

@ -3,17 +3,46 @@
"version": "0.0.1",
"description": "blame grammar for tree-sitter",
"main": "bindings/node",
"types": "bindings/node",
"keywords": [
"parsing",
"incremental"
],
"files": [
"grammar.js",
"binding.gyp",
"prebuilds/**",
"bindings/node/*",
"queries/*",
"src/**",
"*.wasm"
],
"dependencies": {
"nan": "^2.12.1"
"node-addon-api": "^8.0.0",
"node-gyp-build": "^4.8.1"
},
"peerDependencies": {
"tree-sitter": "^0.23.0"
},
"peerDependenciesMeta": {
"tree_sitter": {
"optional": true
}
},
"devDependencies": {
"tree-sitter-cli": "^0.20.7"
"tree-sitter-cli": "^0.23.0",
"prebuildify": "^6.0.1"
},
"scripts": {
"test": "tree-sitter test"
}
}
"install": "node-gyp-build",
"prestart": "tree-sitter build --wasm",
"start": "tree-sitter playground",
"test": "node --test bindings/node/*_test.js"
},
"tree-sitter": [
{
"scope": "source.blame",
"injection-regex": "^blame$"
}
]
}

1
ql/buramu/tree-sitter-blame/src/grammar.json
Просмотреть файл

@ -123,4 +123,3 @@
"inline": [],
"supertypes": []
}

76
ql/buramu/tree-sitter-blame/src/parser.c
Просмотреть файл

@ -1,7 +1,6 @@
#include <tree_sitter/parser.h>
#include "tree_sitter/parser.h"
#if defined(__GNUC__) || defined(__clang__)
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wmissing-field-initializers"
#endif
@ -16,7 +15,7 @@
#define MAX_ALIAS_SEQUENCE_LENGTH 4
#define PRODUCTION_ID_COUNT 14
enum {
enum ts_symbol_identifiers {
anon_sym_today_COLON = 1,
anon_sym_file_COLON = 2,
anon_sym_LF = 3,
@ -132,7 +131,7 @@ static const TSSymbolMetadata ts_symbol_metadata[] = {
},
};
enum {
enum ts_field_identifiers {
field_blame_entry = 1,
field_date = 2,
field_file_entry = 3,
@ -242,26 +241,21 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (lookahead == 'f') ADVANCE(18);
if (lookahead == 'l') ADVANCE(10);
if (lookahead == 't') ADVANCE(23);
if (lookahead == '\t' ||
lookahead == '\n' ||
lookahead == '\r' ||
lookahead == ' ') SKIP(0)
if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == ' ') SKIP(0);
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(48);
END_STATE();
case 1:
if (lookahead == '\n') ADVANCE(40);
if (lookahead == '\t' ||
lookahead == '\r' ||
lookahead == ' ') SKIP(1)
if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == ' ') SKIP(1);
END_STATE();
case 2:
if (lookahead == ' ') ADVANCE(39);
END_STATE();
case 3:
if (lookahead == ' ') ADVANCE(43);
if (lookahead == '\t' ||
lookahead == '\n' ||
lookahead == '\r') SKIP(3)
if (('\t' <= lookahead && lookahead <= '\r')) SKIP(3);
if (('-' <= lookahead && lookahead <= '9') ||
('A' <= lookahead && lookahead <= 'Z') ||
lookahead == '_' ||
@ -340,10 +334,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (lookahead == 'y') ADVANCE(7);
END_STATE();
case 28:
if (lookahead == '\t' ||
lookahead == '\n' ||
lookahead == '\r' ||
lookahead == ' ') SKIP(28)
if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == ' ') SKIP(28);
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(35);
END_STATE();
case 29:
@ -371,10 +363,8 @@ static bool ts_lex(TSLexer *lexer, TSStateId state) {
if (eof) ADVANCE(37);
if (lookahead == 'f') ADVANCE(18);
if (lookahead == 'l') ADVANCE(10);
if (lookahead == '\t' ||
lookahead == '\n' ||
lookahead == '\r' ||
lookahead == ' ') SKIP(36)
if (('\t' <= lookahead && lookahead <= '\r') ||
lookahead == ' ') SKIP(36);
if (('0' <= lookahead && lookahead <= '9')) ADVANCE(49);
END_STATE();
case 37:
@ -623,25 +613,25 @@ static const TSParseActionEntry ts_parse_actions[] = {
[0] = {.entry = {.count = 0, .reusable = false}},
[1] = {.entry = {.count = 1, .reusable = false}}, RECOVER(),
[3] = {.entry = {.count = 1, .reusable = true}}, SHIFT(15),
[5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, .production_id = 6),
[5] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 3, 0, 6),
[7] = {.entry = {.count = 1, .reusable = true}}, SHIFT(19),
[9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, .production_id = 8),
[11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, .production_id = 9),
[9] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_file_entry, 4, 0, 8),
[11] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 2, 0, 9),
[13] = {.entry = {.count = 1, .reusable = true}}, SHIFT(11),
[15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10),
[17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, .production_id = 10), SHIFT_REPEAT(19),
[20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, .production_id = 12),
[22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13),
[24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, .production_id = 13), SHIFT_REPEAT(11),
[27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, .production_id = 1),
[15] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10),
[17] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 2, 0, 10), SHIFT_REPEAT(19),
[20] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_entry, 3, 0, 12),
[22] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13),
[24] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 2, 0, 13), SHIFT_REPEAT(11),
[27] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 1, 0, 1),
[29] = {.entry = {.count = 1, .reusable = true}}, SHIFT(17),
[31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, .production_id = 4),
[33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5),
[35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, .production_id = 5), SHIFT_REPEAT(17),
[38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, .production_id = 11),
[40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, .production_id = 7),
[42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, .production_id = 2),
[44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, .production_id = 3),
[31] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym_blame_info, 2, 0, 4),
[33] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5),
[35] = {.entry = {.count = 2, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 2, 0, 5), SHIFT_REPEAT(17),
[38] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_entry_repeat1, 1, 0, 11),
[40] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_file_entry_repeat1, 1, 0, 7),
[42] = {.entry = {.count = 1, .reusable = true}}, REDUCE(sym__today, 2, 0, 2),
[44] = {.entry = {.count = 1, .reusable = true}}, REDUCE(aux_sym_blame_info_repeat1, 1, 0, 3),
[46] = {.entry = {.count = 1, .reusable = true}}, SHIFT(13),
[48] = {.entry = {.count = 1, .reusable = true}}, ACCEPT_INPUT(),
[50] = {.entry = {.count = 1, .reusable = true}}, SHIFT(18),
@ -652,11 +642,15 @@ static const TSParseActionEntry ts_parse_actions[] = {
#ifdef __cplusplus
extern "C" {
#endif
#ifdef _WIN32
#define extern __declspec(dllexport)
#ifdef TREE_SITTER_HIDE_SYMBOLS
#define TS_PUBLIC
#elif defined(_WIN32)
#define TS_PUBLIC __declspec(dllexport)
#else
#define TS_PUBLIC __attribute__((visibility("default")))
#endif
extern const TSLanguage *tree_sitter_blame(void) {
TS_PUBLIC const TSLanguage *tree_sitter_blame(void) {
static const TSLanguage language = {
.version = LANGUAGE_VERSION,
.symbol_count = SYMBOL_COUNT,

68
ql/buramu/tree-sitter-blame/src/tree_sitter/parser.h
Просмотреть файл

@ -13,9 +13,8 @@ extern "C" {
#define ts_builtin_sym_end 0
#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
typedef uint16_t TSStateId;
#ifndef TREE_SITTER_API_H_
typedef uint16_t TSStateId;
typedef uint16_t TSSymbol;
typedef uint16_t TSFieldId;
typedef struct TSLanguage TSLanguage;
@ -48,6 +47,7 @@ struct TSLexer {
uint32_t (*get_column)(TSLexer *);
bool (*is_at_included_range_start)(const TSLexer *);
bool (*eof)(const TSLexer *);
void (*log)(const TSLexer *, const char *, ...);
};
typedef enum {
@ -87,6 +87,11 @@ typedef union {
} entry;
} TSParseActionEntry;
typedef struct {
int32_t start;
int32_t end;
} TSCharacterRange;
struct TSLanguage {
uint32_t version;
uint32_t symbol_count;
@ -126,13 +131,38 @@ struct TSLanguage {
const TSStateId *primary_state_ids;
};
static inline bool set_contains(TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
uint32_t index = 0;
uint32_t size = len - index;
while (size > 1) {
uint32_t half_size = size / 2;
uint32_t mid_index = index + half_size;
TSCharacterRange *range = &ranges[mid_index];
if (lookahead >= range->start && lookahead <= range->end) {
return true;
} else if (lookahead > range->end) {
index = mid_index;
}
size -= half_size;
}
TSCharacterRange *range = &ranges[index];
return (lookahead >= range->start && lookahead <= range->end);
}
/*
* Lexer Macros
*/
#ifdef _MSC_VER
#define UNUSED __pragma(warning(suppress : 4101))
#else
#define UNUSED __attribute__((unused))
#endif
#define START_LEXER() \
bool result = false; \
bool skip = false; \
UNUSED \
bool eof = false; \
int32_t lookahead; \
goto start; \
@ -148,6 +178,17 @@ struct TSLanguage {
goto next_state; \
}
#define ADVANCE_MAP(...) \
{ \
static const uint16_t map[] = { __VA_ARGS__ }; \
for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) { \
if (map[i] == lookahead) { \
state = map[i + 1]; \
goto next_state; \
} \
} \
}
#define SKIP(state_value) \
{ \
skip = true; \
@ -166,7 +207,7 @@ struct TSLanguage {
* Parse Table Macros
*/
#define SMALL_STATE(id) id - LARGE_STATE_COUNT
#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
#define STATE(id) id
@ -176,7 +217,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value \
.state = (state_value) \
} \
}}
@ -184,7 +225,7 @@ struct TSLanguage {
{{ \
.shift = { \
.type = TSParseActionTypeShift, \
.state = state_value, \
.state = (state_value), \
.repetition = true \
} \
}}
@ -197,14 +238,15 @@ struct TSLanguage {
} \
}}
#define REDUCE(symbol_val, child_count_val, ...) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_val, \
.child_count = child_count_val, \
__VA_ARGS__ \
}, \
#define REDUCE(symbol_name, children, precedence, prod_id) \
{{ \
.reduce = { \
.type = TSParseActionTypeReduce, \
.symbol = symbol_name, \
.child_count = children, \
.dynamic_precedence = precedence, \
.production_id = prod_id \
}, \
}}
#define RECOVER() \

11
ql/extractor/Cargo.toml
Просмотреть файл

@ -7,17 +7,14 @@ edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
tree-sitter = ">= 0.22.6"
tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "fa5c3821dd2161f5c8528a8cbdb258daa6dc4de6"}
tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "5f770f57fa415607ff50e3d237d47c8f11440eb3"}
tree-sitter = ">= 0.23.0"
tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql.git", rev = "c73c31c89cb0019ef56fe8bc1723e7c36e0be607"}
tree-sitter-ql-dbscheme = { git = "https://github.com/tree-sitter/tree-sitter-ql-dbscheme.git", rev = "1980b4b6998a1138d326f863e6168f0f2c0c544d"}
tree-sitter-blame = {path = "../buramu/tree-sitter-blame"}
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "94f5c527b2965465956c2000ed6134dd24daf2a7"}
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json.git", rev = "8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571"}
clap = { version = "4.2", features = ["derive"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3.18", features = ["env-filter"] }
rayon = "1.9.0"
regex = "1.10.4"
codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

8
ql/extractor/src/extractor.rs
Просмотреть файл

@ -27,25 +27,25 @@ pub fn run(options: Options) -> std::io::Result<()> {
languages: vec![
simple::LanguageSpec {
prefix: "ql",
ts_language: tree_sitter_ql::language(),
ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.ql".into(), "*.qll".into()],
},
simple::LanguageSpec {
prefix: "dbscheme",
ts_language: tree_sitter_ql_dbscheme::language(),
ts_language: tree_sitter_ql_dbscheme::LANGUAGE.into(),
node_types: tree_sitter_ql_dbscheme::NODE_TYPES,
file_globs: vec!["*.dbscheme".into()],
},
simple::LanguageSpec {
prefix: "json",
ts_language: tree_sitter_json::language(),
ts_language: tree_sitter_json::LANGUAGE.into(),
node_types: tree_sitter_json::NODE_TYPES,
file_globs: vec!["*.json".into(), "*.jsonl".into(), "*.jsonc".into()],
},
simple::LanguageSpec {
prefix: "blame",
ts_language: tree_sitter_blame::language(),
ts_language: tree_sitter_blame::LANGUAGE.into(),
node_types: tree_sitter_blame::NODE_TYPES,
file_globs: vec!["*.blame".into()],
},

64
ruby/extractor/Cargo.lock сгенерированный
Просмотреть файл

@ -105,9 +105,12 @@ checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
[[package]]
name = "cc"
version = "1.0.98"
version = "1.1.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41c270e7540d725e65ac7f1b212ac8ce349719624d7bcff99f8e2e488e8cf03f"
checksum = "57b6a275aa2903740dc87da01c62040406b8812552e97129a63ea8850a17c6e6"
dependencies = [
"shlex",
]
[[package]]
name = "cfg-if"
@ -356,7 +359,7 @@ dependencies = [
"bstr",
"log",
"regex-automata 0.4.6",
"regex-syntax 0.8.3",
"regex-syntax 0.8.4",
]
[[package]]
@ -580,14 +583,14 @@ dependencies = [
[[package]]
name = "regex"
version = "1.10.4"
version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick",
"memchr",
"regex-automata 0.4.6",
"regex-syntax 0.8.3",
"regex-syntax 0.8.4",
]
[[package]]
@ -607,7 +610,7 @@ checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
dependencies = [
"aho-corasick",
"memchr",
"regex-syntax 0.8.3",
"regex-syntax 0.8.4",
]
[[package]]
@ -618,9 +621,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
[[package]]
name = "regex-syntax"
version = "0.8.3"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56"
checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b"
[[package]]
name = "ryu"
@ -668,6 +671,12 @@ dependencies = [
"lazy_static",
]
[[package]]
name = "shlex"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
[[package]]
name = "smallvec"
version = "1.13.2"
@ -764,47 +773,56 @@ dependencies = [
[[package]]
name = "tree-sitter"
version = "0.22.6"
source = "git+https://github.com/redsun82/tree-sitter.git?rev=1f5c1112ceaa8fc6aff61d1852690407670d2a96#1f5c1112ceaa8fc6aff61d1852690407670d2a96"
version = "0.23.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20f4cd3642c47a85052a887d86704f4eac272969f61b686bdd3f772122aabaff"
dependencies = [
"cc",
"regex",
"regex-syntax 0.8.4",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-embedded-template"
version = "0.21.0"
source = "git+https://github.com/tree-sitter/tree-sitter-embedded-template.git?rev=38d5004a797298dc42c85e7706c5ceac46a3f29f#38d5004a797298dc42c85e7706c5ceac46a3f29f"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-embedded-template.git?rev=62b0a6e45900a7dff7c37da95fec20a09968ba52#62b0a6e45900a7dff7c37da95fec20a09968ba52"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-json"
version = "0.21.0"
source = "git+https://github.com/tree-sitter/tree-sitter-json#94f5c527b2965465956c2000ed6134dd24daf2a7"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-json#8bfdb43f47ad805bb1ce093203cfcbaa8ed2c571"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-language"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2545046bd1473dac6c626659cc2567c6c0ff302fc8b84a56c4243378276f7f57"
[[package]]
name = "tree-sitter-ql"
version = "0.22.5"
source = "git+https://github.com/tree-sitter/tree-sitter-ql#42becd6f8f7bae82c818fa3abb1b6ff34b552310"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-ql#c73c31c89cb0019ef56fe8bc1723e7c36e0be607"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]
name = "tree-sitter-ruby"
version = "0.21.0"
source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=0ffe457fb6aabf064f173fd30ea356845cef2513#0ffe457fb6aabf064f173fd30ea356845cef2513"
version = "0.23.0"
source = "git+https://github.com/tree-sitter/tree-sitter-ruby.git?rev=a66579f70d6f50ffd81a16fc3d3358e2ac173c88#a66579f70d6f50ffd81a16fc3d3358e2ac173c88"
dependencies = [
"cc",
"tree-sitter",
"tree-sitter-language",
]
[[package]]

11
ruby/extractor/Cargo.toml
Просмотреть файл

@ -7,20 +7,17 @@ authors = ["GitHub"]
edition = "2021"
[dependencies]
tree-sitter = ">= 0.22.6"
tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "38d5004a797298dc42c85e7706c5ceac46a3f29f" }
tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "0ffe457fb6aabf064f173fd30ea356845cef2513" }
tree-sitter = ">= 0.23.0"
tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template.git", rev = "62b0a6e45900a7dff7c37da95fec20a09968ba52" }
tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "a66579f70d6f50ffd81a16fc3d3358e2ac173c88" }
clap = { version = "4.2", features = ["derive"] }
tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0"
regex = "1.7.1"
encoding = "0.2"
lazy_static = "1.4.0"
lazy_static = "1.4.0"
# Ideally, we'd like to pull this in via a relative path.
# However, our bazel/rust tooling chokes on this, c.f. https://github.com/bazelbuild/rules_rust/issues/1525
# Therefore, we have a pretty bad hack in place instead, see README.md in the codeql-extractor-fake-crate directory.
codeql-extractor = { path = "codeql-extractor-fake-crate" }
[patch.crates-io]
tree-sitter = { git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96" }

4
ruby/extractor/codeql-extractor-fake-crate/Cargo.toml
Просмотреть файл

@ -7,7 +7,7 @@ authors = ["GitHub"]
[dependencies]
flate2 = "1.0"
globset = "0.4"
tree-sitter = ">= 0.22.6"
tree-sitter = ">= 0.23.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0"
@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" }
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" }
rand = "0.8.5"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

4
ruby/extractor/src/extractor.rs
Просмотреть файл

@ -78,8 +78,8 @@ pub fn run(options: Options) -> std::io::Result<()> {
let file_list = fs::File::open(file_paths::path_from_string(&options.file_list))?;
let language = tree_sitter_ruby::language();
let erb = tree_sitter_embedded_template::language();
let language: Language = tree_sitter_ruby::LANGUAGE.into();
let erb: Language = tree_sitter_embedded_template::LANGUAGE.into();
// Look up tree-sitter kind ids now, to avoid string comparisons when scanning ERB files.
let erb_directive_id = erb.id_for_node_kind("directive", true);
let erb_output_directive_id = erb.id_for_node_kind("output_directive", true);

4
shared/tree-sitter-extractor/Cargo.toml
Просмотреть файл

@ -7,7 +7,7 @@ authors = ["GitHub"]
[dependencies]
flate2 = "1.0"
globset = "0.4"
tree-sitter = ">= 0.22.6"
tree-sitter = ">= 0.23.0"
tracing = "0.1"
tracing-subscriber = { version = "0.3.3", features = ["env-filter"] }
rayon = "1.5.0"
@ -24,5 +24,3 @@ tree-sitter-ql = { git = "https://github.com/tree-sitter/tree-sitter-ql" }
tree-sitter-json = {git = "https://github.com/tree-sitter/tree-sitter-json" }
rand = "0.8.5"
[patch.crates-io]
tree-sitter = {git = "https://github.com/redsun82/tree-sitter.git", rev = "1f5c1112ceaa8fc6aff61d1852690407670d2a96"}

2
shared/tree-sitter-extractor/tests/integration_test.rs
Просмотреть файл

@ -13,7 +13,7 @@ use common::{create_source_dir, expect_trap_file, SourceArchive};
fn simple_extractor() {
let language = simple::LanguageSpec {
prefix: "ql",
ts_language: tree_sitter_ql::language(),
ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.qll".into()],
};

4
shared/tree-sitter-extractor/tests/multiple_languages.rs
Просмотреть файл

@ -12,13 +12,13 @@ use common::{create_source_dir, expect_trap_file, SourceArchive};
fn multiple_language_extractor() {
let lang_ql = simple::LanguageSpec {
prefix: "ql",
ts_language: tree_sitter_ql::language(),
ts_language: tree_sitter_ql::LANGUAGE.into(),
node_types: tree_sitter_ql::NODE_TYPES,
file_globs: vec!["*.qll".into()],
};
let lang_json = simple::LanguageSpec {
prefix: "json",
ts_language: tree_sitter_json::language(),
ts_language: tree_sitter_json::LANGUAGE.into(),
node_types: tree_sitter_json::NODE_TYPES,
file_globs: vec!["*.json".into(), "*Jsonfile".into()],
};