Merge pull request #17491 from github/redsun82/rust-integration-test

Rust: add basic integration tests and fix archiving on Windows

Cargo.lock

@@ -265,7 +265,7 @@ dependencies = [
  "chalk-ir",
  "ena",
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "petgraph",
  "rustc-hash",
  "tracing",
@@ -374,6 +374,7 @@ dependencies = [
  "clap",
  "codeql-extractor",
  "figment",
+ "itertools 0.13.0",
  "log",
  "num-traits",
  "ra_ap_base_db",
@@ -857,6 +858,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.13.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -1333,7 +1343,7 @@ checksum = "c7c38520eb4770af561c34b908431f4e548c3282093cf3daf3c6e566d99a2937"
 dependencies = [
  "arrayvec",
  "either",
- "itertools",
+ "itertools 0.12.1",
  "ra_ap_base_db",
  "ra_ap_cfg",
  "ra_ap_hir_def",
@@ -1365,7 +1375,7 @@ dependencies = [
  "fst",
  "hashbrown 0.14.5",
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "la-arena",
  "ra-ap-rustc_abi",
  "ra-ap-rustc_parse_format",
@@ -1395,7 +1405,7 @@ dependencies = [
  "cov-mark",
  "either",
  "hashbrown 0.14.5",
- "itertools",
+ "itertools 0.12.1",
  "la-arena",
  "ra_ap_base_db",
  "ra_ap_cfg",
@@ -1430,7 +1440,7 @@ dependencies = [
  "either",
  "ena",
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "la-arena",
  "nohash-hasher",
  "oorandom",
@@ -1467,7 +1477,7 @@ dependencies = [
  "either",
  "fst",
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "line-index",
  "memchr",
  "nohash-hasher",
@@ -1513,7 +1523,7 @@ checksum = "82e6f24b61f1ef1f3a756493d1fb7e711b69b2e4d5f4746fcb959313dfd41471"
 dependencies = [
  "anyhow",
  "crossbeam-channel",
- "itertools",
+ "itertools 0.12.1",
  "ra_ap_hir_expand",
  "ra_ap_ide_db",
  "ra_ap_intern",
@@ -1608,7 +1618,7 @@ checksum = "db83d1844c74b22c110c4b8e8f2519be2b1723964008527281a11c3398749756"
 dependencies = [
  "anyhow",
  "cargo_metadata",
- "itertools",
+ "itertools 0.12.1",
  "la-arena",
  "ra_ap_base_db",
  "ra_ap_cfg",
@@ -1632,7 +1642,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "370b302873eeafd07ccc6a714fc9395cae11e385955ccb78081093ee3b86f94e"
 dependencies = [
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "lock_api",
  "oorandom",
  "parking_lot",
@@ -1679,7 +1689,7 @@ checksum = "bb63ff9d6b11b4553fc0835f16705975258905e3b1230fcf1ddbf24c46aff69d"
 dependencies = [
  "always-assert",
  "crossbeam-channel",
- "itertools",
+ "itertools 0.12.1",
  "jod-thread",
  "libc",
  "miow",
@@ -1695,7 +1705,7 @@ dependencies = [
  "cov-mark",
  "either",
  "indexmap 2.5.0",
- "itertools",
+ "itertools 0.12.1",
  "ra-ap-rustc_lexer",
  "ra_ap_parser",
  "ra_ap_stdx",
@@ -1729,7 +1739,7 @@ version = "0.0.232"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "7cb72ee1901baec556f4f2ef77e287d749ac0e973f063990672d6207b076aeac"
 dependencies = [
- "itertools",
+ "itertools 0.12.1",
  "text-size",
 ]
 

rust/extractor/Cargo.toml

@@ -25,3 +25,4 @@ triomphe = "0.1.13"
 argfile = "0.2.1"
 codeql-extractor = { path = "../../shared/tree-sitter-extractor" }
 rust-extractor-macros = { path = "macros" }
+itertools = "0.13.0"

rust/extractor/src/archive.rs

@@ -1,4 +1,4 @@
-use crate::path;
+use codeql_extractor::file_paths;
 use log::{debug, warn};
 use std::fs;
 use std::path::{Path, PathBuf};
@@ -15,12 +15,11 @@ impl Archiver {
     }
 
     fn try_archive(&self, source: &Path) -> std::io::Result<()> {
-        let mut dest = self.root.clone();
-        dest.push(path::key(source));
-        let parent = dest.parent().unwrap();
+        let dest = file_paths::path_for(&self.root, source, "");
         if fs::metadata(&dest).is_ok() {
             return Ok(());
         }
+        let parent = dest.parent().unwrap();
         fs::create_dir_all(parent)?;
         fs::copy(source, dest)?;
         debug!("archived {}", source.display());

rust/extractor/src/main.rs

@@ -1,5 +1,7 @@
 use crate::trap::TrapId;
 use anyhow::Context;
+use itertools::Itertools;
+use log::info;
 use ra_ap_hir::db::DefDatabase;
 use ra_ap_hir::Crate;
 use ra_ap_load_cargo::{load_workspace_at, LoadCargoConfig, ProcMacroServerChoice};
@@ -11,7 +13,6 @@ use std::path::PathBuf;
 mod archive;
 mod config;
 pub mod generated;
-pub mod path;
 mod translate;
 pub mod trap;
 
@@ -23,9 +24,12 @@ fn find_project_manifests(
         .iter()
         .map(|path| AbsPathBuf::assert_utf8(current.join(path)))
         .collect();
-    Ok(ra_ap_project_model::ProjectManifest::discover_all(
-        &abs_files,
-    ))
+    let ret = ra_ap_project_model::ProjectManifest::discover_all(&abs_files);
+    info!(
+        "found manifests: {}",
+        ret.iter().map(|m| format!("{m}")).join(", ")
+    );
+    Ok(ret)
 }
 
 fn main() -> anyhow::Result<()> {

rust/extractor/src/path.rs

@@ -1,10 +0,0 @@
-use std::fs::canonicalize;
-use std::path::{absolute, Path, PathBuf};
-
-pub fn key(p: &Path) -> PathBuf {
-    let normalized = canonicalize(p)
-        .or_else(|_| absolute(p))
-        .unwrap_or_else(|_| p.into());
-    let root = normalized.ancestors().last().unwrap(); // ancestors always yields at least one
-    normalized.strip_prefix(root).unwrap().into() // stripping an ancestor always works
-}

rust/extractor/src/trap.rs

@@ -1,9 +1,8 @@
+use crate::config;
 use crate::config::Compression;
-use crate::{config, path};
-use codeql_extractor::{extractor, trap};
+use codeql_extractor::{extractor, file_paths, trap};
 use log::debug;
 use ra_ap_ide_db::line_index::LineCol;
-use std::ffi::OsString;
 use std::fmt::Debug;
 use std::hash::Hash;
 use std::marker::PhantomData;
@@ -12,7 +11,6 @@ use std::path::{Path, PathBuf};
 pub use trap::Label as UntypedLabel;
 pub use trap::Writer;
 
-//TODO: typed labels
 pub trait AsTrapKeyPart {
     fn as_key_part(&self) -> String;
 }
@@ -210,19 +208,8 @@ impl TrapFileProvider {
     }
 
     pub fn create(&self, category: &str, key: &Path) -> TrapFile {
-        let mut path = PathBuf::from(category);
-        path.push(path::key(key));
-        path.set_extension(
-            path.extension()
-                .map(|e| {
-                    let mut o: OsString = e.to_owned();
-                    o.push(".trap");
-                    o
-                })
-                .unwrap_or("trap".into()),
-        );
+        let path = file_paths::path_for(&self.trap_dir.join(category), key, ".trap");
         debug!("creating trap file {}", path.display());
-        path = self.trap_dir.join(path);
         let mut writer = trap::Writer::new();
         extractor::populate_empty_location(&mut writer);
         TrapFile {

rust/integration-tests/conftest.py

@@ -0,0 +1,15 @@
+import pytest
+import shutil
+
+class _Manifests:
+    def __init__(self, cwd):
+        self.dir = cwd / "manifests"
+
+    def select(self, name: str):
+        (self.dir / name).rename(name)
+        shutil.rmtree(self.dir)
+
+
+@pytest.fixture
+def manifests(cwd):
+    return _Manifests(cwd)

rust/integration-tests/hello-project/functions.expected

@@ -0,0 +1 @@
+| src/main.rs:4:1:6:1 | main |

rust/integration-tests/hello-project/functions.ql

@@ -0,0 +1,5 @@
+import rust
+
+from Function f
+where exists(f.getLocation().getFile().getRelativePath())
+select f

rust/integration-tests/hello-project/manifests/Cargo.toml

@@ -0,0 +1,7 @@
+[workspace]
+[package]
+name = "hello-cargo"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]

rust/integration-tests/hello-project/manifests/rust-project.json

@@ -0,0 +1,7 @@
+{
+  "crates": [{
+    "root_module": "src/main.rs",
+    "edition": "2021",
+    "deps": []
+  }]
+}

rust/integration-tests/hello-project/source_archive.expected

@@ -0,0 +1,4 @@
+src/directory_module/mod.rs
+src/directory_module/nested_module.rs
+src/file_module.rs
+src/main.rs

rust/integration-tests/hello-project/src/directory_module/mod.rs

@@ -0,0 +1 @@
+mod nested_module;

rust/integration-tests/hello-project/src/main.rs

@@ -0,0 +1,6 @@
+mod file_module;
+mod directory_module;
+
+fn main() {
+    println!("Hello, world!");
+}

rust/integration-tests/hello-project/test_project.py

@@ -0,0 +1,7 @@
+def test_cargo(codeql, rust, manifests, check_source_archive):
+    manifests.select("Cargo.toml")
+    codeql.database.create()
+
+def test_rust_project(codeql, rust, manifests, check_source_archive):
+    manifests.select("rust-project.json")
+    codeql.database.create()

rust/integration-tests/hello-workspace/exe/Cargo.toml

@@ -0,0 +1,7 @@
+[package]
+name = "exe"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+lib = { path = "../lib" }

rust/integration-tests/hello-workspace/exe/src/main.rs

@@ -0,0 +1,7 @@
+use lib::a_module::hello;
+
+mod a_module;
+
+fn main() {
+    hello();
+}

rust/integration-tests/hello-workspace/functions.expected

@@ -0,0 +1,2 @@
+| exe/src/main.rs:5:1:7:1 | main |
+| lib/src/a_module/mod.rs:1:1:3:1 | hello |

rust/integration-tests/hello-workspace/functions.ql

@@ -0,0 +1,5 @@
+import rust
+
+from Function f
+where exists(f.getLocation().getFile().getRelativePath())
+select f

rust/integration-tests/hello-workspace/lib/Cargo.toml

@@ -0,0 +1,6 @@
+[package]
+name = "lib"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]

rust/integration-tests/hello-workspace/lib/src/a_module/mod.rs

@@ -0,0 +1,3 @@
+pub fn hello() {
+    println!("Hello, world!");
+}

rust/integration-tests/hello-workspace/lib/src/lib.rs

@@ -0,0 +1 @@
+pub mod a_module;

rust/integration-tests/hello-workspace/manifests/Cargo.toml

@@ -0,0 +1,3 @@
+[workspace]
+members = [ "exe", "lib" ]
+resolver = "2"

rust/integration-tests/hello-workspace/manifests/rust-project.json

@@ -0,0 +1,14 @@
+{
+  "crates": [
+    {
+      "root_module": "exe/src/main.rs",
+      "edition": "2021",
+      "deps": [{"crate": 1, "name": "lib"}]
+    },
+    {
+      "root_module": "lib/src/lib.rs",
+      "edition": "2021",
+      "deps": []
+    }
+  ]
+}

rust/integration-tests/hello-workspace/source_archive.expected

@@ -0,0 +1,4 @@
+exe/src/a_module.rs
+exe/src/main.rs
+lib/src/a_module/mod.rs
+lib/src/lib.rs

rust/integration-tests/hello-workspace/test_workspace.py

@@ -0,0 +1,12 @@
+import pytest
+
+# currently the DB-check fails on actions because of loading files multiple times and assiging multiple locations
+# see https://github.com/github/codeql-team/issues/3365
+@pytest.mark.ql_test("DB-CHECK", xfail="maybe")
+def test_cargo(codeql, rust, manifests, check_source_archive):
+    manifests.select("Cargo.toml")
+    codeql.database.create()
+
+def test_rust_project(codeql, rust, manifests, check_source_archive):
+    manifests.select("rust-project.json")
+    codeql.database.create()

rust/integration-tests/qlpack.lock.yml

@@ -0,0 +1,4 @@
+---
+dependencies: {}
+compiled: false
+lockVersion: 1.0.0

rust/integration-tests/qlpack.yml

@@ -0,0 +1,8 @@
+name: codeql/rust-integration-tests
+groups: [rust, test]
+dependencies:
+  codeql/rust-queries: ${workspace}
+  codeql/rust-all: ${workspace}
+extractor: rust
+tests: .
+warnOnImplicitThis: true

rust/integration-tests/test.py

@@ -1,4 +0,0 @@
-# dummy test to get CI going
-
-def test(codeql, rust):
-    pass

rust/lint.py

@@ -12,7 +12,7 @@ assert cargo, "no cargo binary found on `PATH`"
 
 fmt = subprocess.run([cargo, "fmt", "--all", "--quiet"], cwd=this_dir)
 for manifest in this_dir.rglob("Cargo.toml"):
-    if not manifest.is_relative_to(this_dir / "ql"):
+    if not manifest.is_relative_to(this_dir / "ql") and not manifest.is_relative_to(this_dir / "integration-tests"):
         clippy = subprocess.run([cargo, "clippy", "--fix", "--allow-dirty", "--allow-staged", "--quiet"],
                                 cwd=manifest.parent)
 sys.exit(fmt.returncode or clippy.returncode)