Check each grammar on independent CI tasks (#970)
* Check each grammar on independent CI tasks Each grammar will be checked in search of regression in a independent CI task. This mechanism is going to reduce the overall computation time since tasks are run in parallel. * Join JavaScript and Mozjs tests
This commit is contained in:
Luni-4
GitHub
Родитель
6d729a1de5
Коммит
9a93459980
177
.taskcluster.yml
177
.taskcluster.yml
|
|
@ -157,7 +157,7 @@ tasks:
|
|||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_crates")}
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_mozcpp")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
|
|
@ -174,19 +174,184 @@ tasks:
|
|||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh"
|
||||
./check-grammars-crates.sh tree-sitter-mozcpp"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests.tar.gz:
|
||||
public/json-diffs-and-minimal-tests-mozcpp.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests.tar.gz
|
||||
path: /tmp/json-diffs-and-minimal-tests-mozcpp.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter crates
|
||||
description: rust-code-analysis check tree-sitter crates
|
||||
name: rust-code-analysis check tree-sitter-mozcpp
|
||||
description: rust-code-analysis check tree-sitter-mozcpp grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_mozjs")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 hour'}
|
||||
provisionerId: proj-relman
|
||||
workerType: ci
|
||||
payload:
|
||||
maxRunTime: 3600
|
||||
image: "rust:buster"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-cx"
|
||||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh tree-sitter-mozjs"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests-mozjs.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests-mozjs.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter-mozjs
|
||||
description: rust-code-analysis check tree-sitter-mozjs grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_rust")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 hour'}
|
||||
provisionerId: proj-relman
|
||||
workerType: ci
|
||||
payload:
|
||||
maxRunTime: 3600
|
||||
image: "rust:buster"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-cx"
|
||||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh tree-sitter-rust"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests-rust.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests-rust.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter-rust
|
||||
description: rust-code-analysis check tree-sitter-rust grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_java")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 hour'}
|
||||
provisionerId: proj-relman
|
||||
workerType: ci
|
||||
payload:
|
||||
maxRunTime: 3600
|
||||
image: "rust:buster"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-cx"
|
||||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh tree-sitter-java"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests-java.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests-java.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter-java
|
||||
description: rust-code-analysis check tree-sitter-java grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_python")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 hour'}
|
||||
provisionerId: proj-relman
|
||||
workerType: ci
|
||||
payload:
|
||||
maxRunTime: 3600
|
||||
image: "rust:buster"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-cx"
|
||||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh tree-sitter-python"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests-python.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests-python.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter-python
|
||||
description: rust-code-analysis check tree-sitter-python grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
- taskId: {$eval: as_slugid("check_tree_sitter_typescript")}
|
||||
dependencies:
|
||||
- {$eval: as_slugid("lint_test_task")}
|
||||
- {$eval: as_slugid("windows_test_task")}
|
||||
created: {$fromNow: ''}
|
||||
deadline: {$fromNow: '1 hour'}
|
||||
provisionerId: proj-relman
|
||||
workerType: ci
|
||||
payload:
|
||||
maxRunTime: 3600
|
||||
image: "rust:buster"
|
||||
command:
|
||||
- "/bin/bash"
|
||||
- "-cx"
|
||||
- "git clone --quiet ${repository} &&
|
||||
cd rust-code-analysis &&
|
||||
git -c advice.detachedHead=false checkout ${head_rev} &&
|
||||
./check-grammars-crates.sh tree-sitter-typescript"
|
||||
cache:
|
||||
rust-code-analysis-mozilla-central-repository: /cache
|
||||
artifacts:
|
||||
public/json-diffs-and-minimal-tests-typescript.tar.gz:
|
||||
expires: {$fromNow: '2 weeks'}
|
||||
path: /tmp/json-diffs-and-minimal-tests-typescript.tar.gz
|
||||
type: file
|
||||
scopes:
|
||||
- "docker-worker:cache:rust-code-analysis-mozilla-central-repository"
|
||||
metadata:
|
||||
name: rust-code-analysis check tree-sitter-typescript
|
||||
description: rust-code-analysis check tree-sitter-typescript grammar
|
||||
owner: cdenizet@mozilla.com
|
||||
source: ${repository}/raw/${head_rev}/.taskcluster.yml
|
||||
|
||||
|
|
|
|||
|
|
@ -41,7 +41,6 @@ NEW_SUFFIX = "-new"
|
|||
|
||||
# Extensions parsed by each tree-sitter-grammar
|
||||
EXTENSIONS = {
|
||||
"tree-sitter-javascript": ["*.js", "*.js2"],
|
||||
"tree-sitter-tsx": ["*.tsx"],
|
||||
"tree-sitter-typescript": ["*.ts", "*.jsw", "*.jsmw"],
|
||||
"tree-sitter-java": ["*.java"],
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
# Stop at the first error
|
||||
set -e
|
||||
|
||||
# Get all tree-sitter crates from the analyzed branch Cargo.toml
|
||||
TS_CRATES=`grep "tree-sitter-*" Cargo.toml | tr -d ' '`
|
||||
# Get tree-sitter-grammar
|
||||
TS_CRATE=$1
|
||||
|
||||
# Disable/Enable CI flag
|
||||
RUN_CI="no"
|
||||
|
|
@ -15,34 +15,29 @@ MASTER_CARGO_TOML="master-cargo.toml"
|
|||
# Download master branch Cargo.toml and save it in a temporary file
|
||||
wget -LqO - https://raw.githubusercontent.com/mozilla/rust-code-analysis/master/Cargo.toml | tr -d ' ' > $MASTER_CARGO_TOML
|
||||
|
||||
# For each tree-sitter crate from the analyzed branch Cargo.toml
|
||||
for TS_CRATE in $TS_CRATES
|
||||
do
|
||||
# Get the name of the current crate
|
||||
TS_CRATE_NAME=`echo $TS_CRATE | cut -f1 -d "="`
|
||||
# Get the name of the current crate
|
||||
TS_CRATE_NAME=`echo $TS_CRATE | cut -f1 -d "="`
|
||||
|
||||
# Get the crate name from the master branch Cargo.toml
|
||||
MASTER_TS_CRATE_NAME=`grep $TS_CRATE_NAME $MASTER_CARGO_TOML | head -n 1 | cut -f1 -d "="`
|
||||
# Get the crate name from the master branch Cargo.toml
|
||||
MASTER_TS_CRATE_NAME=`grep $TS_CRATE_NAME $MASTER_CARGO_TOML | head -n 1 | cut -f1 -d "="`
|
||||
|
||||
# If the current crate name is not present in master branch, skip to the next crate
|
||||
if [ -z "$MASTER_TS_CRATE_NAME" ]
|
||||
then
|
||||
continue
|
||||
fi
|
||||
# If the current crate name is not present in master branch, exit the script
|
||||
if [ -z "$MASTER_TS_CRATE_NAME" ]
|
||||
then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Get the same crate from the master branch Cargo.toml
|
||||
MASTER_TS_CRATE=`grep $TS_CRATE $MASTER_CARGO_TOML | head -n 1`
|
||||
# Get the same crate from the master branch Cargo.toml
|
||||
MASTER_TS_CRATE=`grep $TS_CRATE $MASTER_CARGO_TOML | head -n 1`
|
||||
|
||||
# If the current crate has been updated, save the crate name and break the loop
|
||||
if [ -z "$MASTER_TS_CRATE" ]
|
||||
then
|
||||
# Enable CI flag
|
||||
RUN_CI="yes"
|
||||
# Name of tree-sitter crate
|
||||
TREE_SITTER_CRATE=$TS_CRATE_NAME
|
||||
break
|
||||
fi
|
||||
done
|
||||
# If the current crate has been updated, save the crate name
|
||||
if [ -z "$MASTER_TS_CRATE" ]
|
||||
then
|
||||
# Enable CI flag
|
||||
RUN_CI="yes"
|
||||
# Name of tree-sitter crate
|
||||
TREE_SITTER_CRATE=$TS_CRATE_NAME
|
||||
fi
|
||||
|
||||
# Remove temporary master branch Cargo.toml file
|
||||
rm -rf $MASTER_CARGO_TOML
|
||||
|
|
@ -95,9 +90,12 @@ if [ "$(ls -A $COMPARE)" ]; then
|
|||
# Output directory path
|
||||
OUTPUT_DIR=/tmp/output-$TREE_SITTER_CRATE
|
||||
|
||||
# Grammar name (removes tree-sitter- prefix)
|
||||
GRAMMAR_NAME=`echo $TREE_SITTER_CRATE | cut -c 13-`
|
||||
|
||||
# Split files into distinct directories depending on
|
||||
# their metric differences
|
||||
./split-minimal-tests.py -i $COMPARE -o $OUTPUT_DIR -t $MT_THRESHOLD
|
||||
|
||||
tar -czvf /tmp/json-diffs-and-minimal-tests.tar.gz $COMPARE $OUTPUT_DIR
|
||||
tar -czvf /tmp/json-diffs-and-minimal-tests-$GRAMMAR_NAME.tar.gz $COMPARE $OUTPUT_DIR
|
||||
fi
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче