Add RAM and threads options to init action
This commit is contained in:
Chuan-kai Lin
Родитель
2905689d8a
Коммит
70b730eb7d
|
|
@ -0,0 +1,63 @@
|
|||
# Warning: This file is generated automatically, and should not be modified.
|
||||
# Instead, please modify the template in the pr-checks directory and run:
|
||||
# pip install ruamel.yaml && python3 sync.py
|
||||
# to regenerate this file.
|
||||
|
||||
name: PR Check - Extractor ram and threads options test
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
GO111MODULE: auto
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
- v1
|
||||
pull_request:
|
||||
types:
|
||||
- opened
|
||||
- synchronize
|
||||
- reopened
|
||||
- ready_for_review
|
||||
workflow_dispatch: {}
|
||||
jobs:
|
||||
extractor-ram-threads:
|
||||
strategy:
|
||||
matrix:
|
||||
version: [latest]
|
||||
os: [ubuntu-latest]
|
||||
name: Extractor ram and threads options test
|
||||
runs-on: ${{ matrix.os }}
|
||||
steps:
|
||||
- name: Check out repository
|
||||
uses: actions/checkout@v2
|
||||
- name: Prepare test
|
||||
id: prepare-test
|
||||
uses: ./.github/prepare-test
|
||||
with:
|
||||
version: ${{ matrix.version }}
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: java
|
||||
ram: 230
|
||||
threads: 1
|
||||
- name: Assert Results
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${CODEQL_RAM}" != "230" ]; then
|
||||
echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
env:
|
||||
INTERNAL_CODEQL_ACTION_DEBUG_LOC: true
|
||||
|
|
@ -393,3 +393,42 @@ jobs:
|
|||
# Deliberately don't use TEST_MODE here. This is specifically testing
|
||||
# the compatibility with the API.
|
||||
runner/dist/codeql-runner-linux upload --sarif-file src/testdata/empty-sarif.sarif --repository $GITHUB_REPOSITORY --commit $GITHUB_SHA --ref $GITHUB_REF --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
|
||||
|
||||
runner-extractor-ram-threads-options:
|
||||
name: Runner ubuntu extractor RAM and threads options
|
||||
needs: [check-js, check-node-modules]
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v2
|
||||
|
||||
- name: Build runner
|
||||
run: |
|
||||
cd runner
|
||||
npm install
|
||||
npm run build-runner
|
||||
|
||||
- name: Run init
|
||||
run: |
|
||||
runner/dist/codeql-runner-linux init --ram=230 --threads=1 --repository $GITHUB_REPOSITORY --languages java --github-url $GITHUB_SERVER_URL --github-auth ${{ github.token }}
|
||||
|
||||
- name: Assert Results
|
||||
shell: bash
|
||||
run: |
|
||||
. ./codeql-runner/codeql-env.sh
|
||||
if [ "${CODEQL_RAM}" != "230" ]; then
|
||||
echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
## [UNRELEASED]
|
||||
|
||||
No user facing changes.
|
||||
- The `init` step of the Action now supports `ram` and `threads` inputs to limit resource use of CodeQL extractors. These inputs also serve as defaults to the subsequent `analyze` step, which finalizes the database and executes queries. [#738](https://github.com/github/codeql-action/pull/738)
|
||||
|
||||
## 1.0.21 - 28 Oct 2021
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,12 @@ inputs:
|
|||
required: false
|
||||
default: "brutal"
|
||||
ram:
|
||||
description: Override the amount of memory in MB to be used by CodeQL. By default, almost all the memory of the machine is used.
|
||||
description: >-
|
||||
The amount of memory in MB that can be used by CodeQL for database finalization and query execution.
|
||||
By default, this action will use the same amount of memory as previously set in the "init" action.
|
||||
If the "init" action also does not have an explicit "ram" input, this action will use most of the
|
||||
memory available in the system (which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows,
|
||||
and 13GB for macOS).
|
||||
required: false
|
||||
add-snippets:
|
||||
description: Specify whether or not to add code snippets to the output sarif file.
|
||||
|
|
@ -29,7 +34,12 @@ inputs:
|
|||
required: false
|
||||
default: "false"
|
||||
threads:
|
||||
description: The number of threads to be used by CodeQL.
|
||||
description: >-
|
||||
The number of threads that can be used by CodeQL for database finalization and query execution.
|
||||
By default, this action will use the same number of threads as previously set in the "init" action.
|
||||
If the "init" action also does not have an explicit "threads" input, this action will use all the
|
||||
hardware threads available in the system (which for GitHub-hosted runners is 2 for Linux and Windows
|
||||
and 3 for macOS).
|
||||
required: false
|
||||
checkout_path:
|
||||
description: "The path at which the analyzed repository was checked out. Used to relativize any absolute paths in the uploaded SARIF file."
|
||||
|
|
|
|||
|
|
@ -41,6 +41,20 @@ inputs:
|
|||
source-root:
|
||||
description: Path of the root source code directory, relative to $GITHUB_WORKSPACE.
|
||||
required: false
|
||||
ram:
|
||||
description: >-
|
||||
The amount of memory in MB that can be used by CodeQL extractors.
|
||||
By default, CodeQL extractors will use most of the memory available in the system
|
||||
(which for GitHub-hosted runners is 6GB for Linux, 5.5GB for Windows, and 13GB for macOS).
|
||||
This input also sets the amount of memory that can later be used by the "analyze" action.
|
||||
required: false
|
||||
threads:
|
||||
description: >-
|
||||
The number of threads that can be used by CodeQL extractors.
|
||||
By default, CodeQL extractors will use all the hardware threads available in the system
|
||||
(which for GitHub-hosted runners is 2 for Linux and Windows and 3 for macOS).
|
||||
This input also sets the number of threads that can later be used by the "analyze" action.
|
||||
required: false
|
||||
outputs:
|
||||
codeql-path:
|
||||
description: The path of the CodeQL binary used for analysis
|
||||
|
|
|
|||
|
|
@ -0,0 +1,75 @@
|
|||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const sinon = __importStar(require("sinon"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const analyze = __importStar(require("./analyze"));
|
||||
const configUtils = __importStar(require("./config-utils"));
|
||||
const testing_utils_1 = require("./testing-utils");
|
||||
const util = __importStar(require("./util"));
|
||||
(0, testing_utils_1.setupTests)(ava_1.default);
|
||||
// This test needs to be in its own file so that ava would run it in its own
|
||||
// nodejs process. The code being tested is in analyze-action.ts, which runs
|
||||
// immediately on load. So the file needs to be loaded during part of the test,
|
||||
// and that can happen only once per nodejs process. If multiple such tests are
|
||||
// in the same test file, ava would run them in the same nodejs process, and all
|
||||
// but the first test would fail.
|
||||
(0, ava_1.default)("analyze action with RAM & threads from environment variables", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
|
||||
process.env["GITHUB_REPOSITORY"] = "fake/repository";
|
||||
sinon
|
||||
.stub(actionsUtil, "createStatusReportBase")
|
||||
.resolves({});
|
||||
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
|
||||
sinon.stub(configUtils, "getConfig").resolves({
|
||||
languages: [],
|
||||
});
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub.withArgs("token").returns("fake-token");
|
||||
requiredInputStub.withArgs("upload-database").returns("false");
|
||||
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
|
||||
optionalInputStub.withArgs("cleanup-level").returns("none");
|
||||
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
|
||||
// When there are no action inputs for RAM and threads, the action uses
|
||||
// environment variables (passed down from the init action) to set RAM and
|
||||
// threads usage.
|
||||
process.env["CODEQL_THREADS"] = "-1";
|
||||
process.env["CODEQL_RAM"] = "4992";
|
||||
const runFinalizeStub = sinon.stub(analyze, "runFinalize");
|
||||
const runQueriesStub = sinon.stub(analyze, "runQueries");
|
||||
const analyzeAction = require("./analyze-action");
|
||||
// When analyze-action.ts loads, it runs an async function from the top
|
||||
// level but does not wait for it to finish. To ensure that calls to
|
||||
// runFinalize and runQueries are correctly captured by spies, we explicitly
|
||||
// wait for the action promise to complete before starting verification.
|
||||
await analyzeAction.runPromise;
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=analyze-action-env.test.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"analyze-action-env.test.js","sourceRoot":"","sources":["../src/analyze-action-env.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,8DAA8D,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IAC/E,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,uEAAuE;QACvE,0EAA0E;QAC1E,iBAAiB;QACjB,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,IAAI,CAAC;QACrC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
||||
|
|
@ -0,0 +1,75 @@
|
|||
"use strict";
|
||||
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
|
||||
}) : (function(o, m, k, k2) {
|
||||
if (k2 === undefined) k2 = k;
|
||||
o[k2] = m[k];
|
||||
}));
|
||||
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
||||
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
||||
}) : function(o, v) {
|
||||
o["default"] = v;
|
||||
});
|
||||
var __importStar = (this && this.__importStar) || function (mod) {
|
||||
if (mod && mod.__esModule) return mod;
|
||||
var result = {};
|
||||
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
||||
__setModuleDefault(result, mod);
|
||||
return result;
|
||||
};
|
||||
var __importDefault = (this && this.__importDefault) || function (mod) {
|
||||
return (mod && mod.__esModule) ? mod : { "default": mod };
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
const ava_1 = __importDefault(require("ava"));
|
||||
const sinon = __importStar(require("sinon"));
|
||||
const actionsUtil = __importStar(require("./actions-util"));
|
||||
const analyze = __importStar(require("./analyze"));
|
||||
const configUtils = __importStar(require("./config-utils"));
|
||||
const testing_utils_1 = require("./testing-utils");
|
||||
const util = __importStar(require("./util"));
|
||||
(0, testing_utils_1.setupTests)(ava_1.default);
|
||||
// This test needs to be in its own file so that ava would run it in its own
|
||||
// nodejs process. The code being tested is in analyze-action.ts, which runs
|
||||
// immediately on load. So the file needs to be loaded during part of the test,
|
||||
// and that can happen only once per nodejs process. If multiple such tests are
|
||||
// in the same test file, ava would run them in the same nodejs process, and all
|
||||
// but the first test would fail.
|
||||
(0, ava_1.default)("analyze action with RAM & threads from action inputs", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
|
||||
process.env["GITHUB_REPOSITORY"] = "fake/repository";
|
||||
sinon
|
||||
.stub(actionsUtil, "createStatusReportBase")
|
||||
.resolves({});
|
||||
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
|
||||
sinon.stub(configUtils, "getConfig").resolves({
|
||||
languages: [],
|
||||
});
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub.withArgs("token").returns("fake-token");
|
||||
requiredInputStub.withArgs("upload-database").returns("false");
|
||||
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
|
||||
optionalInputStub.withArgs("cleanup-level").returns("none");
|
||||
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
|
||||
process.env["CODEQL_THREADS"] = "1";
|
||||
process.env["CODEQL_RAM"] = "4992";
|
||||
// Action inputs have precedence over environment variables.
|
||||
optionalInputStub.withArgs("threads").returns("-1");
|
||||
optionalInputStub.withArgs("ram").returns("3012");
|
||||
const runFinalizeStub = sinon.stub(analyze, "runFinalize");
|
||||
const runQueriesStub = sinon.stub(analyze, "runQueries");
|
||||
const analyzeAction = require("./analyze-action");
|
||||
// When analyze-action.ts loads, it runs an async function from the top
|
||||
// level but does not wait for it to finish. To ensure that calls to
|
||||
// runFinalize and runQueries are correctly captured by spies, we explicitly
|
||||
// wait for the action promise to complete before starting verification.
|
||||
await analyzeAction.runPromise;
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
|
||||
});
|
||||
});
|
||||
//# sourceMappingURL=analyze-action-input.test.js.map
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"version":3,"file":"analyze-action-input.test.js","sourceRoot":"","sources":["../src/analyze-action-input.test.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;AAAA,8CAAuB;AACvB,6CAA+B;AAE/B,4DAA8C;AAC9C,mDAAqC;AACrC,4DAA8C;AAC9C,mDAA+D;AAC/D,6CAA+B;AAE/B,IAAA,0BAAU,EAAC,aAAI,CAAC,CAAC;AAEjB,4EAA4E;AAC5E,4EAA4E;AAC5E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,iCAAiC;AAEjC,IAAA,aAAI,EAAC,sDAAsD,EAAE,KAAK,EAAE,CAAC,EAAE,EAAE;IACvE,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE;QACrC,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,GAAG,iBAAiB,CAAC;QACrD,KAAK;aACF,IAAI,CAAC,WAAW,EAAE,wBAAwB,CAAC;aAC3C,QAAQ,CAAC,EAAkC,CAAC,CAAC;QAChD,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAC3D,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC,QAAQ,CAAC;YAC5C,SAAS,EAAE,EAAE;SACmB,CAAC,CAAC;QACpC,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,OAAO,CAAC,YAAY,CAAC,CAAC;QAC1D,iBAAiB,CAAC,QAAQ,CAAC,iBAAiB,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QAC/D,MAAM,iBAAiB,GAAG,KAAK,CAAC,IAAI,CAAC,WAAW,EAAE,kBAAkB,CAAC,CAAC;QACtE,iBAAiB,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAC5D,IAAA,gCAAgB,EAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAEjC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,GAAG,CAAC;QACpC,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,GAAG,MAAM,CAAC;QAEnC,4DAA4D;QAC5D,iBAAiB,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACpD,iBAAiB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;QAElD,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,aAAa,CAAC,CAAC;QAC3D,MAAM,cAAc,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,YAAY,CAAC,CAAC;QACzD,MAAM,aAAa,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAC;QAElD,uEAAuE;QACvE,oEAAoE;QACpE,4EAA4E;QAC5E,wEAAwE;QACxE,MAAM,aAAa,CAAC,UAAU,CAAC;QAE/B,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC/D,CAAC,CAAC,SAAS,CAAC,eAAe,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;QAC7D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAC9D,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,YAAY,CAAC,CAAC;IAC9D,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
||||
|
|
@ -19,6 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.runPromise = exports.sendStatusReport = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const path = __importStar(require("path"));
|
||||
const core = __importStar(require("@actions/core"));
|
||||
|
|
@ -44,6 +45,7 @@ async function sendStatusReport(startedAt, stats, error) {
|
|||
};
|
||||
await actionsUtil.sendStatusReport(statusReport);
|
||||
}
|
||||
exports.sendStatusReport = sendStatusReport;
|
||||
async function run() {
|
||||
const startedAt = new Date();
|
||||
let uploadStats = undefined;
|
||||
|
|
@ -65,8 +67,8 @@ async function run() {
|
|||
url: util.getRequiredEnvParam("GITHUB_SERVER_URL"),
|
||||
};
|
||||
const outputDir = actionsUtil.getRequiredInput("output");
|
||||
const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger);
|
||||
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram"));
|
||||
const threads = util.getThreadsFlag(actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"], logger);
|
||||
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]);
|
||||
await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
|
||||
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
|
||||
runStats = await (0, analyze_1.runQueries)(outputDir, memory, util.getAddSnippetsFlag(actionsUtil.getRequiredInput("add-snippets")), threads, actionsUtil.getOptionalInput("category"), config, logger);
|
||||
|
|
@ -134,9 +136,10 @@ async function run() {
|
|||
await sendStatusReport(startedAt, undefined);
|
||||
}
|
||||
}
|
||||
exports.runPromise = run();
|
||||
async function runWrapper() {
|
||||
try {
|
||||
await run();
|
||||
await exports.runPromise;
|
||||
}
|
||||
catch (error) {
|
||||
core.setFailed(`analyze action failed: ${error}`);
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -113,9 +113,14 @@ async function run() {
|
|||
core.exportVariable("GOFLAGS", goFlags);
|
||||
core.warning("Passing the GOFLAGS env parameter to the init action is deprecated. Please move this to the analyze action.");
|
||||
}
|
||||
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
|
||||
const codeqlRam = process.env["CODEQL_RAM"] || "6500";
|
||||
core.exportVariable("CODEQL_RAM", codeqlRam);
|
||||
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
|
||||
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
|
||||
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"
|
||||
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
|
||||
// for details.
|
||||
core.exportVariable("CODEQL_RAM", process.env["CODEQL_RAM"] ||
|
||||
(0, util_1.getMemoryFlagValue)((0, actions_util_1.getOptionalInput)("ram")).toString());
|
||||
core.exportVariable("CODEQL_THREADS", (0, util_1.getThreadsFlagValue)((0, actions_util_1.getOptionalInput)("threads"), logger).toString());
|
||||
const sourceRoot = path.resolve((0, util_1.getRequiredEnvParam)("GITHUB_WORKSPACE"), (0, actions_util_1.getOptionalInput)("source-root") || "");
|
||||
const tracerConfig = await (0, init_1.runInit)(codeql, config, sourceRoot, "Runner.Worker.exe", undefined);
|
||||
if (tracerConfig !== undefined) {
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -54,11 +54,14 @@ function getToolsDir(userInput) {
|
|||
return toolsDir;
|
||||
}
|
||||
const codeqlEnvJsonFilename = "codeql-env.json";
|
||||
function loadTracerEnvironment(config) {
|
||||
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
|
||||
return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
|
||||
}
|
||||
// Imports the environment from codeqlEnvJsonFilename if not already present
|
||||
function importTracerEnvironment(config) {
|
||||
if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
|
||||
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
|
||||
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
|
||||
const env = loadTracerEnvironment(config);
|
||||
for (const key of Object.keys(env)) {
|
||||
process.env[key] = env[key];
|
||||
}
|
||||
|
|
@ -116,6 +119,12 @@ program
|
|||
.option("--debug", "Print more verbose output", false)
|
||||
.option("--trace-process-name <string>", "(Advanced, windows-only) Inject a windows tracer of this process into a process with the given process name.")
|
||||
.option("--trace-process-level <number>", "(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up.")
|
||||
.option("--ram <number>", "The amount of memory in MB that can be used by CodeQL extractors. " +
|
||||
"By default, CodeQL extractors will use most of the memory available in the system. " +
|
||||
'This input also sets the amount of memory that can later be used by the "analyze" command.')
|
||||
.option("--threads <number>", "The number of threads that can be used by CodeQL extractors. " +
|
||||
"By default, CodeQL extractors will use all the hardware threads available in the system. " +
|
||||
'This input also sets the number of threads that can later be used by the "analyze" command.')
|
||||
.action(async (cmd) => {
|
||||
const logger = (0, logging_1.getRunnerLogger)(cmd.debug);
|
||||
try {
|
||||
|
|
@ -134,6 +143,13 @@ program
|
|||
};
|
||||
const gitHubVersion = await (0, util_1.getGitHubVersion)(apiDetails);
|
||||
(0, util_1.checkGitHubVersionInRange)(gitHubVersion, logger, util_1.Mode.runner);
|
||||
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
|
||||
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
|
||||
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"
|
||||
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
|
||||
// for details.
|
||||
process.env["CODEQL_RAM"] = (0, util_1.getMemoryFlagValue)(cmd.ram).toString();
|
||||
process.env["CODEQL_THREADS"] = (0, util_1.getThreadsFlagValue)(cmd.threads, logger).toString();
|
||||
let codeql;
|
||||
if (cmd.codeqlPath !== undefined) {
|
||||
codeql = await (0, codeql_1.getCodeQL)(cmd.codeqlPath);
|
||||
|
|
@ -240,10 +256,15 @@ program
|
|||
.option("--checkout-path <path>", "Checkout path. Default is the current working directory.")
|
||||
.option("--no-upload", "Do not upload results after analysis.")
|
||||
.option("--output-dir <dir>", "Directory to output SARIF files to. Default is in the temp directory.")
|
||||
.option("--ram <ram>", "Amount of memory to use when running queries. Default is to use all available memory.")
|
||||
.option("--ram <ram>", "The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
|
||||
'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
|
||||
'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
|
||||
"memory available in the system.")
|
||||
.option("--no-add-snippets", "Specify whether to include code snippets in the sarif output.")
|
||||
.option("--threads <threads>", "Number of threads to use when running queries. " +
|
||||
"Default is to use all available cores.")
|
||||
.option("--threads <threads>", "The number of threads that can be used by CodeQL for database finalization and query execution. " +
|
||||
'By default, this command will use the same number of threads as previously set in the "init" command. ' +
|
||||
'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
|
||||
"hardware threads available in the system.")
|
||||
.option("--temp-dir <dir>", 'Directory to use for temporary files. Default is "./codeql-runner".')
|
||||
.option("--category <category>", "String used by Code Scanning for matching the analyses.")
|
||||
.option("--debug", "Print more verbose output", false)
|
||||
|
|
@ -262,8 +283,15 @@ program
|
|||
url: (0, util_1.parseGitHubUrl)(cmd.githubUrl),
|
||||
};
|
||||
const outputDir = cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
|
||||
const threads = (0, util_1.getThreadsFlag)(cmd.threads, logger);
|
||||
const memory = (0, util_1.getMemoryFlag)(cmd.ram);
|
||||
let initEnv = {};
|
||||
try {
|
||||
initEnv = loadTracerEnvironment(config);
|
||||
}
|
||||
catch (err) {
|
||||
// The init command did not generate a tracer environment file
|
||||
}
|
||||
const threads = (0, util_1.getThreadsFlag)(cmd.threads || initEnv["CODEQL_THREADS"], logger);
|
||||
const memory = (0, util_1.getMemoryFlag)(cmd.ram || initEnv["CODEQL_RAM"]);
|
||||
await (0, analyze_1.runFinalize)(outputDir, threads, memory, config, logger);
|
||||
await (0, analyze_1.runQueries)(outputDir, memory, (0, util_1.getAddSnippetsFlag)(cmd.addSnippets), threads, cmd.category, config, logger);
|
||||
if (!cmd.upload) {
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -19,7 +19,7 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|||
return result;
|
||||
};
|
||||
Object.defineProperty(exports, "__esModule", { value: true });
|
||||
exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.GITHUB_DOTCOM_URL = void 0;
|
||||
exports.codeQlVersionAbove = exports.isHTTPError = exports.HTTPError = exports.getRequiredEnvParam = exports.isActions = exports.getMode = exports.enrichEnvironment = exports.initializeEnvironment = exports.Mode = exports.assertNever = exports.getGitHubAuth = exports.apiVersionInRange = exports.DisallowedAPIVersionReason = exports.checkGitHubVersionInRange = exports.getGitHubVersion = exports.GitHubVariant = exports.parseGitHubUrl = exports.getCodeQLDatabasePath = exports.getThreadsFlag = exports.getThreadsFlagValue = exports.getAddSnippetsFlag = exports.getMemoryFlag = exports.getMemoryFlagValue = exports.withTmpDir = exports.getToolNames = exports.getExtraOptionsEnvParam = exports.GITHUB_DOTCOM_URL = void 0;
|
||||
const fs = __importStar(require("fs"));
|
||||
const os = __importStar(require("os"));
|
||||
const path = __importStar(require("path"));
|
||||
|
|
@ -93,13 +93,13 @@ function getSystemReservedMemoryMegaBytes() {
|
|||
return 1024 * (process.platform === "win32" ? 1.5 : 1);
|
||||
}
|
||||
/**
|
||||
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
|
||||
* specified, the total available memory will be used minus a threshold
|
||||
* reserved for the OS.
|
||||
* Get the value of the codeql `--ram` flag as configured by the `ram` input.
|
||||
* If no value was specified, the total available memory will be used minus a
|
||||
* threshold reserved for the OS.
|
||||
*
|
||||
* @returns string
|
||||
* @returns {number} the amount of RAM to use, in megabytes
|
||||
*/
|
||||
function getMemoryFlag(userInput) {
|
||||
function getMemoryFlagValue(userInput) {
|
||||
let memoryToUseMegaBytes;
|
||||
if (userInput) {
|
||||
memoryToUseMegaBytes = Number(userInput);
|
||||
|
|
@ -113,7 +113,18 @@ function getMemoryFlag(userInput) {
|
|||
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
|
||||
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
|
||||
}
|
||||
return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
|
||||
return Math.floor(memoryToUseMegaBytes);
|
||||
}
|
||||
exports.getMemoryFlagValue = getMemoryFlagValue;
|
||||
/**
|
||||
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
|
||||
* specified, the total available memory will be used minus a threshold
|
||||
* reserved for the OS.
|
||||
*
|
||||
* @returns string
|
||||
*/
|
||||
function getMemoryFlag(userInput) {
|
||||
return `--ram=${getMemoryFlagValue(userInput)}`;
|
||||
}
|
||||
exports.getMemoryFlag = getMemoryFlag;
|
||||
/**
|
||||
|
|
@ -130,14 +141,14 @@ function getAddSnippetsFlag(userInput) {
|
|||
}
|
||||
exports.getAddSnippetsFlag = getAddSnippetsFlag;
|
||||
/**
|
||||
* Get the codeql `--threads` value specified for the `threads` input.
|
||||
* If no value was specified, all available threads will be used.
|
||||
* Get the value of the codeql `--threads` flag specified for the `threads`
|
||||
* input. If no value was specified, all available threads will be used.
|
||||
*
|
||||
* The value will be capped to the number of available CPUs.
|
||||
*
|
||||
* @returns string
|
||||
* @returns {number}
|
||||
*/
|
||||
function getThreadsFlag(userInput, logger) {
|
||||
function getThreadsFlagValue(userInput, logger) {
|
||||
let numThreads;
|
||||
const maxThreads = os.cpus().length;
|
||||
if (userInput) {
|
||||
|
|
@ -159,7 +170,19 @@ function getThreadsFlag(userInput, logger) {
|
|||
// Default to using all threads
|
||||
numThreads = maxThreads;
|
||||
}
|
||||
return `--threads=${numThreads}`;
|
||||
return numThreads;
|
||||
}
|
||||
exports.getThreadsFlagValue = getThreadsFlagValue;
|
||||
/**
|
||||
* Get the codeql `--threads` flag specified for the `threads` input.
|
||||
* If no value was specified, all available threads will be used.
|
||||
*
|
||||
* The value will be capped to the number of available CPUs.
|
||||
*
|
||||
* @returns string
|
||||
*/
|
||||
function getThreadsFlag(userInput, logger) {
|
||||
return `--threads=${getThreadsFlagValue(userInput, logger)}`;
|
||||
}
|
||||
exports.getThreadsFlag = getThreadsFlag;
|
||||
/**
|
||||
|
|
|
|||
Различия файлов скрыты, потому что одна или несколько строк слишком длинны
|
|
@ -0,0 +1,29 @@
|
|||
name: "Extractor ram and threads options test"
|
||||
description: "Tests passing RAM and threads limits to extractors"
|
||||
versions: ["latest"]
|
||||
os: ["ubuntu-latest"]
|
||||
steps:
|
||||
- uses: ./../action/init
|
||||
with:
|
||||
languages: java
|
||||
ram: 230
|
||||
threads: 1
|
||||
- name: Assert Results
|
||||
shell: bash
|
||||
run: |
|
||||
if [ "${CODEQL_RAM}" != "230" ]; then
|
||||
echo "CODEQL_RAM is '${CODEQL_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_RAM}" != "230" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_RAM is '${CODEQL_EXTRACTOR_JAVA_RAM}' instead of 230"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_THREADS is '${CODEQL_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
if [ "${CODEQL_EXTRACTOR_JAVA_THREADS}" != "1" ]; then
|
||||
echo "CODEQL_EXTRACTOR_JAVA_THREADS is '${CODEQL_EXTRACTOR_JAVA_THREADS}' instead of 1"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -47,5 +47,7 @@ predicate areNotEquivalent(YAMLValue x, YAMLValue y) {
|
|||
from ActionDeclaration actionA, ActionDeclaration actionB, string inputName
|
||||
where actionA.getName() < actionB.getName() // prevent duplicates which are permutations of the names
|
||||
and areNotEquivalent(actionA.getInput(inputName), actionB.getInput(inputName))
|
||||
// ram and threads inputs in different actions are supposed to have different description
|
||||
and inputName != "ram" and inputName != "threads"
|
||||
select actionA, "Action $@ and action $@ both declare input $@, however their definitions are not identical. This may be confusing to users.",
|
||||
actionA, actionA.getName(), actionB, actionB.getName(), inputName, inputName
|
||||
|
|
|
|||
|
|
@ -0,0 +1,58 @@
|
|||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import * as analyze from "./analyze";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { setupTests, setupActionsVars } from "./testing-utils";
|
||||
import * as util from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
// This test needs to be in its own file so that ava would run it in its own
|
||||
// nodejs process. The code being tested is in analyze-action.ts, which runs
|
||||
// immediately on load. So the file needs to be loaded during part of the test,
|
||||
// and that can happen only once per nodejs process. If multiple such tests are
|
||||
// in the same test file, ava would run them in the same nodejs process, and all
|
||||
// but the first test would fail.
|
||||
|
||||
test("analyze action with RAM & threads from environment variables", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
|
||||
process.env["GITHUB_REPOSITORY"] = "fake/repository";
|
||||
sinon
|
||||
.stub(actionsUtil, "createStatusReportBase")
|
||||
.resolves({} as actionsUtil.StatusReportBase);
|
||||
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
|
||||
sinon.stub(configUtils, "getConfig").resolves({
|
||||
languages: [],
|
||||
} as unknown as configUtils.Config);
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub.withArgs("token").returns("fake-token");
|
||||
requiredInputStub.withArgs("upload-database").returns("false");
|
||||
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
|
||||
optionalInputStub.withArgs("cleanup-level").returns("none");
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
// When there are no action inputs for RAM and threads, the action uses
|
||||
// environment variables (passed down from the init action) to set RAM and
|
||||
// threads usage.
|
||||
process.env["CODEQL_THREADS"] = "-1";
|
||||
process.env["CODEQL_RAM"] = "4992";
|
||||
|
||||
const runFinalizeStub = sinon.stub(analyze, "runFinalize");
|
||||
const runQueriesStub = sinon.stub(analyze, "runQueries");
|
||||
const analyzeAction = require("./analyze-action");
|
||||
|
||||
// When analyze-action.ts loads, it runs an async function from the top
|
||||
// level but does not wait for it to finish. To ensure that calls to
|
||||
// runFinalize and runQueries are correctly captured by spies, we explicitly
|
||||
// wait for the action promise to complete before starting verification.
|
||||
await analyzeAction.runPromise;
|
||||
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=4992");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=4992");
|
||||
});
|
||||
});
|
||||
|
|
@ -0,0 +1,59 @@
|
|||
import test from "ava";
|
||||
import * as sinon from "sinon";
|
||||
|
||||
import * as actionsUtil from "./actions-util";
|
||||
import * as analyze from "./analyze";
|
||||
import * as configUtils from "./config-utils";
|
||||
import { setupTests, setupActionsVars } from "./testing-utils";
|
||||
import * as util from "./util";
|
||||
|
||||
setupTests(test);
|
||||
|
||||
// This test needs to be in its own file so that ava would run it in its own
|
||||
// nodejs process. The code being tested is in analyze-action.ts, which runs
|
||||
// immediately on load. So the file needs to be loaded during part of the test,
|
||||
// and that can happen only once per nodejs process. If multiple such tests are
|
||||
// in the same test file, ava would run them in the same nodejs process, and all
|
||||
// but the first test would fail.
|
||||
|
||||
test("analyze action with RAM & threads from action inputs", async (t) => {
|
||||
await util.withTmpDir(async (tmpDir) => {
|
||||
process.env["GITHUB_SERVER_URL"] = "fake-server-url";
|
||||
process.env["GITHUB_REPOSITORY"] = "fake/repository";
|
||||
sinon
|
||||
.stub(actionsUtil, "createStatusReportBase")
|
||||
.resolves({} as actionsUtil.StatusReportBase);
|
||||
sinon.stub(actionsUtil, "sendStatusReport").resolves(true);
|
||||
sinon.stub(configUtils, "getConfig").resolves({
|
||||
languages: [],
|
||||
} as unknown as configUtils.Config);
|
||||
const requiredInputStub = sinon.stub(actionsUtil, "getRequiredInput");
|
||||
requiredInputStub.withArgs("token").returns("fake-token");
|
||||
requiredInputStub.withArgs("upload-database").returns("false");
|
||||
const optionalInputStub = sinon.stub(actionsUtil, "getOptionalInput");
|
||||
optionalInputStub.withArgs("cleanup-level").returns("none");
|
||||
setupActionsVars(tmpDir, tmpDir);
|
||||
|
||||
process.env["CODEQL_THREADS"] = "1";
|
||||
process.env["CODEQL_RAM"] = "4992";
|
||||
|
||||
// Action inputs have precedence over environment variables.
|
||||
optionalInputStub.withArgs("threads").returns("-1");
|
||||
optionalInputStub.withArgs("ram").returns("3012");
|
||||
|
||||
const runFinalizeStub = sinon.stub(analyze, "runFinalize");
|
||||
const runQueriesStub = sinon.stub(analyze, "runQueries");
|
||||
const analyzeAction = require("./analyze-action");
|
||||
|
||||
// When analyze-action.ts loads, it runs an async function from the top
|
||||
// level but does not wait for it to finish. To ensure that calls to
|
||||
// runFinalize and runQueries are correctly captured by spies, we explicitly
|
||||
// wait for the action promise to complete before starting verification.
|
||||
await analyzeAction.runPromise;
|
||||
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[1], "--threads=-1");
|
||||
t.deepEqual(runFinalizeStub.firstCall.args[2], "--ram=3012");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[3], "--threads=-1");
|
||||
t.deepEqual(runQueriesStub.firstCall.args[1], "--ram=3012");
|
||||
});
|
||||
});
|
||||
|
|
@ -31,7 +31,7 @@ interface FinishStatusReport
|
|||
extends actionsUtil.StatusReportBase,
|
||||
AnalysisStatusReport {}
|
||||
|
||||
async function sendStatusReport(
|
||||
export async function sendStatusReport(
|
||||
startedAt: Date,
|
||||
stats: AnalysisStatusReport | undefined,
|
||||
error?: Error
|
||||
|
|
@ -91,10 +91,12 @@ async function run() {
|
|||
};
|
||||
const outputDir = actionsUtil.getRequiredInput("output");
|
||||
const threads = util.getThreadsFlag(
|
||||
actionsUtil.getOptionalInput("threads"),
|
||||
actionsUtil.getOptionalInput("threads") || process.env["CODEQL_THREADS"],
|
||||
logger
|
||||
);
|
||||
const memory = util.getMemoryFlag(actionsUtil.getOptionalInput("ram"));
|
||||
const memory = util.getMemoryFlag(
|
||||
actionsUtil.getOptionalInput("ram") || process.env["CODEQL_RAM"]
|
||||
);
|
||||
await runFinalize(outputDir, threads, memory, config, logger);
|
||||
if (actionsUtil.getRequiredInput("skip-queries") !== "true") {
|
||||
runStats = await runQueries(
|
||||
|
|
@ -188,9 +190,11 @@ async function run() {
|
|||
}
|
||||
}
|
||||
|
||||
export const runPromise = run();
|
||||
|
||||
async function runWrapper() {
|
||||
try {
|
||||
await run();
|
||||
await runPromise;
|
||||
} catch (error) {
|
||||
core.setFailed(`analyze action failed: ${error}`);
|
||||
console.log(error);
|
||||
|
|
|
|||
|
|
@ -32,6 +32,8 @@ import {
|
|||
getGitHubVersion,
|
||||
codeQlVersionAbove,
|
||||
enrichEnvironment,
|
||||
getMemoryFlagValue,
|
||||
getThreadsFlagValue,
|
||||
} from "./util";
|
||||
|
||||
// eslint-disable-next-line import/no-commonjs
|
||||
|
|
@ -204,9 +206,20 @@ async function run() {
|
|||
);
|
||||
}
|
||||
|
||||
// Setup CODEQL_RAM flag (todo improve this https://github.com/github/dsp-code-scanning/issues/935)
|
||||
const codeqlRam = process.env["CODEQL_RAM"] || "6500";
|
||||
core.exportVariable("CODEQL_RAM", codeqlRam);
|
||||
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
|
||||
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
|
||||
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"
|
||||
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
|
||||
// for details.
|
||||
core.exportVariable(
|
||||
"CODEQL_RAM",
|
||||
process.env["CODEQL_RAM"] ||
|
||||
getMemoryFlagValue(getOptionalInput("ram")).toString()
|
||||
);
|
||||
core.exportVariable(
|
||||
"CODEQL_THREADS",
|
||||
getThreadsFlagValue(getOptionalInput("threads"), logger).toString()
|
||||
);
|
||||
|
||||
const sourceRoot = path.resolve(
|
||||
getRequiredEnvParam("GITHUB_WORKSPACE"),
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ import {
|
|||
Mode,
|
||||
codeQlVersionAbove,
|
||||
enrichEnvironment,
|
||||
getMemoryFlagValue,
|
||||
getThreadsFlagValue,
|
||||
} from "./util";
|
||||
|
||||
// eslint-disable-next-line import/no-commonjs
|
||||
|
|
@ -53,11 +55,15 @@ function getToolsDir(userInput: string | undefined): string {
|
|||
|
||||
const codeqlEnvJsonFilename = "codeql-env.json";
|
||||
|
||||
function loadTracerEnvironment(config: Config): { [name: string]: string } {
|
||||
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
|
||||
return JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
|
||||
}
|
||||
|
||||
// Imports the environment from codeqlEnvJsonFilename if not already present
|
||||
function importTracerEnvironment(config: Config) {
|
||||
if (!("ODASA_TRACER_CONFIGURATION" in process.env)) {
|
||||
const jsonEnvFile = path.join(config.tempDir, codeqlEnvJsonFilename);
|
||||
const env = JSON.parse(fs.readFileSync(jsonEnvFile).toString("utf-8"));
|
||||
const env = loadTracerEnvironment(config);
|
||||
for (const key of Object.keys(env)) {
|
||||
process.env[key] = env[key];
|
||||
}
|
||||
|
|
@ -109,6 +115,8 @@ interface InitArgs {
|
|||
githubAuth: string;
|
||||
githubAuthStdin: boolean;
|
||||
debug: boolean;
|
||||
ram: string | undefined;
|
||||
threads: string | undefined;
|
||||
}
|
||||
|
||||
program
|
||||
|
|
@ -167,6 +175,18 @@ program
|
|||
"--trace-process-level <number>",
|
||||
"(Advanced, windows-only) Inject a windows tracer of this process into a parent process <number> levels up."
|
||||
)
|
||||
.option(
|
||||
"--ram <number>",
|
||||
"The amount of memory in MB that can be used by CodeQL extractors. " +
|
||||
"By default, CodeQL extractors will use most of the memory available in the system. " +
|
||||
'This input also sets the amount of memory that can later be used by the "analyze" command.'
|
||||
)
|
||||
.option(
|
||||
"--threads <number>",
|
||||
"The number of threads that can be used by CodeQL extractors. " +
|
||||
"By default, CodeQL extractors will use all the hardware threads available in the system. " +
|
||||
'This input also sets the number of threads that can later be used by the "analyze" command.'
|
||||
)
|
||||
.action(async (cmd: InitArgs) => {
|
||||
const logger = getRunnerLogger(cmd.debug);
|
||||
|
||||
|
|
@ -195,6 +215,17 @@ program
|
|||
const gitHubVersion = await getGitHubVersion(apiDetails);
|
||||
checkGitHubVersionInRange(gitHubVersion, logger, Mode.runner);
|
||||
|
||||
// Limit RAM and threads for extractors. When running extractors, the CodeQL CLI obeys the
|
||||
// CODEQL_RAM and CODEQL_THREADS environment variables to decide how much RAM and how many
|
||||
// threads it would ask extractors to use. See help text for the "--ram" and "--threads"
|
||||
// options at https://codeql.github.com/docs/codeql-cli/manual/database-trace-command/
|
||||
// for details.
|
||||
process.env["CODEQL_RAM"] = getMemoryFlagValue(cmd.ram).toString();
|
||||
process.env["CODEQL_THREADS"] = getThreadsFlagValue(
|
||||
cmd.threads,
|
||||
logger
|
||||
).toString();
|
||||
|
||||
let codeql: CodeQL;
|
||||
if (cmd.codeqlPath !== undefined) {
|
||||
codeql = await getCodeQL(cmd.codeqlPath);
|
||||
|
|
@ -402,7 +433,10 @@ program
|
|||
)
|
||||
.option(
|
||||
"--ram <ram>",
|
||||
"Amount of memory to use when running queries. Default is to use all available memory."
|
||||
"The amount of memory in MB that can be used by CodeQL for database finalization and query execution. " +
|
||||
'By default, this command will use the same amount of memory as previously set in the "init" command. ' +
|
||||
'If the "init" command also does not have an explicit "ram" flag, this command will use most of the ' +
|
||||
"memory available in the system."
|
||||
)
|
||||
.option(
|
||||
"--no-add-snippets",
|
||||
|
|
@ -410,8 +444,10 @@ program
|
|||
)
|
||||
.option(
|
||||
"--threads <threads>",
|
||||
"Number of threads to use when running queries. " +
|
||||
"Default is to use all available cores."
|
||||
"The number of threads that can be used by CodeQL for database finalization and query execution. " +
|
||||
'By default, this command will use the same number of threads as previously set in the "init" command. ' +
|
||||
'If the "init" command also does not have an explicit "threads" flag, this command will use all the ' +
|
||||
"hardware threads available in the system."
|
||||
)
|
||||
.option(
|
||||
"--temp-dir <dir>",
|
||||
|
|
@ -447,8 +483,17 @@ program
|
|||
|
||||
const outputDir =
|
||||
cmd.outputDir || path.join(config.tempDir, "codeql-sarif");
|
||||
const threads = getThreadsFlag(cmd.threads, logger);
|
||||
const memory = getMemoryFlag(cmd.ram);
|
||||
let initEnv: { [name: string]: string } = {};
|
||||
try {
|
||||
initEnv = loadTracerEnvironment(config);
|
||||
} catch (err) {
|
||||
// The init command did not generate a tracer environment file
|
||||
}
|
||||
const threads = getThreadsFlag(
|
||||
cmd.threads || initEnv["CODEQL_THREADS"],
|
||||
logger
|
||||
);
|
||||
const memory = getMemoryFlag(cmd.ram || initEnv["CODEQL_RAM"]);
|
||||
await runFinalize(outputDir, threads, memory, config, logger);
|
||||
await runQueries(
|
||||
outputDir,
|
||||
|
|
|
|||
50
src/util.ts
50
src/util.ts
|
|
@ -85,13 +85,13 @@ function getSystemReservedMemoryMegaBytes(): number {
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
|
||||
* specified, the total available memory will be used minus a threshold
|
||||
* reserved for the OS.
|
||||
* Get the value of the codeql `--ram` flag as configured by the `ram` input.
|
||||
* If no value was specified, the total available memory will be used minus a
|
||||
* threshold reserved for the OS.
|
||||
*
|
||||
* @returns string
|
||||
* @returns {number} the amount of RAM to use, in megabytes
|
||||
*/
|
||||
export function getMemoryFlag(userInput: string | undefined): string {
|
||||
export function getMemoryFlagValue(userInput: string | undefined): number {
|
||||
let memoryToUseMegaBytes: number;
|
||||
if (userInput) {
|
||||
memoryToUseMegaBytes = Number(userInput);
|
||||
|
|
@ -104,7 +104,18 @@ export function getMemoryFlag(userInput: string | undefined): string {
|
|||
const reservedMemoryMegaBytes = getSystemReservedMemoryMegaBytes();
|
||||
memoryToUseMegaBytes = totalMemoryMegaBytes - reservedMemoryMegaBytes;
|
||||
}
|
||||
return `--ram=${Math.floor(memoryToUseMegaBytes)}`;
|
||||
return Math.floor(memoryToUseMegaBytes);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the codeql `--ram` flag as configured by the `ram` input. If no value was
|
||||
* specified, the total available memory will be used minus a threshold
|
||||
* reserved for the OS.
|
||||
*
|
||||
* @returns string
|
||||
*/
|
||||
export function getMemoryFlag(userInput: string | undefined): string {
|
||||
return `--ram=${getMemoryFlagValue(userInput)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -123,17 +134,17 @@ export function getAddSnippetsFlag(
|
|||
}
|
||||
|
||||
/**
|
||||
* Get the codeql `--threads` value specified for the `threads` input.
|
||||
* If no value was specified, all available threads will be used.
|
||||
* Get the value of the codeql `--threads` flag specified for the `threads`
|
||||
* input. If no value was specified, all available threads will be used.
|
||||
*
|
||||
* The value will be capped to the number of available CPUs.
|
||||
*
|
||||
* @returns string
|
||||
* @returns {number}
|
||||
*/
|
||||
export function getThreadsFlag(
|
||||
export function getThreadsFlagValue(
|
||||
userInput: string | undefined,
|
||||
logger: Logger
|
||||
): string {
|
||||
): number {
|
||||
let numThreads: number;
|
||||
const maxThreads = os.cpus().length;
|
||||
if (userInput) {
|
||||
|
|
@ -158,7 +169,22 @@ export function getThreadsFlag(
|
|||
// Default to using all threads
|
||||
numThreads = maxThreads;
|
||||
}
|
||||
return `--threads=${numThreads}`;
|
||||
return numThreads;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the codeql `--threads` flag specified for the `threads` input.
|
||||
* If no value was specified, all available threads will be used.
|
||||
*
|
||||
* The value will be capped to the number of available CPUs.
|
||||
*
|
||||
* @returns string
|
||||
*/
|
||||
export function getThreadsFlag(
|
||||
userInput: string | undefined,
|
||||
logger: Logger
|
||||
): string {
|
||||
return `--threads=${getThreadsFlagValue(userInput, logger)}`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче