github
/
codeql-action
зеркало из https://github.com/github/codeql-action.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Clean up running queries workflow now that the queries are determined by the CLI

This commit is contained in:
Henry Mercer 2024-01-05 19:11:02 +00:00
Родитель 8516954d60
Коммит 838a022982
9 изменённых файлов: 25 добавлений и 258 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

19
lib/analyze.js сгенерированный
Просмотреть файл

@ -137,7 +137,9 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeRunQueries = new Date().getTime();
await runQueryGroup(language, "all", undefined, undefined, true);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseRunQueries(databasePath, queryFlags, features);
logger.debug(`Finished running queries for ${language}.`);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@ -205,21 +207,6 @@ async function runQueries(sarifFolder, memoryFlag, addSnippetsFlag, threadsFlag,
const databasePath = util.getCodeQLDatabasePath(config, language);
return await codeql.databasePrintBaseline(databasePath);
}
async function runQueryGroup(language, type, querySuiteContents, searchPath, optimizeForLastQueryRun) {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = querySuiteContents
? `${databasePath}-queries-${type}.qls`
: undefined;
if (querySuiteContents && querySuitePath) {
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(`Query suite file for ${language}-${type}...\n${querySuiteContents}`);
}
await codeql.databaseRunQueries(databasePath, searchPath, querySuitePath, queryFlags, optimizeForLastQueryRun, features);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
exports.runQueries = runQueries;
async function runFinalize(outputDir, threadsFlag, memoryFlag, config, logger, features) {

2
lib/analyze.js.map
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

74
lib/analyze.test.js сгенерированный
Просмотреть файл

@ -40,13 +40,12 @@ const uploadLib = __importStar(require("./upload-lib"));
const util = __importStar(require("./util"));
(0, testing_utils_1.setupTests)(ava_1.default);
/**
* Checks that the duration fields are populated for the correct language. Also checks the correct
* search paths are set in the database analyze invocation.
* Checks the status report produced by the analyze Action.
*
* Mocks the QA telemetry feature flag and checks the appropriate status report fields.
* - Checks that the duration fields are populated for the correct language.
* - Checks that the QA telemetry status report fields are populated when the QA feature flag is enabled.
*/
(0, ava_1.default)("status report fields and search path setting", async (t) => {
let searchPathsUsed = [];
(0, ava_1.default)("status report fields", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
(0, testing_utils_1.setupActionsVars)(tmpDir, tmpDir);
const memoryFlag = "";
@ -55,10 +54,8 @@ const util = __importStar(require("./util"));
sinon.stub(uploadLib, "validateSarifFileSchema");
for (const language of Object.values(languages_1.Language)) {
(0, codeql_1.setCodeQL)({
databaseRunQueries: async () => { },
packDownload: async () => ({ packs: [] }),
databaseRunQueries: async (_db, searchPath) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (_db, _queriesRun, sarifFile) => {
fs.writeFileSync(sarifFile, JSON.stringify({
runs: [
@ -98,7 +95,6 @@ const util = __importStar(require("./util"));
},
databasePrintBaseline: async () => "",
});
searchPathsUsed = [];
const config = {
languages: [language],
originalUserInput: {},
@ -135,64 +131,4 @@ const util = __importStar(require("./util"));
}
});
});
function mockCodeQL() {
return {
getVersion: async () => (0, testing_utils_1.makeVersionInfo)("1.0.0"),
databaseRunQueries: sinon.spy(),
databaseInterpretResults: async () => "",
databasePrintBaseline: async () => "",
};
}
function createBaseConfig(tmpDir) {
return {
languages: [],
originalUserInput: {},
tempDir: "tempDir",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
},
dbLocation: path.resolve(tmpDir, "codeql_databases"),
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
}
async function runQueriesWithConfig(config, features) {
for (const language of config.languages) {
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
}
return (0, analyze_1.runQueries)("sarif-folder", "--memFlag", "--addSnippetsFlag", "--threadsFlag", undefined, config, (0, logging_1.getRunnerLogger)(true), (0, testing_utils_1.createFeatures)(features));
}
function getDatabaseRunQueriesCalls(mock) {
return mock.databaseRunQueries.getCalls();
}
(0, ava_1.default)("optimizeForLastQueryRun for one language", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp];
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true]);
});
});
(0, ava_1.default)("optimizeForLastQueryRun for two languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
(0, codeql_1.setCodeQL)(codeql);
const config = createBaseConfig(tmpDir);
config.languages = [languages_1.Language.cpp, languages_1.Language.java];
await runQueriesWithConfig(config, []);
t.deepEqual(getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]), [true, true]);
});
});
//# sourceMappingURL=analyze.test.js.map

2
lib/analyze.test.js.map
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

11
lib/codeql.js сгенерированный
Просмотреть файл

@ -447,7 +447,7 @@ async function getCodeQLForCmd(cmd, checkVersion) {
throw new Error(`Unexpected output from codeql resolve build-environment: ${e} in\n${output}`);
}
},
async databaseRunQueries(databasePath, extraSearchPath, querySuitePath, flags, optimizeForLastQueryRun, features) {
async databaseRunQueries(databasePath, flags, features) {
const codeqlArgs = [
"database",
"run-queries",
@ -457,16 +457,9 @@ async function getCodeQLForCmd(cmd, checkVersion) {
"-v",
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (optimizeForLastQueryRun &&
(await util.supportExpectDiscardedCache(this))) {
if (await util.supportExpectDiscardedCache(this)) {
codeqlArgs.push("--expect-discarded-cache");
}
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
if (querySuitePath) {
codeqlArgs.push(querySuitePath);
}
if (await features.getValue(feature_flags_1.Feature.EvaluatorFineGrainedParallelismEnabled, this)) {
codeqlArgs.push("--intra-layer-parallelism");
}

2
lib/codeql.js.map
Просмотреть файл

Различия файлов скрыты, потому что одна или несколько строк слишком длинны

114
src/analyze.test.ts
Просмотреть файл

@ -4,31 +4,25 @@ import * as path from "path";
import test from "ava";
import * as sinon from "sinon";
import { runQueries, QueriesStatusReport } from "./analyze";
import { CodeQL, setCodeQL } from "./codeql";
import { runQueries } from "./analyze";
import { setCodeQL } from "./codeql";
import { Config } from "./config-utils";
import { Feature } from "./feature-flags";
import { Language } from "./languages";
import { getRunnerLogger } from "./logging";
import {
setupTests,
setupActionsVars,
createFeatures,
makeVersionInfo,
} from "./testing-utils";
import { setupTests, setupActionsVars, createFeatures } from "./testing-utils";
import * as uploadLib from "./upload-lib";
import * as util from "./util";
setupTests(test);
/**
* Checks that the duration fields are populated for the correct language. Also checks the correct
* search paths are set in the database analyze invocation.
* Checks the status report produced by the analyze Action.
*
* Mocks the QA telemetry feature flag and checks the appropriate status report fields.
* - Checks that the duration fields are populated for the correct language.
* - Checks that the QA telemetry status report fields are populated when the QA feature flag is enabled.
*/
test("status report fields and search path setting", async (t) => {
let searchPathsUsed: Array<string | undefined> = [];
test("status report fields", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
setupActionsVars(tmpDir, tmpDir);
@ -39,13 +33,8 @@ test("status report fields and search path setting", async (t) => {
for (const language of Object.values(Language)) {
setCodeQL({
databaseRunQueries: async () => {},
packDownload: async () => ({ packs: [] }),
databaseRunQueries: async (
_db: string,
searchPath: string | undefined,
) => {
searchPathsUsed.push(searchPath);
},
databaseInterpretResults: async (
_db: string,
_queriesRun: string[],
@ -93,7 +82,6 @@ test("status report fields and search path setting", async (t) => {
databasePrintBaseline: async () => "",
});
searchPathsUsed = [];
const config: Config = {
languages: [language],
originalUserInput: {},
@ -140,89 +128,3 @@ test("status report fields and search path setting", async (t) => {
}
});
});
function mockCodeQL(): Partial<CodeQL> {
return {
getVersion: async () => makeVersionInfo("1.0.0"),
databaseRunQueries: sinon.spy(),
databaseInterpretResults: async () => "",
databasePrintBaseline: async () => "",
};
}
function createBaseConfig(tmpDir: string): Config {
return {
languages: [],
originalUserInput: {},
tempDir: "tempDir",
codeQLCmd: "",
gitHubVersion: {
type: util.GitHubVariant.DOTCOM,
} as util.GitHubVersion,
dbLocation: path.resolve(tmpDir, "codeql_databases"),
debugMode: false,
debugArtifactName: util.DEFAULT_DEBUG_ARTIFACT_NAME,
debugDatabaseName: util.DEFAULT_DEBUG_DATABASE_NAME,
augmentationProperties: {
packsInputCombines: false,
queriesInputCombines: false,
},
trapCaches: {},
trapCacheDownloadTime: 0,
};
}
async function runQueriesWithConfig(
config: Config,
features: Feature[],
): Promise<QueriesStatusReport> {
for (const language of config.languages) {
fs.mkdirSync(util.getCodeQLDatabasePath(config, language), {
recursive: true,
});
}
return runQueries(
"sarif-folder",
"--memFlag",
"--addSnippetsFlag",
"--threadsFlag",
undefined,
config,
getRunnerLogger(true),
createFeatures(features),
);
}
function getDatabaseRunQueriesCalls(mock: Partial<CodeQL>) {
return (mock.databaseRunQueries as sinon.SinonSpy).getCalls();
}
test("optimizeForLastQueryRun for one language", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp];
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true],
);
});
});
test("optimizeForLastQueryRun for two languages", async (t) => {
return await util.withTmpDir(async (tmpDir) => {
const codeql = mockCodeQL();
setCodeQL(codeql);
const config: Config = createBaseConfig(tmpDir);
config.languages = [Language.cpp, Language.java];
await runQueriesWithConfig(config, []);
t.deepEqual(
getDatabaseRunQueriesCalls(codeql).map((c) => c.args[4]),
[true, true],
);
});
});

36
src/analyze.ts
Просмотреть файл

@ -268,7 +268,9 @@ export async function runQueries(
// another to interpret the results.
logger.startGroup(`Running queries for ${language}`);
const startTimeRunQueries = new Date().getTime();
await runQueryGroup(language, "all", undefined, undefined, true);
const databasePath = util.getCodeQLDatabasePath(config, language);
await codeql.databaseRunQueries(databasePath, queryFlags, features);
logger.debug(`Finished running queries for ${language}.`);
// TODO should not be using `builtin` here. We should be using `all` instead.
// The status report does not support `all` yet.
statusReport[`analyze_builtin_queries_${language}_duration_ms`] =
@ -380,38 +382,6 @@ export async function runQueries(
const databasePath = util.getCodeQLDatabasePath(config, language);
return await codeql.databasePrintBaseline(databasePath);
}
async function runQueryGroup(
language: Language,
type: string,
querySuiteContents: string | undefined,
searchPath: string | undefined,
optimizeForLastQueryRun: boolean,
): Promise<string | undefined> {
const databasePath = util.getCodeQLDatabasePath(config, language);
// Pass the queries to codeql using a file instead of using the command
// line to avoid command line length restrictions, particularly on windows.
const querySuitePath = querySuiteContents
? `${databasePath}-queries-${type}.qls`
: undefined;
if (querySuiteContents && querySuitePath) {
fs.writeFileSync(querySuitePath, querySuiteContents);
logger.debug(
`Query suite file for ${language}-${type}...\n${querySuiteContents}`,
);
}
await codeql.databaseRunQueries(
databasePath,
searchPath,
querySuitePath,
queryFlags,
optimizeForLastQueryRun,
features,
);
logger.debug(`BQRS results produced for ${language} (queries: ${type})"`);
return querySuitePath;
}
}
export async function runFinalize(

23
src/codeql.ts
Просмотреть файл

@ -162,19 +162,10 @@ export interface CodeQL {
): Promise<void>;
/**
* Run 'codeql database run-queries'.
*
* @param optimizeForLastQueryRun Whether to apply additional optimization for
* the last database query run in the action.
* It is always safe to set it to false.
* It should be set to true only for the very
* last databaseRunQueries() call.
*/
databaseRunQueries(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string | undefined,
flags: string[],
optimizeForLastQueryRun: boolean,
features: FeatureEnablement,
): Promise<void>;
/**
@ -802,10 +793,7 @@ export async function getCodeQLForCmd(
},
async databaseRunQueries(
databasePath: string,
extraSearchPath: string | undefined,
querySuitePath: string | undefined,
flags: string[],
optimizeForLastQueryRun: boolean,
features: FeatureEnablement,
): Promise<void> {
const codeqlArgs = [
@ -817,18 +805,9 @@ export async function getCodeQLForCmd(
"-v",
...getExtraOptionsFromEnv(["database", "run-queries"]),
];
if (
optimizeForLastQueryRun &&
(await util.supportExpectDiscardedCache(this))
) {
if (await util.supportExpectDiscardedCache(this)) {
codeqlArgs.push("--expect-discarded-cache");
}
if (extraSearchPath !== undefined) {
codeqlArgs.push("--additional-packs", extraSearchPath);
}
if (querySuitePath) {
codeqlArgs.push(querySuitePath);
}
if (
await features.getValue(
Feature.EvaluatorFineGrainedParallelismEnabled,