diff --git a/extensions/ql-vscode/scripts/find-deadcode.ts b/extensions/ql-vscode/scripts/find-deadcode.ts
index 2c0375022..12e3fb03e 100644
--- a/extensions/ql-vscode/scripts/find-deadcode.ts
+++ b/extensions/ql-vscode/scripts/find-deadcode.ts
@@ -9,7 +9,7 @@ function ignoreFile(file: string): boolean {
     containsPath(".storybook", file) ||
     containsPath(join("src", "stories"), file) ||
     pathsEqual(
-      join("test", "vscode-tests", "jest-runner-installed-extensions.ts"),
+      join("test", "vscode-tests", "jest-runner-vscode-codeql-cli.ts"),
       file,
     ) ||
     basename(file) === "jest.config.ts" ||
diff --git a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts
index 3bd6d399c..e781b2cb3 100644
--- a/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts
+++ b/extensions/ql-vscode/test/vscode-tests/activated-extension/jest.config.ts
@@ -4,7 +4,7 @@ import baseConfig from "../jest.config.base";
 
 const config: Config = {
   ...baseConfig,
-  runner: "<rootDir>/../jest-runner-installed-extensions.ts",
+  runner: "<rootDir>/../jest-runner-vscode-codeql-cli.ts",
   setupFilesAfterEnv: ["<rootDir>/jest.setup.ts"],
 };
 
diff --git a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts
index ff7eac9da..fe634b66c 100644
--- a/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts
+++ b/extensions/ql-vscode/test/vscode-tests/cli-integration/jest.config.ts
@@ -4,7 +4,7 @@ import baseConfig from "../jest.config.base";
 
 const config: Config = {
   ...baseConfig,
-  runner: "<rootDir>/../jest-runner-installed-extensions.ts",
+  runner: "<rootDir>/../jest-runner-vscode-codeql-cli.ts",
   setupFilesAfterEnv: ["<rootDir>/jest.setup.ts"],
   // CLI integration tests call into the CLI and execute queries, so these are expected to take a lot longer
   // than the default 5 seconds.
diff --git a/extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts b/extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts
deleted file mode 100644
index 869ce06ba..000000000
--- a/extensions/ql-vscode/test/vscode-tests/jest-runner-installed-extensions.ts
+++ /dev/null
@@ -1,65 +0,0 @@
-import { spawnSync } from "child_process";
-import { dirname } from "path";
-
-import type * as JestRunner from "jest-runner";
-import type { RunnerOptions } from "jest-runner-vscode";
-import VSCodeTestRunner from "jest-runner-vscode";
-import { cosmiconfig } from "cosmiconfig";
-import {
-  downloadAndUnzipVSCode,
-  resolveCliArgsFromVSCodeExecutablePath,
-} from "@vscode/test-electron";
-import { ensureCli } from "./ensureCli";
-
-export default class JestRunnerInstalledExtensions extends VSCodeTestRunner {
-  async runTests(
-    tests: JestRunner.Test[],
-    watcher: JestRunner.TestWatcher,
-    onStart: JestRunner.OnTestStart,
-    onResult: JestRunner.OnTestSuccess,
-    onFailure: JestRunner.OnTestFailure,
-  ): Promise<void> {
-    // The CLI integration tests require certain extensions to be installed, which needs to happen before the tests are
-    // actually run. The below code will resolve the path to the VSCode executable, and then use that to install the
-    // required extensions.
-
-    const installedOnVsCodeVersions =
-      new Set<`${RunnerOptions["version"]}-${RunnerOptions["platform"]}`>();
-
-    for (const test of tests) {
-      const testDir = dirname(test.path);
-
-      const options: RunnerOptions =
-        ((await cosmiconfig("jest-runner-vscode").search(testDir))
-          ?.config as RunnerOptions) ?? {};
-
-      const { version, platform } = options;
-      const versionKey = `${version}-${platform}` as const;
-
-      if (installedOnVsCodeVersions.has(versionKey)) {
-        continue;
-      }
-
-      const vscodeExecutablePath = await downloadAndUnzipVSCode(
-        version,
-        platform,
-      );
-
-      console.log(`Installing required extensions for ${vscodeExecutablePath}`);
-
-      const [cli, ...args] =
-        resolveCliArgsFromVSCodeExecutablePath(vscodeExecutablePath);
-
-      spawnSync(cli, args, {
-        encoding: "utf-8",
-        stdio: "inherit",
-      });
-
-      installedOnVsCodeVersions.add(versionKey);
-    }
-
-    await ensureCli(true);
-
-    return super.runTests(tests, watcher, onStart, onResult, onFailure);
-  }
-}
diff --git a/extensions/ql-vscode/test/vscode-tests/jest-runner-vscode-codeql-cli.ts b/extensions/ql-vscode/test/vscode-tests/jest-runner-vscode-codeql-cli.ts
new file mode 100644
index 000000000..1d92e5331
--- /dev/null
+++ b/extensions/ql-vscode/test/vscode-tests/jest-runner-vscode-codeql-cli.ts
@@ -0,0 +1,20 @@
+import type * as JestRunner from "jest-runner";
+import VSCodeTestRunner from "jest-runner-vscode";
+import { ensureCli } from "./ensureCli";
+
+export default class JestRunnerVscodeCodeqlCli extends VSCodeTestRunner {
+  async runTests(
+    tests: JestRunner.Test[],
+    watcher: JestRunner.TestWatcher,
+    onStart: JestRunner.OnTestStart,
+    onResult: JestRunner.OnTestSuccess,
+    onFailure: JestRunner.OnTestFailure,
+  ): Promise<void> {
+    // The CLI integration tests require the CLI to be available. We do not want to install the CLI
+    // when VS Code is already running because this will not give any feedback to the test runner. Instead,
+    // we'll download the CLI now and pass the path to the CLI to VS Code.
+    await ensureCli(true);
+
+    return super.runTests(tests, watcher, onStart, onResult, onFailure);
+  }
+}