Add integration tests with the CLI

This commit adds integration tests that run commands using the CLI. This change introduces a number of enhancements in order to get there. 1. Augments the index-template.ts file so that it downloads an appropriate cli version if requested. 2. Adds the ensureCli.ts that performs the download if a a suitable version is not already installed. See the comments in the file for how this is done. 3. Changes how run-integration-tests is done so that the directories run are specified through a cli argument. 4. Updates the main.yml workflow so that it also runs the cli-integration tests. 5. Takes advantage of the return value of the call to `activate` on the extension. This allows the integration tests to have access to internal variables of the extension like the context, cli, and query server. 6. And of course, adds a handful of simple tests that ensure we have a cli installed of the correct version.
2020-10-26 08:34:09 -07:00 · 2020-10-26 08:34:09 -07:00 · 16eac45822
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@ -20,17 +20,17 @@ jobs:

      - uses: actions/setup-node@v1
        with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'

      - name: Install dependencies
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm install
        shell: bash

      - name: Build
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm run build
        shell: bash

@ -61,24 +61,23 @@ jobs:

      - uses: actions/setup-node@v1
        with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'

-      # We have to build the dependencies in `lib` before running any tests.
      - name: Install dependencies
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm install
        shell: bash

      - name: Build
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm run build
        shell: bash

      - name: Lint
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm run lint

      - name: Install CodeQL
@ -91,27 +90,71 @@ jobs:
        shell: bash

      - name: Run unit tests (Linux)
+        working-directory: extensions/ql-vscode
        if: matrix.os == 'ubuntu-latest'
        run: |
-          cd extensions/ql-vscode
          CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test

      - name: Run unit tests (Windows)
        if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
          npm run test

      - name: Run integration tests (Linux)
        if: matrix.os == 'ubuntu-latest'
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          sudo apt-get install xvfb
          /usr/bin/xvfb-run npm run integration

      - name: Run integration tests (Windows)
        if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
        run: |
-          cd extensions/ql-vscode
          npm run integration
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ['v2.3.1', 'v2.3.0']
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.14.0'
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          /usr/bin/xvfb-run npm run cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run cli-integration
--- a/.gitignore
+++ b/.gitignore
@ -4,6 +4,7 @@
 # Generated files
 /dist/
 out/
+build/
 server/
 node_modules/
 gen/
--- a/.vscode/launch.json
+++ b/.vscode/launch.json
@ -77,6 +77,22 @@
      "outFiles": [
        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
      ],
+    },
+    {
+      "name": "Launch Integration Tests - With CLI",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
+      ],
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "outFiles": [
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+      ],
    }
  ]
 }
--- a/extensions/ql-vscode/package.json
+++ b/extensions/ql-vscode/package.json
@ -708,7 +708,8 @@
    "watch:extension": "tsc --watch",
    "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
    "preintegration": "rm -rf ./out/vscode-tests && gulp",
-    "integration": "node ./out/vscode-tests/run-integration-tests.js",
+    "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
+    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
    "update-vscode": "node ./node_modules/vscode/bin/install",
    "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
    "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",
--- a/extensions/ql-vscode/src/astViewer.ts
+++ b/extensions/ql-vscode/src/astViewer.ts
@ -40,7 +40,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
  public db: DatabaseItem | undefined;

  private _onDidChangeTreeData =
-    new EventEmitter<AstItem | undefined>();
+    this.push(new EventEmitter<AstItem | undefined>());
  readonly onDidChangeTreeData: Event<AstItem | undefined> =
    this._onDidChangeTreeData.event;

--- a/extensions/ql-vscode/src/config.ts
+++ b/extensions/ql-vscode/src/config.ts
@ -52,7 +52,8 @@ const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];

 export interface DistributionConfig {
-  customCodeQlPath?: string;
+  readonly customCodeQlPath?: string;
+  updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
  includePrerelease: boolean;
  personalAccessToken?: string;
  ownerName?: string;
@ -149,6 +150,10 @@ export class DistributionConfigListener extends ConfigListener implements Distri
    return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
  }

+  public async updateCustomCodeQlPath(newPath: string | undefined) {
+    await CUSTOM_CODEQL_PATH_SETTING.updateValue(newPath, ConfigurationTarget.Global);
+  }
+
  protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
    this.handleDidChangeConfigurationForRelevantSettings(DISTRIBUTION_CHANGE_SETTINGS, e);
  }
--- a/extensions/ql-vscode/src/databases-ui.ts
+++ b/extensions/ql-vscode/src/databases-ui.ts
@ -81,7 +81,7 @@ class DatabaseTreeDataProvider extends DisposableObject
  implements TreeDataProvider<DatabaseItem> {
  private _sortOrder = SortOrder.NameAsc;

-  private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
+  private readonly _onDidChangeTreeData = this.push(new EventEmitter<DatabaseItem | undefined>());
  private currentDatabaseItem: DatabaseItem | undefined;

  constructor(
--- a/extensions/ql-vscode/src/distribution.ts
+++ b/extensions/ql-vscode/src/distribution.ts
@ -49,16 +49,36 @@ export interface DistributionProvider {
 }

 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._config = config;
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
+
+  /**
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  public static getRequiredAssetName(): string {
+    switch (os.platform()) {
+      case 'linux':
+        return 'codeql-linux64.zip';
+      case 'darwin':
+        return 'codeql-osx64.zip';
+      case 'win32':
+        return 'codeql-win64.zip';
+      default:
+        return 'codeql.zip';
+    }
+  }
+
+  constructor(
+    public readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    extensionContext: ExtensionContext
+  ) {
    this._onDidChangeDistribution = config.onDidChangeConfiguration;
-    this._updateCheckRateLimiter = new InvocationRateLimiter(
+    this.extensionSpecificDistributionManager =
+      new ExtensionSpecificDistributionManager(config, versionRange, extensionContext);
+    this.updateCheckRateLimiter = new InvocationRateLimiter(
      extensionContext,
      'extensionSpecificDistributionUpdateCheck',
-      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+      () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution()
    );
-    this._versionRange = versionRange;
  }

  /**
@ -95,9 +115,9 @@ export class DistributionManager implements DistributionProvider {
     * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
     * includePrerelease config option is set.
     */
-    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this.config.includePrerelease;

-    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+    if (!semver.satisfies(version, this.versionRange, { includePrerelease })) {
      return {
        distribution,
        kind: FindDistributionResultKind.IncompatibleDistribution,
@ -126,9 +146,9 @@ export class DistributionManager implements DistributionProvider {
   */
  async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
    // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath) {
-      if (!await fs.pathExists(this._config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
+    if (this.config.customCodeQlPath) {
+      if (!await fs.pathExists(this.config.customCodeQlPath)) {
+        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
          'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
          'that a CodeQL executable exists at the specified path or remove the setting.');
        return undefined;
@ -137,18 +157,18 @@ export class DistributionManager implements DistributionProvider {
      // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
      if (
        deprecatedCodeQlLauncherName() &&
-        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        this.config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
        await this.hasNewLauncherName()
      ) {
        warnDeprecatedLauncher();
      }
      return {
-        codeQlPath: this._config.customCodeQlPath,
+        codeQlPath: this.config.customCodeQlPath,
        kind: DistributionKind.CustomPathConfig
      };
    }

-    const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionSpecificCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
    if (extensionSpecificCodeQlPath !== undefined) {
      return {
        codeQlPath: extensionSpecificCodeQlPath,
@ -181,12 +201,12 @@ export class DistributionManager implements DistributionProvider {
  public async checkForUpdatesToExtensionManagedDistribution(
    minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
    const distribution = await this.getDistributionWithoutVersionCheck();
-    const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
    if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
      // A distribution is present but it isn't managed by the extension.
      return createInvalidLocationResult();
    }
-    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    const updateCheckResult = await this.updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
    switch (updateCheckResult.kind) {
      case InvocationRateLimiterResultKind.Invoked:
        return updateCheckResult.result;
@ -202,7 +222,7 @@ export class DistributionManager implements DistributionProvider {
   */
  public installExtensionManagedDistributionRelease(release: Release,
    progressCallback?: helpers.ProgressCallback): Promise<void> {
-    return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
+    return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
  }

  public get onDidChangeDistribution(): Event<void> | undefined {
@ -215,27 +235,27 @@ export class DistributionManager implements DistributionProvider {
   * installation. False otherwise.
   */
  private async hasNewLauncherName(): Promise<boolean> {
-    if (!this._config.customCodeQlPath) {
+    if (!this.config.customCodeQlPath) {
      // not managed externally
      return false;
    }
-    const dir = path.dirname(this._config.customCodeQlPath);
+    const dir = path.dirname(this.config.customCodeQlPath);
    const newLaunderPath = path.join(dir, codeQlLauncherName());
    return await fs.pathExists(newLaunderPath);
  }

-  private readonly _config: DistributionConfig;
-  private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
-  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
+  private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
  private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionRange: semver.Range;
 }

 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
-    this._extensionContext = extensionContext;
-    this._config = config;
-    this._versionRange = versionRange;
+  constructor(
+    private readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    private readonly extensionContext: ExtensionContext
+  ) {
+    /**/
  }

  public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@ -299,7 +319,7 @@ class ExtensionSpecificDistributionManager {
    }

    // Filter assets to the unique one that we require.
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
    const assets = release.assets.filter(asset => asset.name === requiredAssetName);
    if (assets.length === 0) {
      throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
@ -366,22 +386,12 @@ class ExtensionSpecificDistributionManager {
    }
  }

-  /**
-   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
-   */
-  private getRequiredAssetName(): string {
-    if (os.platform() === 'linux') return 'codeql-linux64.zip';
-    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
-    if (os.platform() === 'win32') return 'codeql-win64.zip';
-    return 'codeql.zip';
-  }
-
  private async getLatestRelease(): Promise<Release> {
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
    logger.log(`Searching for latest release including ${requiredAssetName}.`);
    return this.createReleasesApiConsumer().getLatestRelease(
-      this._versionRange,
-      this._config.includePrerelease,
+      this.versionRange,
+      this.config.includePrerelease,
      release => {
        const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
        if (matchingAssets.length === 0) {
@ -399,23 +409,23 @@ class ExtensionSpecificDistributionManager {
  }

  private createReleasesApiConsumer(): ReleasesApiConsumer {
-    const ownerName = this._config.ownerName ? this._config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
-    const repositoryName = this._config.repositoryName ? this._config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
-    return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
+    const ownerName = this.config.ownerName ? this.config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
+    const repositoryName = this.config.repositoryName ? this.config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
+    return new ReleasesApiConsumer(ownerName, repositoryName, this.config.personalAccessToken);
  }

  private async bumpDistributionFolderIndex(): Promise<void> {
-    const index = this._extensionContext.globalState.get(
+    const index = this.extensionContext.globalState.get(
      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    await this._extensionContext.globalState.update(
+    await this.extensionContext.globalState.update(
      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
  }

  private getDistributionStoragePath(): string {
    // Use an empty string for the initial distribution for backwards compatibility.
-    const distributionFolderIndex = this._extensionContext.globalState.get(
+    const distributionFolderIndex = this.extensionContext.globalState.get(
      ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
-    return path.join(this._extensionContext.globalStoragePath,
+    return path.join(this.extensionContext.globalStoragePath,
      ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
  }

@ -425,17 +435,13 @@ class ExtensionSpecificDistributionManager {
  }

  private getInstalledRelease(): Release | undefined {
-    return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
+    return this.extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
  }

  private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this.extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
  }

-  private readonly _config: DistributionConfig;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _versionRange: semver.Range;
-
  private static readonly _currentDistributionFolderBaseName = 'distribution';
  private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
  private static readonly _installedReleaseStateKey = 'distributionRelease';
@ -576,7 +582,7 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
  }));
 }

-function codeQlLauncherName(): string {
+export function codeQlLauncherName(): string {
  return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }

--- a/extensions/ql-vscode/src/extension.ts
+++ b/extensions/ql-vscode/src/extension.ts
@ -113,16 +113,37 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
  });
 }

-export async function activate(ctx: ExtensionContext): Promise<void> {
+/**
+ * The publicly available interface for this extension. This is to
+ * be used in our tests.
+ */
+export interface CodeQLExtensionInterface {
+  readonly ctx: ExtensionContext;
+  readonly cliServer: CodeQLCliServer;
+  readonly qs: qsClient.QueryServerClient;
+  readonly distributionManager: DistributionManager;
+}
+
+/**
+ * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
+ * available afer activation is complete. This will happen if there is no cli
+ * installed when the extension starts. Downloading and installing the cli
+ * will happen at a later time.
+ *
+ * @param ctx The extension context
+ *
+ * @returns CodeQLExtensionInterface
+ */
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
  logger.log('Starting CodeQL extension');

+  const distributionConfigListener = new DistributionConfigListener();
  initializeLogging(ctx);
  languageSupport.install();

-  const distributionConfigListener = new DistributionConfigListener();
  ctx.subscriptions.push(distributionConfigListener);
  const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
+  const distributionManager = new DistributionManager(distributionConfigListener, codeQlVersionRange, ctx);

  const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';

@ -253,14 +274,14 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
    return result;
  }

-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<CodeQLExtensionInterface | {}> {
    await installOrUpdateDistribution(config);

    // Display the warnings even if the extension has already activated.
    const distributionResult = await getDistributionDisplayingDistributionWarnings();
-
+    let extensionInterface: CodeQLExtensionInterface | {} = {};
    if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(ctx, distributionManager);
    } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
      registerErrorStubs([checkForUpdatesCommand], command => async () => {
        const installActionName = 'Install CodeQL CLI';
@ -268,7 +289,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
          items: [installActionName]
        });
        if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate({
+          await installOrUpdateThenTryActivate({
            isUserInitiated: true,
            shouldDisplayMessageWhenNoUpdates: false,
            allowAutoUpdating: true
@ -276,6 +297,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
        }
      });
    }
+    return extensionInterface;
  }

  ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
@ -289,7 +311,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
    allowAutoUpdating: true
  })));

-  await installOrUpdateThenTryActivate({
+  return await installOrUpdateThenTryActivate({
    isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
    shouldDisplayMessageWhenNoUpdates: false,

@ -302,7 +324,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 async function activateWithInstalledDistribution(
  ctx: ExtensionContext,
  distributionManager: DistributionManager
-): Promise<void> {
+): Promise<CodeQLExtensionInterface> {
  beganMainExtensionActivation = true;
  // Remove any error stubs command handlers left over from first part
  // of activation.
@ -354,6 +376,7 @@ async function activateWithInstalledDistribution(

  logger.log('Initializing query history manager.');
  const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  ctx.subscriptions.push(queryHistoryConfigurationListener);
  const showResults = async (item: CompletedQuery) =>
    showResultsForCompletedQuery(item, WebviewReveal.Forced);

@ -647,6 +670,13 @@ async function activateWithInstalledDistribution(
  commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');

  logger.log('Successfully finished extension initialization.');
+
+  return {
+    ctx,
+    cliServer,
+    qs,
+    distributionManager
+  };
 }

 function getContextStoragePath(ctx: ExtensionContext) {
--- a/extensions/ql-vscode/src/helpers.ts
+++ b/extensions/ql-vscode/src/helpers.ts
@ -121,7 +121,7 @@ export function commandRunner(
 ): Disposable {
  return commands.registerCommand(commandId, async (...args: any[]) => {
    try {
-      await task(...args);
+      return await task(...args);
    } catch (e) {
      if (e instanceof UserCancellationException) {
        // User has cancelled this action manually
@ -133,6 +133,7 @@ export function commandRunner(
      } else {
        showAndLogErrorMessage(e.message || e);
      }
+      return undefined;
    }
  });
 }
@ -158,7 +159,7 @@ export function commandRunnerWithProgress<R>(
      ...progressOptions
    };
    try {
-      await withProgress(progressOptionsWithDefaults, task, ...args);
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
    } catch (e) {
      if (e instanceof UserCancellationException) {
        // User has cancelled this action manually
@ -170,6 +171,7 @@ export function commandRunnerWithProgress<R>(
      } else {
        showAndLogErrorMessage(e.message || e);
      }
+      return undefined;
    }
  });
 }
--- a/extensions/ql-vscode/src/query-history.ts
+++ b/extensions/ql-vscode/src/query-history.ts
@ -59,16 +59,15 @@ interface QueryHistoryDataProvider extends vscode.TreeDataProvider<CompletedQuer
 /**
 * Tree data provider for the query history view.
 */
-class HistoryTreeDataProvider implements QueryHistoryDataProvider {
+class HistoryTreeDataProvider extends DisposableObject implements QueryHistoryDataProvider {
  /**
   * XXX: This idiom for how to get a `.fire()`-able event emitter was
   * cargo culted from another vscode extension. It seems rather
   * involved and I hope there's something better that can be done
   * instead.
   */
-  private _onDidChangeTreeData: vscode.EventEmitter<
-    CompletedQuery | undefined
-  > = new vscode.EventEmitter<CompletedQuery | undefined>();
+  private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
+
  readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
    ._onDidChangeTreeData.event;

@ -82,6 +81,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
  private current: CompletedQuery | undefined;

  constructor(extensionPath: string) {
+    super();
    this.failedIconPath = path.join(
      extensionPath,
      FAILED_QUERY_HISTORY_ITEM_ICON
@ -135,7 +135,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
    return this.current;
  }

-  push(item: CompletedQuery): void {
+  pushQuery(item: CompletedQuery): void {
    this.current = item;
    this.history.push(item);
    this.refresh();
@ -204,6 +204,7 @@ export class QueryHistoryManager extends DisposableObject {
      canSelectMany: true,
    });
    this.push(this.treeView);
+    this.push(treeDataProvider);

    // Lazily update the tree view selection due to limitations of TreeView API (see
    // `updateTreeViewSelectionIfVisible` doc for details)
@ -513,7 +514,7 @@ export class QueryHistoryManager extends DisposableObject {

  addQuery(info: QueryWithResults): CompletedQuery {
    const item = new CompletedQuery(info, this.queryHistoryConfigListener);
-    this.treeDataProvider.push(item);
+    this.treeDataProvider.pushQuery(item);
    this.updateTreeViewSelectionIfVisible();
    return item;
  }
--- a/extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts
+++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts
@ -0,0 +1,4 @@
+import { runTestsInDirectory } from '../index-template';
+export function run(): Promise<void> {
+  return runTestsInDirectory(__dirname, true);
+}
--- a/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts
@ -0,0 +1,51 @@
+import 'mocha';
+import 'sinon-chai';
+
+
+import { expect } from 'chai';
+import { ConfigurationTarget, workspace, extensions } from 'vscode';
+import { SemVer } from 'semver';
+
+import { CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+
+
+/**
+ * Perform proper integration tests by running the CLI
+ */
+describe('Use cli', function() {
+  this.timeout(60000);
+
+  let cli: CodeQLCliServer;
+
+  beforeEach(async () => {
+    // Set it here before activation to ensure we don't accidentally try to download a cli
+    await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+    }
+  });
+
+  afterEach(() => {
+    cli.dispose();
+  });
+
+  it('should have the correct version of the cli', async () => {
+    expect(
+      (await cli.getVersion()).toString()
+    ).to.eq(
+      new SemVer(process.env.CLI_VERSION || '').toString()
+    );
+  });
+
+  it('should resolve ram', async () => {
+    const result = await (cli as any).resolveRam(8192);
+    expect(result).to.deep.eq([
+      '-J-Xmx4096M',
+      '--off-heap-ram=4096'
+    ]);
+  });
+});
--- a/extensions/ql-vscode/src/vscode-tests/ensureCli.ts
+++ b/extensions/ql-vscode/src/vscode-tests/ensureCli.ts
@ -0,0 +1,129 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { DistributionManager, extractZipArchive, codeQlLauncherName } from '../distribution';
+import fetch from 'node-fetch';
+
+/**
+ * This module ensures that the proper CLI is available for tests of the extension.
+ * There are two environment variables to control this module:
+ *
+ *     - CLI_VERSION: The version of the CLI to install. Defaults to the most recent
+ *       version. Note that for now, we must maintain the default version by hand.
+ *
+ *     - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped.
+ *       The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is
+ *       `${CLI_BASE_DIR}/${CLI_VERSION}`
+ *
+ * After downloading and unzipping, a new environment variable is set:
+ *
+ *     - CLI_PATH: Points to the cli executable for the specified CLI_VERSION. This
+ *       is variable is available in the unit tests and will be used as the value
+ *       for `codeQL.cli.executablePath`.
+ *
+ * As an optimization, the cli will not be unzipped again if the executable already
+ * exists. And the cli will not be re-downloaded if the zip already exists.
+ */
+
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  process.exit(-1);
+});
+
+const _1MB = 1024 * 1024;
+const _10MB = _1MB * 10;
+
+// CLI version to test. Hard code the latest as default. And be sure
+// to update the env if it is not otherwise set.
+const CLI_VERSION = process.env.CLI_VERSION || 'v2.3.1';
+process.env.CLI_VERSION = CLI_VERSION;
+
+// Base dir where CLIs will be downloaded into
+// By default, put it in the `build` directory in the root of the extension.
+const CLI_BASE_DIR = process.env.CLI_DIR || path.normalize(path.join(__dirname, '../../build/cli'));
+
+export async function ensureCli(useCli: boolean) {
+  try {
+    if (!useCli) {
+      console.log('Not downloading CLI. It is not being used.');
+      return;
+    }
+
+    const assetName = DistributionManager.getRequiredAssetName();
+    const url = getCliDownloadUrl(assetName);
+    const unzipDir = getCliUnzipDir();
+    const downloadedFilePath = getDownloadFilePath(assetName);
+    const executablePath = path.join(getCliUnzipDir(), 'codeql', codeQlLauncherName());
+
+    // Use this environment variable to se to the `codeQL.cli.executablePath` in tests
+    process.env.CLI_PATH = executablePath;
+
+    if (fs.existsSync(executablePath)) {
+      console.log(`CLI version ${CLI_VERSION} is found ${executablePath}. Not going to download again.`);
+      return;
+    }
+
+    if (!fs.existsSync(downloadedFilePath)) {
+      console.log(`CLI version ${CLI_VERSION} zip file not found. Downloading from '${url}' into '${downloadedFilePath}'.`);
+
+      const assetStream = await fetch(url);
+      const contentLength = Number(assetStream.headers.get('content-length') || 0);
+      console.log('Total content size', Math.round(contentLength / _1MB), 'MB');
+      const archiveFile = fs.createWriteStream(downloadedFilePath);
+      const body = assetStream.body;
+      await new Promise((resolve, reject) => {
+        let numBytesDownloaded = 0;
+        let lastMessage = 0;
+        body.on('data', (data) => {
+          numBytesDownloaded += data.length;
+          if (numBytesDownloaded - lastMessage > _10MB) {
+            console.log('Downloaded', Math.round(numBytesDownloaded / _1MB), 'MB');
+            lastMessage = numBytesDownloaded;
+          }
+          archiveFile.write(data);
+        });
+        body.on('finish', () => {
+          archiveFile.end(() => {
+            console.log('Finished download into', downloadedFilePath);
+            resolve();
+          });
+        });
+        body.on('error', reject);
+      });
+    } else {
+      console.log(`CLI version ${CLI_VERSION} zip file found at '${downloadedFilePath}'.`);
+    }
+
+    console.log(`Unzipping into '${unzipDir}'`);
+    fs.mkdirpSync(unzipDir);
+    await extractZipArchive(downloadedFilePath, unzipDir);
+    console.log('Done.');
+  } catch (e) {
+    console.error('Failed to download CLI.');
+    console.error(e);
+    process.exit(-1);
+  }
+}
+
+/**
+ * Url to download from
+ */
+function getCliDownloadUrl(assetName: string) {
+  return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`;
+}
+
+/**
+ * Directory to place the downloaded cli into
+ */
+function getDownloadFilePath(assetName: string) {
+  const dir = path.join(CLI_BASE_DIR, 'assets');
+  fs.mkdirpSync(dir);
+  return path.join(dir, assetName);
+}
+
+/**
+ * Directory to unzip the downloaded cli into.
+ */
+function getCliUnzipDir() {
+  return path.join(CLI_BASE_DIR, CLI_VERSION);
+}
--- a/extensions/ql-vscode/src/vscode-tests/index-template.ts
+++ b/extensions/ql-vscode/src/vscode-tests/index-template.ts
@ -1,6 +1,18 @@
 import * as path from 'path';
 import * as Mocha from 'mocha';
 import * as glob from 'glob';
+import { ensureCli } from './ensureCli';
+
+
+// Use this handler to avoid swallowing unhandled rejections.
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  // Must use a setTimeout in order to ensure the log is fully flushed before exiting
+  setTimeout(() => {
+    process.exit(-1);
+  }, 2000);
+});

 /**
 * Helper function that runs all Mocha tests found in the
@ -26,13 +38,15 @@ import * as glob from 'glob';
 * After https://github.com/microsoft/TypeScript/issues/420 is implemented,
 * this pattern can be expressed more neatly using a module interface.
 */
-export function runTestsInDirectory(testsRoot: string): Promise<void> {
+export async function runTestsInDirectory(testsRoot: string, useCli = false): Promise<void> {
  // Create the mocha test
  const mocha = new Mocha({
    ui: 'bdd',
    color: true
  });

+  await ensureCli(useCli);
+
  return new Promise((c, e) => {
    console.log(`Adding test cases from ${testsRoot}`);
    glob('**/**.test.js', { cwd: testsRoot }, (err, files) => {
--- a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts
@ -9,6 +9,7 @@ import * as determiningSelectedQueryTest from './determining-selected-query-test
 chai.use(chaiAsPromised);

 describe('launching with a minimal workspace', async () => {
+
  const ext = vscode.extensions.getExtension('GitHub.vscode-codeql');
  it('should install the extension', () => {
    assert(ext);
@ -19,6 +20,9 @@ describe('launching with a minimal workspace', async () => {
  });

  it('should activate the extension when a .ql file is opened', async function() {
+    this.timeout(60000);
+    await delay();
+
    const folders = vscode.workspace.workspaceFolders;
    assert(folders && folders.length === 1);
    const folderPath = folders![0].uri.fsPath;
@ -26,10 +30,13 @@ describe('launching with a minimal workspace', async () => {
    const document = await vscode.workspace.openTextDocument(documentPath);
    assert(document.languageId === 'ql');
    // Delay slightly so that the extension has time to activate.
-    this.timeout(4000);
-    await new Promise(resolve => setTimeout(resolve, 2000));
+    await delay();
    assert(ext!.isActive);
  });
+
+  async function delay() {
+    await new Promise(resolve => setTimeout(resolve, 4000));
+  }
 });

 determiningSelectedQueryTest.run();
--- a/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts
@ -41,8 +41,6 @@ describe('databases', () => {
  let sandbox: sinon.SinonSandbox;
  let dir: tmp.DirResult;

-
-
  beforeEach(() => {
    dir = tmp.dirSync();

@ -86,6 +84,7 @@ describe('databases', () => {

  afterEach(async () => {
    dir.removeCallback();
+    databaseManager.dispose();
    sandbox.restore();
  });

@ -425,7 +424,7 @@ describe('databases', () => {
        datasetUri: databaseUri
      } as DatabaseContents,
      MOCK_DB_OPTIONS,
-      dbChangedHandler
+      dbChangedHandler,
    );
  }

--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts
@ -15,19 +15,25 @@ const expect = chai.expect;

 describe('AstViewer', () => {
  let astRoots: AstItem[];
-  let viewer: AstViewer;
+  let viewer: AstViewer | undefined;
+  let sandbox: sinon.SinonSandbox;
  beforeEach(async () => {
+    sandbox = sinon.createSandbox();
    // the ast is stored in yaml because there are back pointers
    // making a json representation impossible.
    // The complication here is that yaml files are not copied into the 'out' directory by tsc.
    astRoots = await buildAst();

-    sinon.stub(commands, 'registerCommand');
-    sinon.stub(commands, 'executeCommand');
+    sandbox.stub(commands, 'registerCommand');
+    sandbox.stub(commands, 'executeCommand');
  });

  afterEach(() => {
-    sinon.restore();
+    sandbox.restore();
+    if (viewer) {
+      viewer.dispose();
+      viewer = undefined;
+    }
  });

  it('should update the viewer roots', () => {
@ -56,7 +62,6 @@ describe('AstViewer', () => {
    doSelectionTest(undefined, new Range(2, 3, 4, 5));
  });

-
  function doSelectionTest(
    expectedSelection: any,
    selectionRange: Range | undefined,
@ -65,7 +70,7 @@ describe('AstViewer', () => {
    const item = {} as DatabaseItem;
    viewer = new AstViewer();
    viewer.updateRoots(astRoots, item, fsPath);
-    const spy = sinon.spy();
+    const spy = sandbox.spy();
    (viewer as any).treeView.reveal = spy;
    Object.defineProperty((viewer as any).treeView, 'visible', {
      value: true
--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts
@ -252,8 +252,12 @@ describe('Launcher path', () => {
    expect(result).to.equal(undefined);
  });

-  it('should not warn when deprecated launcher is used, but no new launcher is available', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+  it('should not warn when deprecated launcher is used, but no new launcher is available', async function() {
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
    launcherThatExists = 'codeql.cmd';

    const result = await manager.getCodeQlPathWithoutVersionCheck();
@ -265,7 +269,11 @@ describe('Launcher path', () => {
  });

  it('should warn when deprecated launcher is used, and new launcher is available', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
    launcherThatExists = ''; // pretend both launchers exist

    const result = await manager.getCodeQlPathWithoutVersionCheck();
@ -277,7 +285,11 @@ describe('Launcher path', () => {
  });

  it('should warn when launcher path is incorrect', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
    launcherThatExists = 'xxx'; // pretend neither launcher exists

    const result = await manager.getCodeQlPathWithoutVersionCheck();
--- a/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
+++ b/extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts
@ -23,6 +23,7 @@ describe('query-history', () => {

  beforeEach(() => {
    sandbox = sinon.createSandbox();
+
    showTextDocumentSpy = sandbox.stub(vscode.window, 'showTextDocument');
    showInformationMessageSpy = sandbox.stub(
      vscode.window,
--- a/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
+++ b/extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts
@ -1,16 +1,17 @@
 import * as path from 'path';
 import * as os from 'os';
-import { runTests } from 'vscode-test';
+import * as cp from 'child_process';
+import {
+  runTests,
+  downloadAndUnzipVSCode,
+  resolveCliPathFromVSCodeExecutablePath
+} from 'vscode-test';
+import { assertNever } from '../helpers-pure';
+
+// For some reason, `TestOptions` is not exported directly from `vscode-test`,
+// but we can be tricky and import directly from the out file.
+import { TestOptions } from 'vscode-test/out/runTest';

-// A subset of the fields in TestOptions from vscode-test, which we
-// would simply use instead, but for the fact that it doesn't export
-// it.
-type Suite = {
-  extensionDevelopmentPath: string;
-  extensionTestsPath: string;
-  launchArgs: string[];
-  version?: string;
-};

 // Which version of vscode to test against. Can set to 'stable' or
 // 'insiders' or an explicit version number. See runTest.d.ts in
@ -22,11 +23,21 @@ type Suite = {
 // testing against old versions if necessary.
 const VSCODE_VERSION = 'stable';

+// List if test dirs
+//   - no-workspace - Tests with no workspace selected upon launch.
+//   - minimal-workspace - Tests with a simple workspace selected upon launch.
+//   - cli-integration - Tests that require a cli to invoke actual commands
+enum TestDir {
+  NoWorskspace = 'no-workspace',
+  MinimalWorskspace = 'minimal-workspace',
+  CliIntegration = 'cli-integration'
+}
+
 /**
 * Run an integration test suite `suite`, retrying if it segfaults, at
 * most `tries` times.
 */
-async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise<void> {
+async function runTestsWithRetryOnSegfault(suite: TestOptions, tries: number): Promise<void> {
  for (let t = 0; t < tries; t++) {
    try {
      // Download and unzip VS Code if necessary, and run the integration test suite.
@ -58,34 +69,33 @@ async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise
 */
 async function main() {
  try {
-    // The folder containing the Extension Manifest package.json
-    // Passed to `--extensionDevelopmentPath`.
    const extensionDevelopmentPath = path.resolve(__dirname, '../..');
+    const vscodeExecutablePath = await downloadAndUnzipVSCode(VSCODE_VERSION);

-    // List of integration test suites.
-    // The path to the extension test runner script is passed to --extensionTestsPath.
-    const integrationTestSuites: Suite[] = [
-      // Tests with no workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'no-workspace', 'index'),
-        launchArgs: ['--disable-extensions'],
-      },
-      // Tests with a simple workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'minimal-workspace', 'index'),
-        launchArgs: [
-          path.resolve(__dirname, '../../test/data'),
-          '--disable-extensions',
-        ]
-      }
-    ];

-    for (const integrationTestSuite of integrationTestSuites) {
-      await runTestsWithRetryOnSegfault(integrationTestSuite, 3);
+    // Which tests to run. Use a comma-separated list of directories.
+    const testDirsString = process.argv[2];
+    const dirs = testDirsString.split(',').map(dir => dir.trim().toLocaleLowerCase());
+
+    if (dirs.includes(TestDir.CliIntegration)) {
+      console.log('Installing required extensions');
+      const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
+      cp.spawnSync(cliPath, ['--install-extension', 'hbenl.vscode-test-explorer'], {
+        encoding: 'utf-8',
+        stdio: 'inherit'
+      });
+    }
+
+    console.log(`Running integration tests in these directories: ${dirs}`);
+    for (const dir of dirs) {
+      console.log(`Next integration test dir: ${dir}`);
+      await runTestsWithRetryOnSegfault({
+        version: VSCODE_VERSION,
+        vscodeExecutablePath,
+        extensionDevelopmentPath,
+        extensionTestsPath: path.resolve(__dirname, dir, 'index'),
+        launchArgs: getLaunchArgs(dir as TestDir)
+      }, 3);
    }
  } catch (err) {
    console.error(`Unexpected exception while running tests: ${err}`);
@ -94,3 +104,25 @@ async function main() {
 }

 main();
+
+
+function getLaunchArgs(dir: TestDir) {
+  switch (dir) {
+    case TestDir.NoWorskspace:
+      return [
+        '--disable-extensions'
+      ];
+
+    case TestDir.MinimalWorskspace:
+      return [
+        '--disable-extensions',
+        path.resolve(__dirname, '../../test/data')
+      ];
+
+    case TestDir.CliIntegration:
+      return undefined;
+
+    default:
+      assertNever(dir);
+  }
+}
--- a/extensions/ql-vscode/test/data/.gitignore
+++ b/extensions/ql-vscode/test/data/.gitignore
@ -0,0 +1 @@
+.vscode