Add integration tests with the CLI

This commit adds integration tests that run commands using the CLI. This
change introduces a number of enhancements in order to get there.

1. Augments the index-template.ts file so that it downloads an
appropriate cli version if requested.
2. Adds the ensureCli.ts that performs the download if a a suitable
version is not already installed. See the comments in the file for how
this is done.
3. Changes how run-integration-tests is done so that the directories
run are specified through a cli argument.
4. Updates the main.yml workflow so that it also runs the
cli-integration tests.
5. Takes advantage of the return value of the call to `activate` on the
extension. This allows the integration tests to have access to internal
variables of the extension like the context, cli, and query server.
6. And of course, adds a handful of simple tests that ensure we have a
cli installed of the correct version.

.github/workflows/main.yml

@@ -20,17 +20,17 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
@@ -61,24 +61,23 @@ jobs:
 
       - uses: actions/setup-node@v1
         with:
-          node-version: '10.18.1'
+          node-version: '14.14.0'
 
-      # We have to build the dependencies in `lib` before running any tests.
       - name: Install dependencies
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm install
         shell: bash
 
       - name: Build
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run build
         shell: bash
 
       - name: Lint
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run lint
 
       - name: Install CodeQL
@@ -91,27 +90,71 @@ jobs:
         shell: bash
 
       - name: Run unit tests (Linux)
+        working-directory: extensions/ql-vscode
         if: matrix.os == 'ubuntu-latest'
         run: |
-          cd extensions/ql-vscode
           CODEQL_PATH=$GITHUB_WORKSPACE/codeql-home/codeql/codeql npm run test
 
       - name: Run unit tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           $env:CODEQL_PATH=$(Join-Path $env:GITHUB_WORKSPACE -ChildPath 'codeql-home/codeql/codeql.exe')
           npm run test
 
       - name: Run integration tests (Linux)
         if: matrix.os == 'ubuntu-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           sudo apt-get install xvfb
           /usr/bin/xvfb-run npm run integration
 
       - name: Run integration tests (Windows)
         if: matrix.os == 'windows-latest'
+        working-directory: extensions/ql-vscode
         run: |
-          cd extensions/ql-vscode
           npm run integration
+
+  cli-test:
+    name: CLI Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, windows-latest]
+        version: ['v2.3.1', 'v2.3.0']
+    env:
+      CLI_VERSION: ${{ matrix.version }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+        with:
+          fetch-depth: 1
+
+      - uses: actions/setup-node@v1
+        with:
+          node-version: '14.14.0'
+
+      - name: Install dependencies
+        working-directory: extensions/ql-vscode
+        run: |
+          npm install
+        shell: bash
+
+      - name: Build
+        working-directory: extensions/ql-vscode
+        run: |
+          npm run build
+        shell: bash
+
+      - name: Run CLI tests (Linux)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'ubuntu-latest'
+        run: |
+          /usr/bin/xvfb-run npm run cli-integration
+
+      - name: Run CLI tests (Windows)
+        working-directory: extensions/ql-vscode
+        if: matrix.os == 'windows-latest'
+        run: |
+          npm run cli-integration

.gitignore

@@ -4,6 +4,7 @@
 # Generated files
 /dist/
 out/
+build/
 server/
 node_modules/
 gen/

.vscode/launch.json

@@ -77,6 +77,22 @@
       "outFiles": [
         "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
       ],
+    },
+    {
+      "name": "Launch Integration Tests - With CLI",
+      "type": "extensionHost",
+      "request": "launch",
+      "runtimeExecutable": "${execPath}",
+      "args": [
+        "--extensionDevelopmentPath=${workspaceRoot}/extensions/ql-vscode",
+        "--extensionTestsPath=${workspaceRoot}/extensions/ql-vscode/out/vscode-tests/cli-integration/index",
+        "${workspaceRoot}/extensions/ql-vscode/test/data"
+      ],
+      "stopOnEntry": false,
+      "sourceMaps": true,
+      "outFiles": [
+        "${workspaceRoot}/extensions/ql-vscode/out/**/*.js",
+      ],
     }
   ]
 }

extensions/ql-vscode/package.json

@@ -708,7 +708,8 @@
     "watch:extension": "tsc --watch",
     "test": "mocha --exit -r ts-node/register test/pure-tests/**/*.ts",
     "preintegration": "rm -rf ./out/vscode-tests && gulp",
-    "integration": "node ./out/vscode-tests/run-integration-tests.js",
+    "integration": "node ./out/vscode-tests/run-integration-tests.js no-workspace,minimal-workspace",
+    "cli-integration": "npm run preintegration && node ./out/vscode-tests/run-integration-tests.js cli-integration",
     "update-vscode": "node ./node_modules/vscode/bin/install",
     "format": "tsfmt -r && eslint src test --ext .ts,.tsx --fix",
     "lint": "eslint src test --ext .ts,.tsx --max-warnings=0",

extensions/ql-vscode/src/astViewer.ts

@@ -40,7 +40,7 @@ class AstViewerDataProvider extends DisposableObject implements TreeDataProvider
   public db: DatabaseItem | undefined;
 
   private _onDidChangeTreeData =
-    new EventEmitter<AstItem | undefined>();
+    this.push(new EventEmitter<AstItem | undefined>());
   readonly onDidChangeTreeData: Event<AstItem | undefined> =
     this._onDidChangeTreeData.event;
 

extensions/ql-vscode/src/config.ts

@@ -52,7 +52,8 @@ const QUERY_HISTORY_FORMAT_SETTING = new Setting('format', QUERY_HISTORY_SETTING
 const DISTRIBUTION_CHANGE_SETTINGS = [CUSTOM_CODEQL_PATH_SETTING, INCLUDE_PRERELEASE_SETTING, PERSONAL_ACCESS_TOKEN_SETTING];
 
 export interface DistributionConfig {
-  customCodeQlPath?: string;
+  readonly customCodeQlPath?: string;
+  updateCustomCodeQlPath: (newPath: string | undefined) => Promise<void>;
   includePrerelease: boolean;
   personalAccessToken?: string;
   ownerName?: string;
@@ -149,6 +150,10 @@ export class DistributionConfigListener extends ConfigListener implements Distri
     return PERSONAL_ACCESS_TOKEN_SETTING.getValue() || undefined;
   }
 
+  public async updateCustomCodeQlPath(newPath: string | undefined) {
+    await CUSTOM_CODEQL_PATH_SETTING.updateValue(newPath, ConfigurationTarget.Global);
+  }
+
   protected handleDidChangeConfiguration(e: ConfigurationChangeEvent): void {
     this.handleDidChangeConfigurationForRelevantSettings(DISTRIBUTION_CHANGE_SETTINGS, e);
   }

extensions/ql-vscode/src/databases-ui.ts

@@ -81,7 +81,7 @@ class DatabaseTreeDataProvider extends DisposableObject
   implements TreeDataProvider<DatabaseItem> {
   private _sortOrder = SortOrder.NameAsc;
 
-  private readonly _onDidChangeTreeData = new EventEmitter<DatabaseItem | undefined>();
+  private readonly _onDidChangeTreeData = this.push(new EventEmitter<DatabaseItem | undefined>());
   private currentDatabaseItem: DatabaseItem | undefined;
 
   constructor(

extensions/ql-vscode/src/distribution.ts

@@ -49,16 +49,36 @@ export interface DistributionProvider {
 }
 
 export class DistributionManager implements DistributionProvider {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
+
-    this._config = config;
+  /**
-    this._extensionSpecificDistributionManager = new ExtensionSpecificDistributionManager(extensionContext, config, versionRange);
+   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
+   */
+  public static getRequiredAssetName(): string {
+    switch (os.platform()) {
+      case 'linux':
+        return 'codeql-linux64.zip';
+      case 'darwin':
+        return 'codeql-osx64.zip';
+      case 'win32':
+        return 'codeql-win64.zip';
+      default:
+        return 'codeql.zip';
+    }
+  }
+
+  constructor(
+    public readonly config: DistributionConfig,
+    private readonly versionRange: semver.Range,
+    extensionContext: ExtensionContext
+  ) {
     this._onDidChangeDistribution = config.onDidChangeConfiguration;
-    this._updateCheckRateLimiter = new InvocationRateLimiter(
+    this.extensionSpecificDistributionManager =
+      new ExtensionSpecificDistributionManager(config, versionRange, extensionContext);
+    this.updateCheckRateLimiter = new InvocationRateLimiter(
       extensionContext,
       'extensionSpecificDistributionUpdateCheck',
-      () => this._extensionSpecificDistributionManager.checkForUpdatesToDistribution()
+      () => this.extensionSpecificDistributionManager.checkForUpdatesToDistribution()
     );
-    this._versionRange = versionRange;
   }
 
   /**
@@ -95,9 +115,9 @@ export class DistributionManager implements DistributionProvider {
      * - If the user is using an extension-managed CLI, then prereleases are only accepted when the
      * includePrerelease config option is set.
      */
-    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this._config.includePrerelease;
+    const includePrerelease = distribution.kind !== DistributionKind.ExtensionManaged || this.config.includePrerelease;
 
-    if (!semver.satisfies(version, this._versionRange, { includePrerelease })) {
+    if (!semver.satisfies(version, this.versionRange, { includePrerelease })) {
       return {
         distribution,
         kind: FindDistributionResultKind.IncompatibleDistribution,
@@ -126,9 +146,9 @@ export class DistributionManager implements DistributionProvider {
    */
   async getDistributionWithoutVersionCheck(): Promise<Distribution | undefined> {
     // Check config setting, then extension specific distribution, then PATH.
-    if (this._config.customCodeQlPath) {
+    if (this.config.customCodeQlPath) {
-      if (!await fs.pathExists(this._config.customCodeQlPath)) {
+      if (!await fs.pathExists(this.config.customCodeQlPath)) {
-        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this._config.customCodeQlPath}" ` +
+        showAndLogErrorMessage(`The CodeQL executable path is specified as "${this.config.customCodeQlPath}" ` +
           'by a configuration setting, but a CodeQL executable could not be found at that path. Please check ' +
           'that a CodeQL executable exists at the specified path or remove the setting.');
         return undefined;
@@ -137,18 +157,18 @@ export class DistributionManager implements DistributionProvider {
       // emit a warning if using a deprecated launcher and a non-deprecated launcher exists
       if (
         deprecatedCodeQlLauncherName() &&
-        this._config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
+        this.config.customCodeQlPath.endsWith(deprecatedCodeQlLauncherName()!) &&
         await this.hasNewLauncherName()
       ) {
         warnDeprecatedLauncher();
       }
       return {
-        codeQlPath: this._config.customCodeQlPath,
+        codeQlPath: this.config.customCodeQlPath,
         kind: DistributionKind.CustomPathConfig
       };
     }
 
-    const extensionSpecificCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionSpecificCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (extensionSpecificCodeQlPath !== undefined) {
       return {
         codeQlPath: extensionSpecificCodeQlPath,
@@ -181,12 +201,12 @@ export class DistributionManager implements DistributionProvider {
   public async checkForUpdatesToExtensionManagedDistribution(
     minSecondsSinceLastUpdateCheck: number): Promise<DistributionUpdateCheckResult> {
     const distribution = await this.getDistributionWithoutVersionCheck();
-    const extensionManagedCodeQlPath = await this._extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
+    const extensionManagedCodeQlPath = await this.extensionSpecificDistributionManager.getCodeQlPathWithoutVersionCheck();
     if (distribution?.codeQlPath !== extensionManagedCodeQlPath) {
       // A distribution is present but it isn't managed by the extension.
       return createInvalidLocationResult();
     }
-    const updateCheckResult = await this._updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
+    const updateCheckResult = await this.updateCheckRateLimiter.invokeFunctionIfIntervalElapsed(minSecondsSinceLastUpdateCheck);
     switch (updateCheckResult.kind) {
       case InvocationRateLimiterResultKind.Invoked:
         return updateCheckResult.result;
@@ -202,7 +222,7 @@ export class DistributionManager implements DistributionProvider {
    */
   public installExtensionManagedDistributionRelease(release: Release,
     progressCallback?: helpers.ProgressCallback): Promise<void> {
-    return this._extensionSpecificDistributionManager.installDistributionRelease(release, progressCallback);
+    return this.extensionSpecificDistributionManager!.installDistributionRelease(release, progressCallback);
   }
 
   public get onDidChangeDistribution(): Event<void> | undefined {
@@ -215,27 +235,27 @@ export class DistributionManager implements DistributionProvider {
    * installation. False otherwise.
    */
   private async hasNewLauncherName(): Promise<boolean> {
-    if (!this._config.customCodeQlPath) {
+    if (!this.config.customCodeQlPath) {
       // not managed externally
       return false;
     }
-    const dir = path.dirname(this._config.customCodeQlPath);
+    const dir = path.dirname(this.config.customCodeQlPath);
     const newLaunderPath = path.join(dir, codeQlLauncherName());
     return await fs.pathExists(newLaunderPath);
   }
 
-  private readonly _config: DistributionConfig;
+  private readonly extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
-  private readonly _extensionSpecificDistributionManager: ExtensionSpecificDistributionManager;
+  private readonly updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
-  private readonly _updateCheckRateLimiter: InvocationRateLimiter<DistributionUpdateCheckResult>;
   private readonly _onDidChangeDistribution: Event<void> | undefined;
-  private readonly _versionRange: semver.Range;
 }
 
 class ExtensionSpecificDistributionManager {
-  constructor(extensionContext: ExtensionContext, config: DistributionConfig, versionRange: semver.Range) {
+  constructor(
-    this._extensionContext = extensionContext;
+    private readonly config: DistributionConfig,
-    this._config = config;
+    private readonly versionRange: semver.Range,
-    this._versionRange = versionRange;
+    private readonly extensionContext: ExtensionContext
+  ) {
+    /**/
   }
 
   public async getCodeQlPathWithoutVersionCheck(): Promise<string | undefined> {
@@ -299,7 +319,7 @@ class ExtensionSpecificDistributionManager {
     }
 
     // Filter assets to the unique one that we require.
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
     const assets = release.assets.filter(asset => asset.name === requiredAssetName);
     if (assets.length === 0) {
       throw new Error(`Invariant violation: chose a release to install that didn't have ${requiredAssetName}`);
@@ -366,22 +386,12 @@ class ExtensionSpecificDistributionManager {
     }
   }
 
-  /**
-   * Get the name of the codeql cli installation we prefer to install, based on our current platform.
-   */
-  private getRequiredAssetName(): string {
-    if (os.platform() === 'linux') return 'codeql-linux64.zip';
-    if (os.platform() === 'darwin') return 'codeql-osx64.zip';
-    if (os.platform() === 'win32') return 'codeql-win64.zip';
-    return 'codeql.zip';
-  }
-
   private async getLatestRelease(): Promise<Release> {
-    const requiredAssetName = this.getRequiredAssetName();
+    const requiredAssetName = DistributionManager.getRequiredAssetName();
     logger.log(`Searching for latest release including ${requiredAssetName}.`);
     return this.createReleasesApiConsumer().getLatestRelease(
-      this._versionRange,
+      this.versionRange,
-      this._config.includePrerelease,
+      this.config.includePrerelease,
       release => {
         const matchingAssets = release.assets.filter(asset => asset.name === requiredAssetName);
         if (matchingAssets.length === 0) {
@@ -399,23 +409,23 @@ class ExtensionSpecificDistributionManager {
   }
 
   private createReleasesApiConsumer(): ReleasesApiConsumer {
-    const ownerName = this._config.ownerName ? this._config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
+    const ownerName = this.config.ownerName ? this.config.ownerName : DEFAULT_DISTRIBUTION_OWNER_NAME;
-    const repositoryName = this._config.repositoryName ? this._config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
+    const repositoryName = this.config.repositoryName ? this.config.repositoryName : DEFAULT_DISTRIBUTION_REPOSITORY_NAME;
-    return new ReleasesApiConsumer(ownerName, repositoryName, this._config.personalAccessToken);
+    return new ReleasesApiConsumer(ownerName, repositoryName, this.config.personalAccessToken);
   }
 
   private async bumpDistributionFolderIndex(): Promise<void> {
-    const index = this._extensionContext.globalState.get(
+    const index = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0);
-    await this._extensionContext.globalState.update(
+    await this.extensionContext.globalState.update(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, index + 1);
   }
 
   private getDistributionStoragePath(): string {
     // Use an empty string for the initial distribution for backwards compatibility.
-    const distributionFolderIndex = this._extensionContext.globalState.get(
+    const distributionFolderIndex = this.extensionContext.globalState.get(
       ExtensionSpecificDistributionManager._currentDistributionFolderIndexStateKey, 0) || '';
-    return path.join(this._extensionContext.globalStoragePath,
+    return path.join(this.extensionContext.globalStoragePath,
       ExtensionSpecificDistributionManager._currentDistributionFolderBaseName + distributionFolderIndex);
   }
 
@@ -425,17 +435,13 @@ class ExtensionSpecificDistributionManager {
   }
 
   private getInstalledRelease(): Release | undefined {
-    return this._extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
+    return this.extensionContext.globalState.get(ExtensionSpecificDistributionManager._installedReleaseStateKey);
   }
 
   private async storeInstalledRelease(release: Release | undefined): Promise<void> {
-    await this._extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
+    await this.extensionContext.globalState.update(ExtensionSpecificDistributionManager._installedReleaseStateKey, release);
   }
 
-  private readonly _config: DistributionConfig;
-  private readonly _extensionContext: ExtensionContext;
-  private readonly _versionRange: semver.Range;
-
   private static readonly _currentDistributionFolderBaseName = 'distribution';
   private static readonly _currentDistributionFolderIndexStateKey = 'distributionFolderIndex';
   private static readonly _installedReleaseStateKey = 'distributionRelease';
@@ -576,7 +582,7 @@ export async function extractZipArchive(archivePath: string, outPath: string): P
   }));
 }
 
-function codeQlLauncherName(): string {
+export function codeQlLauncherName(): string {
   return (os.platform() === 'win32') ? 'codeql.exe' : 'codeql';
 }
 

extensions/ql-vscode/src/extension.ts

@@ -113,16 +113,37 @@ function registerErrorStubs(excludedCommands: string[], stubGenerator: (command:
   });
 }
 
-export async function activate(ctx: ExtensionContext): Promise<void> {
+/**
+ * The publicly available interface for this extension. This is to
+ * be used in our tests.
+ */
+export interface CodeQLExtensionInterface {
+  readonly ctx: ExtensionContext;
+  readonly cliServer: CodeQLCliServer;
+  readonly qs: qsClient.QueryServerClient;
+  readonly distributionManager: DistributionManager;
+}
+
+/**
+ * Returns the CodeQLExtensionInterface, or an empty object if the interface is not
+ * available afer activation is complete. This will happen if there is no cli
+ * installed when the extension starts. Downloading and installing the cli
+ * will happen at a later time.
+ *
+ * @param ctx The extension context
+ *
+ * @returns CodeQLExtensionInterface
+ */
+export async function activate(ctx: ExtensionContext): Promise<CodeQLExtensionInterface | {}> {
   logger.log('Starting CodeQL extension');
 
+  const distributionConfigListener = new DistributionConfigListener();
   initializeLogging(ctx);
   languageSupport.install();
 
-  const distributionConfigListener = new DistributionConfigListener();
   ctx.subscriptions.push(distributionConfigListener);
   const codeQlVersionRange = DEFAULT_DISTRIBUTION_VERSION_RANGE;
-  const distributionManager = new DistributionManager(ctx, distributionConfigListener, codeQlVersionRange);
+  const distributionManager = new DistributionManager(distributionConfigListener, codeQlVersionRange, ctx);
 
   const shouldUpdateOnNextActivationKey = 'shouldUpdateOnNextActivation';
 
@@ -253,14 +274,14 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     return result;
   }
 
-  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<void> {
+  async function installOrUpdateThenTryActivate(config: DistributionUpdateConfig): Promise<CodeQLExtensionInterface | {}> {
     await installOrUpdateDistribution(config);
 
     // Display the warnings even if the extension has already activated.
     const distributionResult = await getDistributionDisplayingDistributionWarnings();
-
+    let extensionInterface: CodeQLExtensionInterface | {} = {};
     if (!beganMainExtensionActivation && distributionResult.kind !== FindDistributionResultKind.NoDistribution) {
-      await activateWithInstalledDistribution(ctx, distributionManager);
+      extensionInterface = await activateWithInstalledDistribution(ctx, distributionManager);
     } else if (distributionResult.kind === FindDistributionResultKind.NoDistribution) {
       registerErrorStubs([checkForUpdatesCommand], command => async () => {
         const installActionName = 'Install CodeQL CLI';
@@ -268,7 +289,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
           items: [installActionName]
         });
         if (chosenAction === installActionName) {
-          installOrUpdateThenTryActivate({
+          await installOrUpdateThenTryActivate({
             isUserInitiated: true,
             shouldDisplayMessageWhenNoUpdates: false,
             allowAutoUpdating: true
@@ -276,6 +297,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
         }
       });
     }
+    return extensionInterface;
   }
 
   ctx.subscriptions.push(distributionConfigListener.onDidChangeConfiguration(() => installOrUpdateThenTryActivate({
@@ -289,7 +311,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
     allowAutoUpdating: true
   })));
 
-  await installOrUpdateThenTryActivate({
+  return await installOrUpdateThenTryActivate({
     isUserInitiated: !!ctx.globalState.get(shouldUpdateOnNextActivationKey),
     shouldDisplayMessageWhenNoUpdates: false,
 
@@ -302,7 +324,7 @@ export async function activate(ctx: ExtensionContext): Promise<void> {
 async function activateWithInstalledDistribution(
   ctx: ExtensionContext,
   distributionManager: DistributionManager
-): Promise<void> {
+): Promise<CodeQLExtensionInterface> {
   beganMainExtensionActivation = true;
   // Remove any error stubs command handlers left over from first part
   // of activation.
@@ -354,6 +376,7 @@ async function activateWithInstalledDistribution(
 
   logger.log('Initializing query history manager.');
   const queryHistoryConfigurationListener = new QueryHistoryConfigListener();
+  ctx.subscriptions.push(queryHistoryConfigurationListener);
   const showResults = async (item: CompletedQuery) =>
     showResultsForCompletedQuery(item, WebviewReveal.Forced);
 
@@ -647,6 +670,13 @@ async function activateWithInstalledDistribution(
   commands.executeCommand('codeQLDatabases.removeOrphanedDatabases');
 
   logger.log('Successfully finished extension initialization.');
+
+  return {
+    ctx,
+    cliServer,
+    qs,
+    distributionManager
+  };
 }
 
 function getContextStoragePath(ctx: ExtensionContext) {

extensions/ql-vscode/src/helpers.ts

@@ -121,7 +121,7 @@ export function commandRunner(
 ): Disposable {
   return commands.registerCommand(commandId, async (...args: any[]) => {
     try {
-      await task(...args);
+      return await task(...args);
     } catch (e) {
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
@@ -133,6 +133,7 @@ export function commandRunner(
       } else {
         showAndLogErrorMessage(e.message || e);
       }
+      return undefined;
     }
   });
 }
@@ -158,7 +159,7 @@ export function commandRunnerWithProgress<R>(
       ...progressOptions
     };
     try {
-      await withProgress(progressOptionsWithDefaults, task, ...args);
+      return await withProgress(progressOptionsWithDefaults, task, ...args);
     } catch (e) {
       if (e instanceof UserCancellationException) {
         // User has cancelled this action manually
@@ -170,6 +171,7 @@ export function commandRunnerWithProgress<R>(
       } else {
         showAndLogErrorMessage(e.message || e);
       }
+      return undefined;
     }
   });
 }

extensions/ql-vscode/src/query-history.ts

@@ -59,16 +59,15 @@ interface QueryHistoryDataProvider extends vscode.TreeDataProvider<CompletedQuer
 /**
  * Tree data provider for the query history view.
  */
-class HistoryTreeDataProvider implements QueryHistoryDataProvider {
+class HistoryTreeDataProvider extends DisposableObject implements QueryHistoryDataProvider {
   /**
    * XXX: This idiom for how to get a `.fire()`-able event emitter was
    * cargo culted from another vscode extension. It seems rather
    * involved and I hope there's something better that can be done
    * instead.
    */
-  private _onDidChangeTreeData: vscode.EventEmitter<
+  private _onDidChangeTreeData = super.push(new vscode.EventEmitter<CompletedQuery | undefined>());
-    CompletedQuery | undefined
+
-  > = new vscode.EventEmitter<CompletedQuery | undefined>();
   readonly onDidChangeTreeData: vscode.Event<CompletedQuery | undefined> = this
     ._onDidChangeTreeData.event;
 
@@ -82,6 +81,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
   private current: CompletedQuery | undefined;
 
   constructor(extensionPath: string) {
+    super();
     this.failedIconPath = path.join(
       extensionPath,
       FAILED_QUERY_HISTORY_ITEM_ICON
@@ -135,7 +135,7 @@ class HistoryTreeDataProvider implements QueryHistoryDataProvider {
     return this.current;
   }
 
-  push(item: CompletedQuery): void {
+  pushQuery(item: CompletedQuery): void {
     this.current = item;
     this.history.push(item);
     this.refresh();
@@ -204,6 +204,7 @@ export class QueryHistoryManager extends DisposableObject {
       canSelectMany: true,
     });
     this.push(this.treeView);
+    this.push(treeDataProvider);
 
     // Lazily update the tree view selection due to limitations of TreeView API (see
     // `updateTreeViewSelectionIfVisible` doc for details)
@@ -513,7 +514,7 @@ export class QueryHistoryManager extends DisposableObject {
 
   addQuery(info: QueryWithResults): CompletedQuery {
     const item = new CompletedQuery(info, this.queryHistoryConfigListener);
-    this.treeDataProvider.push(item);
+    this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
     return item;
   }

extensions/ql-vscode/src/vscode-tests/cli-integration/index.ts

@@ -0,0 +1,4 @@
+import { runTestsInDirectory } from '../index-template';
+export function run(): Promise<void> {
+  return runTestsInDirectory(__dirname, true);
+}

extensions/ql-vscode/src/vscode-tests/cli-integration/run-cli.test.ts

@@ -0,0 +1,51 @@
+import 'mocha';
+import 'sinon-chai';
+
+
+import { expect } from 'chai';
+import { ConfigurationTarget, workspace, extensions } from 'vscode';
+import { SemVer } from 'semver';
+
+import { CodeQLCliServer } from '../../cli';
+import { CodeQLExtensionInterface } from '../../extension';
+
+
+/**
+ * Perform proper integration tests by running the CLI
+ */
+describe('Use cli', function() {
+  this.timeout(60000);
+
+  let cli: CodeQLCliServer;
+
+  beforeEach(async () => {
+    // Set it here before activation to ensure we don't accidentally try to download a cli
+    await workspace.getConfiguration().update('codeQL.cli.executablePath', process.env.CLI_PATH, ConfigurationTarget.Global);
+    const extension = await extensions.getExtension<CodeQLExtensionInterface | {}>('GitHub.vscode-codeql')!.activate();
+    if ('cliServer' in extension) {
+      cli = extension.cliServer;
+    } else {
+      throw new Error('Extension not initialized. Make sure cli is downloaded and installed properly.');
+    }
+  });
+
+  afterEach(() => {
+    cli.dispose();
+  });
+
+  it('should have the correct version of the cli', async () => {
+    expect(
+      (await cli.getVersion()).toString()
+    ).to.eq(
+      new SemVer(process.env.CLI_VERSION || '').toString()
+    );
+  });
+
+  it('should resolve ram', async () => {
+    const result = await (cli as any).resolveRam(8192);
+    expect(result).to.deep.eq([
+      '-J-Xmx4096M',
+      '--off-heap-ram=4096'
+    ]);
+  });
+});

extensions/ql-vscode/src/vscode-tests/ensureCli.ts

@@ -0,0 +1,129 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+import { DistributionManager, extractZipArchive, codeQlLauncherName } from '../distribution';
+import fetch from 'node-fetch';
+
+/**
+ * This module ensures that the proper CLI is available for tests of the extension.
+ * There are two environment variables to control this module:
+ *
+ *     - CLI_VERSION: The version of the CLI to install. Defaults to the most recent
+ *       version. Note that for now, we must maintain the default version by hand.
+ *
+ *     - CLI_BASE_DIR: The base dir where the CLI will be downloaded and unzipped.
+ *       The download location is `${CLI_BASE_DIR}/assets` and the unzip loction is
+ *       `${CLI_BASE_DIR}/${CLI_VERSION}`
+ *
+ * After downloading and unzipping, a new environment variable is set:
+ *
+ *     - CLI_PATH: Points to the cli executable for the specified CLI_VERSION. This
+ *       is variable is available in the unit tests and will be used as the value
+ *       for `codeQL.cli.executablePath`.
+ *
+ * As an optimization, the cli will not be unzipped again if the executable already
+ * exists. And the cli will not be re-downloaded if the zip already exists.
+ */
+
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  process.exit(-1);
+});
+
+const _1MB = 1024 * 1024;
+const _10MB = _1MB * 10;
+
+// CLI version to test. Hard code the latest as default. And be sure
+// to update the env if it is not otherwise set.
+const CLI_VERSION = process.env.CLI_VERSION || 'v2.3.1';
+process.env.CLI_VERSION = CLI_VERSION;
+
+// Base dir where CLIs will be downloaded into
+// By default, put it in the `build` directory in the root of the extension.
+const CLI_BASE_DIR = process.env.CLI_DIR || path.normalize(path.join(__dirname, '../../build/cli'));
+
+export async function ensureCli(useCli: boolean) {
+  try {
+    if (!useCli) {
+      console.log('Not downloading CLI. It is not being used.');
+      return;
+    }
+
+    const assetName = DistributionManager.getRequiredAssetName();
+    const url = getCliDownloadUrl(assetName);
+    const unzipDir = getCliUnzipDir();
+    const downloadedFilePath = getDownloadFilePath(assetName);
+    const executablePath = path.join(getCliUnzipDir(), 'codeql', codeQlLauncherName());
+
+    // Use this environment variable to se to the `codeQL.cli.executablePath` in tests
+    process.env.CLI_PATH = executablePath;
+
+    if (fs.existsSync(executablePath)) {
+      console.log(`CLI version ${CLI_VERSION} is found ${executablePath}. Not going to download again.`);
+      return;
+    }
+
+    if (!fs.existsSync(downloadedFilePath)) {
+      console.log(`CLI version ${CLI_VERSION} zip file not found. Downloading from '${url}' into '${downloadedFilePath}'.`);
+
+      const assetStream = await fetch(url);
+      const contentLength = Number(assetStream.headers.get('content-length') || 0);
+      console.log('Total content size', Math.round(contentLength / _1MB), 'MB');
+      const archiveFile = fs.createWriteStream(downloadedFilePath);
+      const body = assetStream.body;
+      await new Promise((resolve, reject) => {
+        let numBytesDownloaded = 0;
+        let lastMessage = 0;
+        body.on('data', (data) => {
+          numBytesDownloaded += data.length;
+          if (numBytesDownloaded - lastMessage > _10MB) {
+            console.log('Downloaded', Math.round(numBytesDownloaded / _1MB), 'MB');
+            lastMessage = numBytesDownloaded;
+          }
+          archiveFile.write(data);
+        });
+        body.on('finish', () => {
+          archiveFile.end(() => {
+            console.log('Finished download into', downloadedFilePath);
+            resolve();
+          });
+        });
+        body.on('error', reject);
+      });
+    } else {
+      console.log(`CLI version ${CLI_VERSION} zip file found at '${downloadedFilePath}'.`);
+    }
+
+    console.log(`Unzipping into '${unzipDir}'`);
+    fs.mkdirpSync(unzipDir);
+    await extractZipArchive(downloadedFilePath, unzipDir);
+    console.log('Done.');
+  } catch (e) {
+    console.error('Failed to download CLI.');
+    console.error(e);
+    process.exit(-1);
+  }
+}
+
+/**
+ * Url to download from
+ */
+function getCliDownloadUrl(assetName: string) {
+  return `https://github.com/github/codeql-cli-binaries/releases/download/${CLI_VERSION}/${assetName}`;
+}
+
+/**
+ * Directory to place the downloaded cli into
+ */
+function getDownloadFilePath(assetName: string) {
+  const dir = path.join(CLI_BASE_DIR, 'assets');
+  fs.mkdirpSync(dir);
+  return path.join(dir, assetName);
+}
+
+/**
+ * Directory to unzip the downloaded cli into.
+ */
+function getCliUnzipDir() {
+  return path.join(CLI_BASE_DIR, CLI_VERSION);
+}

extensions/ql-vscode/src/vscode-tests/index-template.ts

@@ -1,6 +1,18 @@
 import * as path from 'path';
 import * as Mocha from 'mocha';
 import * as glob from 'glob';
+import { ensureCli } from './ensureCli';
+
+
+// Use this handler to avoid swallowing unhandled rejections.
+process.on('unhandledRejection', e => {
+  console.error('Unhandled rejection.');
+  console.error(e);
+  // Must use a setTimeout in order to ensure the log is fully flushed before exiting
+  setTimeout(() => {
+    process.exit(-1);
+  }, 2000);
+});
 
 /**
  * Helper function that runs all Mocha tests found in the
@@ -26,13 +38,15 @@ import * as glob from 'glob';
  * After https://github.com/microsoft/TypeScript/issues/420 is implemented,
  * this pattern can be expressed more neatly using a module interface.
  */
-export function runTestsInDirectory(testsRoot: string): Promise<void> {
+export async function runTestsInDirectory(testsRoot: string, useCli = false): Promise<void> {
   // Create the mocha test
   const mocha = new Mocha({
     ui: 'bdd',
     color: true
   });
 
+  await ensureCli(useCli);
+
   return new Promise((c, e) => {
     console.log(`Adding test cases from ${testsRoot}`);
     glob('**/**.test.js', { cwd: testsRoot }, (err, files) => {

extensions/ql-vscode/src/vscode-tests/minimal-workspace/activation.test.ts

@@ -9,6 +9,7 @@ import * as determiningSelectedQueryTest from './determining-selected-query-test
 chai.use(chaiAsPromised);
 
 describe('launching with a minimal workspace', async () => {
+
   const ext = vscode.extensions.getExtension('GitHub.vscode-codeql');
   it('should install the extension', () => {
     assert(ext);
@@ -19,6 +20,9 @@ describe('launching with a minimal workspace', async () => {
   });
 
   it('should activate the extension when a .ql file is opened', async function() {
+    this.timeout(60000);
+    await delay();
+
     const folders = vscode.workspace.workspaceFolders;
     assert(folders && folders.length === 1);
     const folderPath = folders![0].uri.fsPath;
@@ -26,10 +30,13 @@ describe('launching with a minimal workspace', async () => {
     const document = await vscode.workspace.openTextDocument(documentPath);
     assert(document.languageId === 'ql');
     // Delay slightly so that the extension has time to activate.
-    this.timeout(4000);
+    await delay();
-    await new Promise(resolve => setTimeout(resolve, 2000));
     assert(ext!.isActive);
   });
+
+  async function delay() {
+    await new Promise(resolve => setTimeout(resolve, 4000));
+  }
 });
 
 determiningSelectedQueryTest.run();

extensions/ql-vscode/src/vscode-tests/minimal-workspace/databases.test.ts

@@ -41,8 +41,6 @@ describe('databases', () => {
   let sandbox: sinon.SinonSandbox;
   let dir: tmp.DirResult;
 
-
-
   beforeEach(() => {
     dir = tmp.dirSync();
 
@@ -86,6 +84,7 @@ describe('databases', () => {
 
   afterEach(async () => {
     dir.removeCallback();
+    databaseManager.dispose();
     sandbox.restore();
   });
 
@@ -425,7 +424,7 @@ describe('databases', () => {
         datasetUri: databaseUri
       } as DatabaseContents,
       MOCK_DB_OPTIONS,
-      dbChangedHandler
+      dbChangedHandler,
     );
   }
 

extensions/ql-vscode/src/vscode-tests/no-workspace/astViewer.test.ts

@@ -15,19 +15,25 @@ const expect = chai.expect;
 
 describe('AstViewer', () => {
   let astRoots: AstItem[];
-  let viewer: AstViewer;
+  let viewer: AstViewer | undefined;
+  let sandbox: sinon.SinonSandbox;
   beforeEach(async () => {
+    sandbox = sinon.createSandbox();
     // the ast is stored in yaml because there are back pointers
     // making a json representation impossible.
     // The complication here is that yaml files are not copied into the 'out' directory by tsc.
     astRoots = await buildAst();
 
-    sinon.stub(commands, 'registerCommand');
+    sandbox.stub(commands, 'registerCommand');
-    sinon.stub(commands, 'executeCommand');
+    sandbox.stub(commands, 'executeCommand');
   });
 
   afterEach(() => {
-    sinon.restore();
+    sandbox.restore();
+    if (viewer) {
+      viewer.dispose();
+      viewer = undefined;
+    }
   });
 
   it('should update the viewer roots', () => {
@@ -56,7 +62,6 @@ describe('AstViewer', () => {
     doSelectionTest(undefined, new Range(2, 3, 4, 5));
   });
 
-
   function doSelectionTest(
     expectedSelection: any,
     selectionRange: Range | undefined,
@@ -65,7 +70,7 @@ describe('AstViewer', () => {
     const item = {} as DatabaseItem;
     viewer = new AstViewer();
     viewer.updateRoots(astRoots, item, fsPath);
-    const spy = sinon.spy();
+    const spy = sandbox.spy();
     (viewer as any).treeView.reveal = spy;
     Object.defineProperty((viewer as any).treeView, 'visible', {
       value: true

extensions/ql-vscode/src/vscode-tests/no-workspace/distribution.test.ts

@@ -252,8 +252,12 @@ describe('Launcher path', () => {
     expect(result).to.equal(undefined);
   });
 
-  it('should not warn when deprecated launcher is used, but no new launcher is available', async () => {
+  it('should not warn when deprecated launcher is used, but no new launcher is available', async function() {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = 'codeql.cmd';
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -265,7 +269,11 @@ describe('Launcher path', () => {
   });
 
   it('should warn when deprecated launcher is used, and new launcher is available', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = ''; // pretend both launchers exist
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();
@@ -277,7 +285,11 @@ describe('Launcher path', () => {
   });
 
   it('should warn when launcher path is incorrect', async () => {
-    const manager = new (createModule().DistributionManager)(undefined as any, { customCodeQlPath: pathToCmd } as any, undefined as any);
+    const manager = new (createModule().DistributionManager)(
+      { customCodeQlPath: pathToCmd } as any,
+      {} as any,
+      undefined as any
+    );
     launcherThatExists = 'xxx'; // pretend neither launcher exists
 
     const result = await manager.getCodeQlPathWithoutVersionCheck();

extensions/ql-vscode/src/vscode-tests/no-workspace/query-history.test.ts

@@ -23,6 +23,7 @@ describe('query-history', () => {
 
   beforeEach(() => {
     sandbox = sinon.createSandbox();
+
     showTextDocumentSpy = sandbox.stub(vscode.window, 'showTextDocument');
     showInformationMessageSpy = sandbox.stub(
       vscode.window,

extensions/ql-vscode/src/vscode-tests/run-integration-tests.ts

@@ -1,16 +1,17 @@
 import * as path from 'path';
 import * as os from 'os';
-import { runTests } from 'vscode-test';
+import * as cp from 'child_process';
+import {
+  runTests,
+  downloadAndUnzipVSCode,
+  resolveCliPathFromVSCodeExecutablePath
+} from 'vscode-test';
+import { assertNever } from '../helpers-pure';
+
+// For some reason, `TestOptions` is not exported directly from `vscode-test`,
+// but we can be tricky and import directly from the out file.
+import { TestOptions } from 'vscode-test/out/runTest';
 
-// A subset of the fields in TestOptions from vscode-test, which we
-// would simply use instead, but for the fact that it doesn't export
-// it.
-type Suite = {
-  extensionDevelopmentPath: string;
-  extensionTestsPath: string;
-  launchArgs: string[];
-  version?: string;
-};
 
 // Which version of vscode to test against. Can set to 'stable' or
 // 'insiders' or an explicit version number. See runTest.d.ts in
@@ -22,11 +23,21 @@ type Suite = {
 // testing against old versions if necessary.
 const VSCODE_VERSION = 'stable';
 
+// List if test dirs
+//   - no-workspace - Tests with no workspace selected upon launch.
+//   - minimal-workspace - Tests with a simple workspace selected upon launch.
+//   - cli-integration - Tests that require a cli to invoke actual commands
+enum TestDir {
+  NoWorskspace = 'no-workspace',
+  MinimalWorskspace = 'minimal-workspace',
+  CliIntegration = 'cli-integration'
+}
+
 /**
  * Run an integration test suite `suite`, retrying if it segfaults, at
  * most `tries` times.
  */
-async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise<void> {
+async function runTestsWithRetryOnSegfault(suite: TestOptions, tries: number): Promise<void> {
   for (let t = 0; t < tries; t++) {
     try {
       // Download and unzip VS Code if necessary, and run the integration test suite.
@@ -58,34 +69,33 @@ async function runTestsWithRetryOnSegfault(suite: Suite, tries: number): Promise
  */
 async function main() {
   try {
-    // The folder containing the Extension Manifest package.json
-    // Passed to `--extensionDevelopmentPath`.
     const extensionDevelopmentPath = path.resolve(__dirname, '../..');
+    const vscodeExecutablePath = await downloadAndUnzipVSCode(VSCODE_VERSION);
 
-    // List of integration test suites.
-    // The path to the extension test runner script is passed to --extensionTestsPath.
-    const integrationTestSuites: Suite[] = [
-      // Tests with no workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'no-workspace', 'index'),
-        launchArgs: ['--disable-extensions'],
-      },
-      // Tests with a simple workspace selected upon launch.
-      {
-        version: VSCODE_VERSION,
-        extensionDevelopmentPath: extensionDevelopmentPath,
-        extensionTestsPath: path.resolve(__dirname, 'minimal-workspace', 'index'),
-        launchArgs: [
-          path.resolve(__dirname, '../../test/data'),
-          '--disable-extensions',
-        ]
-      }
-    ];
 
-    for (const integrationTestSuite of integrationTestSuites) {
+    // Which tests to run. Use a comma-separated list of directories.
-      await runTestsWithRetryOnSegfault(integrationTestSuite, 3);
+    const testDirsString = process.argv[2];
+    const dirs = testDirsString.split(',').map(dir => dir.trim().toLocaleLowerCase());
+
+    if (dirs.includes(TestDir.CliIntegration)) {
+      console.log('Installing required extensions');
+      const cliPath = resolveCliPathFromVSCodeExecutablePath(vscodeExecutablePath);
+      cp.spawnSync(cliPath, ['--install-extension', 'hbenl.vscode-test-explorer'], {
+        encoding: 'utf-8',
+        stdio: 'inherit'
+      });
+    }
+
+    console.log(`Running integration tests in these directories: ${dirs}`);
+    for (const dir of dirs) {
+      console.log(`Next integration test dir: ${dir}`);
+      await runTestsWithRetryOnSegfault({
+        version: VSCODE_VERSION,
+        vscodeExecutablePath,
+        extensionDevelopmentPath,
+        extensionTestsPath: path.resolve(__dirname, dir, 'index'),
+        launchArgs: getLaunchArgs(dir as TestDir)
+      }, 3);
     }
   } catch (err) {
     console.error(`Unexpected exception while running tests: ${err}`);
@@ -94,3 +104,25 @@ async function main() {
 }
 
 main();
+
+
+function getLaunchArgs(dir: TestDir) {
+  switch (dir) {
+    case TestDir.NoWorskspace:
+      return [
+        '--disable-extensions'
+      ];
+
+    case TestDir.MinimalWorskspace:
+      return [
+        '--disable-extensions',
+        path.resolve(__dirname, '../../test/data')
+      ];
+
+    case TestDir.CliIntegration:
+      return undefined;
+
+    default:
+      assertNever(dir);
+  }
+}

extensions/ql-vscode/test/data/.gitignore

@@ -0,0 +1 @@
+.vscode