Remember remote queries across restarts

Remote query items will be stored in query history and will remain
available across restarts.

When the extension is restarted, any `InProgress` remote queries will
be monitored until they complete.

When clicked on, a remote query is opened and its results can be
downloaded. The query text and the query file can be opened from the
history menu. A remote query can be deleted as well, which will purge
all results from global storage.

Limitations:

1. Labels are not editable
2. Running multiple queries that each run on the same repository
   will have conflicting results and there will be errors when trying
   to view the results of the second query. This limitation is not new,
   but it is easier to hit now. See #1089.

Both of these limitations will be addressed in future PRs.

Author: aeisenberg

extensions/ql-vscode/src/extension.ts

@@ -90,13 +90,13 @@ import { CodeQlStatusBarHandler } from './status-bar';
 
 import { Credentials } from './authentication';
 import { RemoteQueriesManager } from './remote-queries/remote-queries-manager';
-import { RemoteQuery } from './remote-queries/remote-query';
 import { RemoteQueryResult } from './remote-queries/remote-query-result';
 import { URLSearchParams } from 'url';
 import { RemoteQueriesInterfaceManager } from './remote-queries/remote-queries-interface';
 import * as sampleData from './remote-queries/sample-data';
 import { handleDownloadPacks, handleInstallPackDependencies } from './packaging';
 import { AnalysesResultsManager } from './remote-queries/analyses-results-manager';
+import { RemoteQueryHistoryItem } from './remote-queries/remote-query-history-item';
 
 /**
  * extension.ts
@@ -455,11 +455,15 @@ async function activateWithInstalledDistribution(
     queryStorageDir,
     ctx,
     queryHistoryConfigurationListener,
-    showResults,
     async (from: CompletedLocalQueryInfo, to: CompletedLocalQueryInfo) =>
       showResultsForComparison(from, to),
   );
-  await qhm.readQueryHistory();
+
+  qhm.onWillOpenQueryItem(async item => {
+    if (item.t === 'local' && item.completed) {
+      await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.Forced);
+    }
+  });
 
   ctx.subscriptions.push(qhm);
   void logger.log('Initializing results panel interface.');
@@ -534,7 +538,6 @@ async function activateWithInstalledDistribution(
           source.token,
         );
         item.completeThisQuery(completedQueryInfo);
-        await qhm.writeQueryHistory();
         await showResultsForCompletedQuery(item as CompletedLocalQueryInfo, WebviewReveal.NotForced);
         // Note we must update the query history view after showing results as the
         // display and sorting might depend on the number of results
@@ -543,7 +546,7 @@ async function activateWithInstalledDistribution(
         item.failureReason = e.message;
         throw e;
       } finally {
-        qhm.refreshTreeView();
+        await qhm.refreshTreeView();
         source.dispose();
       }
     }
@@ -834,6 +837,12 @@ async function activateWithInstalledDistribution(
 
   void logger.log('Initializing remote queries interface.');
   const rqm = new RemoteQueriesManager(ctx, cliServer, qhm, queryStorageDir, logger);
+  ctx.subscriptions.push(rqm);
+
+  // wait until after the remote queries manager is initialized to read the query history
+  // since the rqm is notified of queries being added.
+  await qhm.readQueryHistory();
+
 
   registerRemoteQueryTextProvider();
 
@@ -866,9 +875,9 @@ async function activateWithInstalledDistribution(
 
   ctx.subscriptions.push(
     commandRunner('codeQL.monitorRemoteQuery', async (
-      query: RemoteQuery,
+      queryItem: RemoteQueryHistoryItem,
       token: CancellationToken) => {
-      await rqm.monitorRemoteQuery(query, token);
+      await rqm.monitorRemoteQuery(queryItem, token);
     }));
 
   ctx.subscriptions.push(

extensions/ql-vscode/src/query-history.ts

@@ -288,13 +288,24 @@ export class QueryHistoryManager extends DisposableObject {
   queryHistoryScrubber: Disposable | undefined;
   private queryMetadataStorageLocation;
 
+  private readonly _onDidAddQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
+  readonly onDidAddQueryItem: Event<QueryHistoryInfo> = this
+    ._onDidAddQueryItem.event;
+
+  private readonly _onDidRemoveQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
+  readonly onDidRemoveQueryItem: Event<QueryHistoryInfo> = this
+    ._onDidRemoveQueryItem.event;
+
+  private readonly _onWillOpenQueryItem = super.push(new EventEmitter<QueryHistoryInfo>());
+  readonly onWillOpenQueryItem: Event<QueryHistoryInfo> = this
+    ._onWillOpenQueryItem.event;
+
   constructor(
     private qs: QueryServerClient,
     private dbm: DatabaseManager,
     private queryStorageDir: string,
     ctx: ExtensionContext,
     private queryHistoryConfigListener: QueryHistoryConfig,
-    private selectedCallback: (item: CompletedLocalQueryInfo) => Promise<void>,
     private doCompareCallback: (
       from: CompletedLocalQueryInfo,
       to: CompletedLocalQueryInfo
@@ -484,53 +495,54 @@ export class QueryHistoryManager extends DisposableObject {
     void logger.log(`Reading cached query history from '${this.queryMetadataStorageLocation}'.`);
     const history = await slurpQueryHistory(this.queryMetadataStorageLocation, this.queryHistoryConfigListener);
     this.treeDataProvider.allHistory = history;
+    this.treeDataProvider.allHistory.forEach((item) => {
+      this._onDidAddQueryItem.fire(item);
+    });
   }
 
   async writeQueryHistory(): Promise<void> {
-    const toSave = this.treeDataProvider.allHistory.filter(q => q.isCompleted());
-    await splatQueryHistory(toSave, this.queryMetadataStorageLocation);
-  }
-
-  async invokeCallbackOn(queryHistoryItem: QueryHistoryInfo) {
-    if (this.selectedCallback && queryHistoryItem.isCompleted()) {
-      const sc = this.selectedCallback;
-      await sc(queryHistoryItem as CompletedLocalQueryInfo);
-    }
+    await splatQueryHistory(this.treeDataProvider.allHistory, this.queryMetadataStorageLocation);
   }
 
   async handleOpenQuery(
     singleItem: QueryHistoryInfo,
     multiSelect: QueryHistoryInfo[]
   ): Promise<void> {
-    // TODO will support remote queries
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
-    if (!this.assertSingleQuery(finalMultiSelect) || finalSingleItem?.t !== 'local') {
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem) {
       return;
     }
 
+    const queryPath = finalSingleItem.t === 'local'
+      ? finalSingleItem.initialInfo.queryPath
+      : finalSingleItem.remoteQuery.queryFilePath;
+
     const textDocument = await workspace.openTextDocument(
-      Uri.file(finalSingleItem.initialInfo.queryPath)
+      Uri.file(queryPath)
     );
     const editor = await window.showTextDocument(
       textDocument,
       ViewColumn.One
     );
-    const queryText = finalSingleItem.initialInfo.queryText;
-    if (queryText !== undefined && finalSingleItem.initialInfo.isQuickQuery) {
-      await editor.edit((edit) =>
-        edit.replace(
-          textDocument.validateRange(
-            new Range(0, 0, textDocument.lineCount, 0)
-          ),
-          queryText
-        )
-      );
+
+    if (finalSingleItem.t === 'local') {
+      const queryText = finalSingleItem.initialInfo.queryText;
+      if (queryText !== undefined && finalSingleItem.initialInfo.isQuickQuery) {
+        await editor.edit((edit) =>
+          edit.replace(
+            textDocument.validateRange(
+              new Range(0, 0, textDocument.lineCount, 0)
+            ),
+            queryText
+          )
+        );
+      }
     }
   }
 
   async handleRemoveHistoryItem(
     singleItem: QueryHistoryInfo,
-    multiSelect: QueryHistoryInfo[]
+    multiSelect: QueryHistoryInfo[] = []
   ) {
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
     const toDelete = (finalMultiSelect || [finalSingleItem]);
@@ -548,23 +560,21 @@ export class QueryHistoryManager extends DisposableObject {
       } else {
         // Remote queries can be removed locally, but not remotely.
         // The user must cancel the query on GitHub Actions explicitly.
-
         this.treeDataProvider.remove(item);
-        await item.deleteQuery();
         void logger.log(`Deleted ${item.label}.`);
         if (item.status === QueryStatus.InProgress) {
-          void showAndLogInformationMessage(
-            'The remote query is still running on GitHub Actions. To cancel there, you must go to the query run in your browser.'
-          );
+          void logger.log('The remote query is still running on GitHub Actions. To cancel there, you must go to the query run in your browser.');
         }
+
+        this._onDidRemoveQueryItem.fire(item);
       }
 
     }));
     await this.writeQueryHistory();
     const current = this.treeDataProvider.getCurrent();
     if (current !== undefined) {
       await this.treeView.reveal(current, { select: true });
-      await this.invokeCallbackOn(current);
+      await this._onWillOpenQueryItem.fire(current);
     }
   }
 
@@ -635,7 +645,7 @@ export class QueryHistoryManager extends DisposableObject {
       const from = this.compareWithItem || singleItem;
       const to = await this.findOtherQueryToCompare(from, finalMultiSelect);
 
-      if (from.isCompleted() && to?.isCompleted()) {
+      if (from.completed && to?.completed) {
         await this.doCompareCallback(from as CompletedLocalQueryInfo, to as CompletedLocalQueryInfo);
       }
     } catch (e) {
@@ -647,9 +657,8 @@ export class QueryHistoryManager extends DisposableObject {
     singleItem: QueryHistoryInfo,
     multiSelect: QueryHistoryInfo[]
   ) {
-    // TODO will support remote queries
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
-    if (!this.assertSingleQuery(finalMultiSelect) || finalSingleItem?.t !== 'local') {
+    if (!this.assertSingleQuery(finalMultiSelect)) {
       return;
     }
 
@@ -668,7 +677,7 @@ export class QueryHistoryManager extends DisposableObject {
       await this.handleOpenQuery(finalSingleItem, [finalSingleItem]);
     } else {
       // show results on single click
-      await this.invokeCallbackOn(finalSingleItem);
+      await this._onWillOpenQueryItem.fire(finalSingleItem);
     }
   }
 
@@ -713,17 +722,20 @@ export class QueryHistoryManager extends DisposableObject {
   ) {
     const { finalSingleItem, finalMultiSelect } = this.determineSelection(singleItem, multiSelect);
 
-    // TODO will support remote queries
-    if (!this.assertSingleQuery(finalMultiSelect) || finalSingleItem?.t !== 'local') {
+    if (!this.assertSingleQuery(finalMultiSelect) || !finalSingleItem) {
       return;
     }
 
     const params = new URLSearchParams({
-      isQuickEval: String(!!finalSingleItem.initialInfo.quickEvalPosition),
+      isQuickEval: String(!!(finalSingleItem.t === 'local' && finalSingleItem.initialInfo.quickEvalPosition)),
       queryText: encodeURIComponent(await this.getQueryText(finalSingleItem)),
     });
+    const queryId = finalSingleItem.t === 'local'
+      ? finalSingleItem.initialInfo.id
+      : finalSingleItem.queryId;
+
     const uri = Uri.parse(
-      `codeql:${finalSingleItem.initialInfo.id}?${params.toString()}`, true
+      `codeql:${queryId}?${params.toString()}`, true
     );
     const doc = await workspace.openTextDocument(uri);
     await window.showTextDocument(doc, { preview: false });
@@ -809,13 +821,15 @@ export class QueryHistoryManager extends DisposableObject {
   }
 
   async getQueryText(item: QueryHistoryInfo): Promise<string> {
-    // TODO the query text for remote queries is not yet available
-    return item.t === 'local' ? item.initialInfo.queryText : '';
+    return item.t === 'local'
+      ? item.initialInfo.queryText
+      : item.remoteQuery.queryText;
   }
 
   addQuery(item: QueryHistoryInfo) {
     this.treeDataProvider.pushQuery(item);
     this.updateTreeViewSelectionIfVisible();
+    this._onDidAddQueryItem.fire(item);
   }
 
   /**
@@ -1011,7 +1025,8 @@ the file in the file explorer and dragging it into the workspace.`
     };
   }
 
-  refreshTreeView(): void {
+  async refreshTreeView(): Promise<void> {
     this.treeDataProvider.refresh();
+    await this.writeQueryHistory();
   }
 }

extensions/ql-vscode/src/query-results.ts

@@ -291,7 +291,7 @@ export class LocalQueryInfo {
     }
   }
 
-  isCompleted(): boolean {
+  get completed(): boolean {
     return !!this.completedQuery;
   }
 

extensions/ql-vscode/src/query-serialization.ts

@@ -38,7 +38,7 @@ export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConf
           q.completedQuery.dispose = () => { /**/ };
         }
       } else if (q.t === 'remote') {
-        // TODO Remote queries are not implemented yet.
+        // noop
       }
       return q;
     });
@@ -47,8 +47,11 @@ export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConf
     // most likely another workspace has deleted them because the
     // queries aged out.
     return asyncFilter(parsedQueries, async (q) => {
-      if (q.t !== 'local') {
-        return false;
+      if (q.t === 'remote') {
+        // the slurper doesn't know where the remote queries are stored
+        // so we need to assume here that they exist. Later, we check to
+        // see if they exist on disk.
+        return true;
       }
       const resultsPath = q.completedQuery?.query.resultsPaths.resultsPath;
       return !!resultsPath && await fs.pathExists(resultsPath);
@@ -57,6 +60,8 @@ export async function slurpQueryHistory(fsPath: string, config: QueryHistoryConf
     void showAndLogErrorMessage('Error loading query history.', {
       fullMessage: ['Error loading query history.', e.stack].join('\n'),
     });
+    // since the query history is invalid, it should be deleted so this error does not happen on next startup.
+    await fs.remove(fsPath);
     return [];
   }
 }
@@ -75,8 +80,8 @@ export async function splatQueryHistory(queries: QueryHistoryInfo[], fsPath: str
     if (!(await fs.pathExists(fsPath))) {
       await fs.mkdir(path.dirname(fsPath), { recursive: true });
     }
-    // remove incomplete queries since they cannot be recreated on restart
-    const filteredQueries = queries.filter(q => q.t === 'local' && q.completedQuery !== undefined);
+    // remove incomplete local queries since they cannot be recreated on restart
+    const filteredQueries = queries.filter(q => q.t === 'local' ? q.completedQuery !== undefined : true);
     const data = JSON.stringify(filteredQueries, null, 2);
     await fs.writeFile(fsPath, data);
   } catch (e) {

extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts

@@ -261,8 +261,8 @@ export class RemoteQueriesInterfaceManager {
     return this.getPanel().webview.postMessage(msg);
   }
 
-  private getDuration(startTime: Date, endTime: Date): string {
-    const diffInMs = startTime.getTime() - endTime.getTime();
+  private getDuration(startTime: number, endTime: number): string {
+    const diffInMs = startTime - endTime;
     return this.formatDuration(diffInMs);
   }
 
@@ -282,7 +282,8 @@ export class RemoteQueriesInterfaceManager {
     }
   }
 
-  private formatDate = (d: Date): string => {
+  private formatDate = (millis: number): string => {
+    const d = new Date(millis);
     const datePart = d.toLocaleDateString(undefined, { day: 'numeric', month: 'short' });
     const timePart = d.toLocaleTimeString(undefined, { hour: 'numeric', minute: 'numeric', hour12: true });
     return `${datePart} at ${timePart}`;