From 261ffbf08446d385cffd6020cb2c46530d8e5cb7 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Thu, 20 Jan 2022 09:51:25 +0000 Subject: [PATCH 1/3] Download and process analyses results --- extensions/ql-vscode/src/extension.ts | 4 +- .../ql-vscode/src/pure/interface-types.ts | 17 ++-- .../analyses-results-manager.ts | 90 +++++++++++++++++++ .../remote-queries-interface.ts | 49 +++++----- .../remote-queries/remote-queries-manager.ts | 5 +- .../remote-queries/shared/analysis-result.ts | 8 ++ .../src/remote-queries/view/RemoteQueries.tsx | 65 +++++++++++--- 7 files changed, 194 insertions(+), 44 deletions(-) create mode 100644 extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts create mode 100644 extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts diff --git a/extensions/ql-vscode/src/extension.ts b/extensions/ql-vscode/src/extension.ts index 0cd0dc70670..ec580f19573 100644 --- a/extensions/ql-vscode/src/extension.ts +++ b/extensions/ql-vscode/src/extension.ts @@ -84,6 +84,7 @@ import { URLSearchParams } from 'url'; import { RemoteQueriesInterfaceManager } from './remote-queries/remote-queries-interface'; import { sampleRemoteQuery, sampleRemoteQueryResult } from './remote-queries/sample-data'; import { handleDownloadPacks, handleInstallPackDependencies } from './packaging'; +import { AnalysesResultsManager } from './remote-queries/analyses-results-manager'; /** * extension.ts @@ -817,7 +818,8 @@ async function activateWithInstalledDistribution( ctx.subscriptions.push( commandRunner('codeQL.showFakeRemoteQueryResults', async () => { - const rqim = new RemoteQueriesInterfaceManager(ctx, logger); + const analysisResultsManager = new AnalysesResultsManager(ctx, logger); + const rqim = new RemoteQueriesInterfaceManager(ctx, logger, analysisResultsManager); await rqim.showResults(sampleRemoteQuery, sampleRemoteQueryResult); })); diff --git a/extensions/ql-vscode/src/pure/interface-types.ts b/extensions/ql-vscode/src/pure/interface-types.ts index 080c10f7eaf..9e9ce2aa579 100644 --- a/extensions/ql-vscode/src/pure/interface-types.ts +++ b/extensions/ql-vscode/src/pure/interface-types.ts @@ -1,6 +1,6 @@ import * as sarif from 'sarif'; -import { DownloadLink } from '../remote-queries/download-link'; -import { RemoteQueryResult } from '../remote-queries/shared/remote-query-result'; +import { AnalysisResults } from '../remote-queries/shared/analysis-result'; +import { AnalysisSummary, RemoteQueryResult } from '../remote-queries/shared/remote-query-result'; import { RawResultSet, ResultRow, ResultSetSchema, Column, ResolvableLocationValue } from './bqrs-cli-types'; /** @@ -381,7 +381,8 @@ export type FromRemoteQueriesMessage = | RemoteQueryDownloadAllAnalysesResultsMessage; export type ToRemoteQueriesMessage = - | SetRemoteQueryResultMessage; + | SetRemoteQueryResultMessage + | SetAnalysesResultsMessage; export interface RemoteQueryLoadedMessage { t: 'remoteQueryLoaded'; @@ -392,6 +393,11 @@ export interface SetRemoteQueryResultMessage { queryResult: RemoteQueryResult } +export interface SetAnalysesResultsMessage { + t: 'setAnalysesResults'; + analysesResults: AnalysisResults[]; +} + export interface RemoteQueryErrorMessage { t: 'remoteQueryError'; error: string; @@ -399,11 +405,10 @@ export interface RemoteQueryErrorMessage { export interface RemoteQueryDownloadAnalysisResultsMessage { t: 'remoteQueryDownloadAnalysisResults'; - nwo: string - downloadLink: DownloadLink; + analysisSummary: AnalysisSummary } export interface RemoteQueryDownloadAllAnalysesResultsMessage { t: 'remoteQueryDownloadAllAnalysesResults'; - downloadLink: DownloadLink; + analysisSummaries: AnalysisSummary[]; } diff --git a/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts new file mode 100644 index 00000000000..196427627d5 --- /dev/null +++ b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts @@ -0,0 +1,90 @@ +import { ExtensionContext } from 'vscode'; +import { Credentials } from '../authentication'; +import { Logger } from '../logging'; +import { downloadArtifactFromLink } from './gh-actions-api-client'; +import { DownloadLink } from './download-link'; +import * as path from 'path'; +import * as fs from 'fs-extra'; +import { AnalysisSummary } from './shared/remote-query-result'; +import * as sarif from 'sarif'; +import { AnalysisResults, QueryResult } from './shared/analysis-result'; + +export class AnalysesResultsManager { + // Store for the results of various analyses for a single remote query. + private readonly analysesResults: { [key: string]: QueryResult[] }; + + constructor( + private readonly ctx: ExtensionContext, + private readonly logger: Logger, + ) { + this.analysesResults = {}; + } + + public async downloadAnalysisResults( + analysisSummary: AnalysisSummary, + ): Promise { + const credentials = await Credentials.initialize(this.ctx); + + void this.logger.log(`Downloading and processing results for ${analysisSummary.nwo}`); + + const queryResults = await this.downloadSingleAnalysisResults( + analysisSummary.downloadLink, + credentials); + + this.analysesResults[analysisSummary.nwo] = queryResults; + } + + public async downloadAllResults( + analysisSummaries: AnalysisSummary[], + ): Promise { + const credentials = await Credentials.initialize(this.ctx); + + void this.logger.log('Downloading and processing all results'); + + for (const analysis of analysisSummaries) { + const queryResults = await this.downloadSingleAnalysisResults(analysis.downloadLink, credentials); + this.analysesResults[analysis.nwo] = queryResults; + } + } + + public getFlattenedAnalysesResults(): AnalysisResults[] { + return Object.entries(this.analysesResults).map(([nwo, results]) => ({ nwo, results })); + } + + private async downloadSingleAnalysisResults( + downloadLink: DownloadLink, + credentials: Credentials + ): Promise { + const artifactPath = await downloadArtifactFromLink(credentials, downloadLink); + + if (path.extname(artifactPath) === '.sarif') { + return await this.readResults(artifactPath); + } else { + // Non-problem or problem-path queries are not currently fully supported. + return []; + } + } + + private async readResults(filePath: string): Promise { + const queryResults: QueryResult[] = []; + + const sarifContents = await fs.readFile(filePath, 'utf8'); + const sarifLog = JSON.parse(sarifContents) as sarif.Log; + + // Read the sarif file and extract information that we want to display + // in the UI. For now we're only getting the message texts but we'll gradually + // extract more information based on the UX we want to build. + + sarifLog.runs?.forEach(run => { + run?.results?.forEach(result => { + if (result?.message?.text) { + queryResults.push({ + message: result.message.text + }); + } + }); + }); + + return queryResults; + } +} diff --git a/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts b/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts index 7c26f41988b..5418a550b70 100644 --- a/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts +++ b/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts @@ -7,13 +7,13 @@ import { workspace, } from 'vscode'; import * as path from 'path'; -import * as vscode from 'vscode'; -import * as fs from 'fs-extra'; import { tmpDir } from '../run-queries'; import { ToRemoteQueriesMessage, FromRemoteQueriesMessage, + RemoteQueryDownloadAnalysisResultsMessage, + RemoteQueryDownloadAllAnalysesResultsMessage, } from '../pure/interface-types'; import { Logger } from '../logging'; import { getHtmlForWebview } from '../interface-utils'; @@ -22,12 +22,11 @@ import { AnalysisSummary, RemoteQueryResult } from './remote-query-result'; import { RemoteQuery } from './remote-query'; import { RemoteQueryResult as RemoteQueryResultViewModel } from './shared/remote-query-result'; import { AnalysisSummary as AnalysisResultViewModel } from './shared/remote-query-result'; -import { downloadArtifactFromLink } from './gh-actions-api-client'; -import { Credentials } from '../authentication'; -import { showAndLogWarningMessage, showInformationMessageWithAction } from '../helpers'; +import { showAndLogWarningMessage } from '../helpers'; import { URLSearchParams } from 'url'; import { SHOW_QUERY_TEXT_MSG } from '../query-history'; -import { DownloadLink } from './download-link'; +import { AnalysesResultsManager } from './analyses-results-manager'; +import { AnalysisResults } from './shared/analysis-result'; export class RemoteQueriesInterfaceManager { private panel: WebviewPanel | undefined; @@ -35,8 +34,9 @@ export class RemoteQueriesInterfaceManager { private panelLoadedCallBacks: (() => void)[] = []; constructor( - private ctx: ExtensionContext, - private logger: Logger, + private readonly ctx: ExtensionContext, + private readonly logger: Logger, + private readonly analysesResultsManager: AnalysesResultsManager ) { this.panelLoadedCallBacks.push(() => { void logger.log('Remote queries view loaded'); @@ -190,32 +190,31 @@ export class RemoteQueriesInterfaceManager { await this.openVirtualFile(msg.queryText); break; case 'remoteQueryDownloadAnalysisResults': - await this.handleDownloadLinkClicked(msg.downloadLink); + await this.downloadAnalysisResults(msg); break; case 'remoteQueryDownloadAllAnalysesResults': - await this.handleDownloadLinkClicked(msg.downloadLink); + await this.downloadAllAnalysesResults(msg); break; default: assertNever(msg); } } - private async handleDownloadLinkClicked(downloadLink: DownloadLink): Promise { - const credentials = await Credentials.initialize(this.ctx); + private async downloadAnalysisResults(msg: RemoteQueryDownloadAnalysisResultsMessage): Promise { + await this.analysesResultsManager.downloadAnalysisResults(msg.analysisSummary); + await this.setAnalysisResults(this.analysesResultsManager.getFlattenedAnalysesResults()); + } - const filePath = await downloadArtifactFromLink(credentials, downloadLink); - const isDir = (await fs.stat(filePath)).isDirectory(); - const message = `Result file saved at ${filePath}`; - if (isDir) { - await vscode.window.showInformationMessage(message); - } - else { - const shouldOpenResults = await showInformationMessageWithAction(message, 'Open'); - if (shouldOpenResults) { - const textDocument = await vscode.workspace.openTextDocument(filePath); - await vscode.window.showTextDocument(textDocument, vscode.ViewColumn.One); - } - } + private async downloadAllAnalysesResults(msg: RemoteQueryDownloadAllAnalysesResultsMessage): Promise { + await this.analysesResultsManager.downloadAllResults(msg.analysisSummaries); + await this.setAnalysisResults(this.analysesResultsManager.getFlattenedAnalysesResults()); + } + + private async setAnalysisResults(analysesResults: AnalysisResults[]): Promise { + await this.postMessage({ + t: 'setAnalysesResults', + analysesResults: analysesResults + }); } private postMessage(msg: ToRemoteQueriesMessage): Thenable { diff --git a/extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts b/extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts index 974aa526ef9..d90a0220f13 100644 --- a/extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts +++ b/extensions/ql-vscode/src/remote-queries/remote-queries-manager.ts @@ -12,15 +12,18 @@ import { getRemoteQueryIndex } from './gh-actions-api-client'; import { RemoteQueryResultIndex } from './remote-query-result-index'; import { RemoteQueryResult } from './remote-query-result'; import { DownloadLink } from './download-link'; +import { AnalysesResultsManager } from './analyses-results-manager'; export class RemoteQueriesManager { private readonly remoteQueriesMonitor: RemoteQueriesMonitor; + private readonly analysesResultsManager: AnalysesResultsManager; constructor( private readonly ctx: ExtensionContext, private readonly logger: Logger, private readonly cliServer: CodeQLCliServer ) { + this.analysesResultsManager = new AnalysesResultsManager(ctx, logger); this.remoteQueriesMonitor = new RemoteQueriesMonitor(ctx, logger); } @@ -67,7 +70,7 @@ export class RemoteQueriesManager { const shouldOpenView = await showInformationMessageWithAction(message, 'View'); if (shouldOpenView) { - const rqim = new RemoteQueriesInterfaceManager(this.ctx, this.logger); + const rqim = new RemoteQueriesInterfaceManager(this.ctx, this.logger, this.analysesResultsManager); await rqim.showResults(query, queryResult); } } else if (queryResult.status === 'CompletedUnsuccessfully') { diff --git a/extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts b/extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts new file mode 100644 index 00000000000..73872059369 --- /dev/null +++ b/extensions/ql-vscode/src/remote-queries/shared/analysis-result.ts @@ -0,0 +1,8 @@ +export interface AnalysisResults { + nwo: string; + results: QueryResult[]; +} + +export interface QueryResult { + message?: string; +} diff --git a/extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx b/extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx index 54471a8edaa..38226397c9b 100644 --- a/extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx +++ b/extensions/ql-vscode/src/remote-queries/view/RemoteQueries.tsx @@ -1,18 +1,18 @@ import * as React from 'react'; import { useEffect, useState } from 'react'; import * as Rdom from 'react-dom'; -import { SetRemoteQueryResultMessage } from '../../pure/interface-types'; +import { ToRemoteQueriesMessage } from '../../pure/interface-types'; import { AnalysisSummary, RemoteQueryResult } from '../shared/remote-query-result'; import * as octicons from '../../view/octicons'; import { vscode } from '../../view/vscode-api'; -import { DownloadLink } from '../download-link'; import SectionTitle from './SectionTitle'; import VerticalSpace from './VerticalSpace'; import Badge from './Badge'; import ViewTitle from './ViewTitle'; import DownloadButton from './DownloadButton'; +import { AnalysisResults } from '../shared/analysis-result'; const numOfReposInContractedMode = 10; @@ -33,18 +33,17 @@ const emptyQueryResult: RemoteQueryResult = { analysisSummaries: [] }; -const downloadAnalysisResults = (nwo: string, link: DownloadLink) => { +const downloadAnalysisResults = (analysisSummary: AnalysisSummary) => { vscode.postMessage({ t: 'remoteQueryDownloadAnalysisResults', - nwo, - downloadLink: link + analysisSummary }); }; -const downloadAllAnalysesResults = (link: DownloadLink) => { +const downloadAllAnalysesResults = (query: RemoteQueryResult) => { vscode.postMessage({ t: 'remoteQueryDownloadAllAnalysesResults', - downloadLink: link + analysisSummaries: query.analysisSummaries }); }; @@ -86,7 +85,7 @@ const SummaryTitleWithResults = (queryResult: RemoteQueryResult) => ( downloadAllAnalysesResults(queryResult.downloadLink)} /> + onClick={() => downloadAllAnalysesResults(queryResult)} /> ); @@ -104,11 +103,10 @@ const SummaryItem = (props: AnalysisSummary) => ( downloadAnalysisResults(props.nwo, props.downloadLink)} /> + onClick={() => downloadAnalysisResults(props)} /> ); - const Summary = (queryResult: RemoteQueryResult) => { const [repoListExpanded, setRepoListExpanded] = useState(false); const numOfReposToShow = repoListExpanded ? queryResult.analysisSummaries.length : numOfReposInContractedMode; @@ -138,15 +136,59 @@ const Summary = (queryResult: RemoteQueryResult) => { ); }; +const AnalysesResultsTitle = ({ totalAnalysesResults, totalResults }: { totalAnalysesResults: number, totalResults: number }) => { + if (totalAnalysesResults === totalResults) { + return ; + } + + return ; +}; + +const AnalysesResultsDescription = ({ totalAnalysesResults, totalResults }: { totalAnalysesResults: number, totalResults: number }) => { + if (totalAnalysesResults < totalResults) { + return <> + + Some results haven't been downloaded automatically because of their size or because enough were downloaded already. + Download them manually from the list above if you want to see them here. + ; + } + + return <>; +}; + +const AnalysesResults = ({ analysesResults, totalResults }: { analysesResults: AnalysisResults[], totalResults: number }) => { + const totalAnalysesResults = analysesResults.reduce((acc, curr) => acc + curr.results.length, 0); + + if (totalResults === 0) { + return <>; + } + + return ( + <> + + + + + + ); +}; + export function RemoteQueries(): JSX.Element { const [queryResult, setQueryResult] = useState(emptyQueryResult); + const [analysesResults, setAnalysesResults] = useState([]); useEffect(() => { window.addEventListener('message', (evt: MessageEvent) => { if (evt.origin === window.origin) { - const msg: SetRemoteQueryResultMessage = evt.data; + const msg: ToRemoteQueriesMessage = evt.data; if (msg.t === 'setRemoteQueryResult') { setQueryResult(msg.queryResult); + } else if (msg.t === 'setAnalysesResults') { + setAnalysesResults(msg.analysesResults); } } else { // sanitize origin @@ -165,6 +207,7 @@ export function RemoteQueries(): JSX.Element { + ; } catch (err) { console.error(err); From 0d01ded8b09a1b73432e12255523324b453aeab9 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Fri, 21 Jan 2022 16:32:39 +0000 Subject: [PATCH 2/3] Simplify analyses storing --- .../analyses-results-manager.ts | 40 ++++++++++--------- .../remote-queries-interface.ts | 4 +- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts index 196427627d5..80bb6d1c39e 100644 --- a/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts +++ b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts @@ -2,7 +2,6 @@ import { ExtensionContext } from 'vscode'; import { Credentials } from '../authentication'; import { Logger } from '../logging'; import { downloadArtifactFromLink } from './gh-actions-api-client'; -import { DownloadLink } from './download-link'; import * as path from 'path'; import * as fs from 'fs-extra'; import { AnalysisSummary } from './shared/remote-query-result'; @@ -11,27 +10,28 @@ import { AnalysisResults, QueryResult } from './shared/analysis-result'; export class AnalysesResultsManager { // Store for the results of various analyses for a single remote query. - private readonly analysesResults: { [key: string]: QueryResult[] }; + private readonly analysesResults: AnalysisResults[]; constructor( private readonly ctx: ExtensionContext, private readonly logger: Logger, ) { - this.analysesResults = {}; + this.analysesResults = []; } public async downloadAnalysisResults( analysisSummary: AnalysisSummary, ): Promise { + if (this.analysesResults.some(x => x.nwo === analysisSummary.nwo)) { + // We already have the results for this analysis, don't download again. + return; + } + const credentials = await Credentials.initialize(this.ctx); void this.logger.log(`Downloading and processing results for ${analysisSummary.nwo}`); - const queryResults = await this.downloadSingleAnalysisResults( - analysisSummary.downloadLink, - credentials); - - this.analysesResults[analysisSummary.nwo] = queryResults; + await this.downloadSingleAnalysisResults(analysisSummary, credentials); } public async downloadAllResults( @@ -42,27 +42,31 @@ export class AnalysesResultsManager { void this.logger.log('Downloading and processing all results'); for (const analysis of analysisSummaries) { - const queryResults = await this.downloadSingleAnalysisResults(analysis.downloadLink, credentials); - this.analysesResults[analysis.nwo] = queryResults; + await this.downloadSingleAnalysisResults(analysis, credentials); } } - public getFlattenedAnalysesResults(): AnalysisResults[] { - return Object.entries(this.analysesResults).map(([nwo, results]) => ({ nwo, results })); + public getAnalysesResults(): AnalysisResults[] { + return [...this.analysesResults]; } private async downloadSingleAnalysisResults( - downloadLink: DownloadLink, + analysis: AnalysisSummary, credentials: Credentials - ): Promise { - const artifactPath = await downloadArtifactFromLink(credentials, downloadLink); + ): Promise { + const artifactPath = await downloadArtifactFromLink(credentials, analysis.downloadLink); + + let analysisResults: AnalysisResults; if (path.extname(artifactPath) === '.sarif') { - return await this.readResults(artifactPath); + const queryResults = await this.readResults(artifactPath); + analysisResults = { nwo: analysis.nwo, results: queryResults }; } else { - // Non-problem or problem-path queries are not currently fully supported. - return []; + void this.logger.log('Non-problem or problem-path queries are not currently fully supported.'); + analysisResults = { nwo: analysis.nwo, results: [] }; } + + this.analysesResults.push(analysisResults); } private async readResults(filePath: string): Promise { diff --git a/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts b/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts index 5418a550b70..28b671c00ee 100644 --- a/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts +++ b/extensions/ql-vscode/src/remote-queries/remote-queries-interface.ts @@ -202,12 +202,12 @@ export class RemoteQueriesInterfaceManager { private async downloadAnalysisResults(msg: RemoteQueryDownloadAnalysisResultsMessage): Promise { await this.analysesResultsManager.downloadAnalysisResults(msg.analysisSummary); - await this.setAnalysisResults(this.analysesResultsManager.getFlattenedAnalysesResults()); + await this.setAnalysisResults(this.analysesResultsManager.getAnalysesResults()); } private async downloadAllAnalysesResults(msg: RemoteQueryDownloadAllAnalysesResultsMessage): Promise { await this.analysesResultsManager.downloadAllResults(msg.analysisSummaries); - await this.setAnalysisResults(this.analysesResultsManager.getFlattenedAnalysesResults()); + await this.setAnalysisResults(this.analysesResultsManager.getAnalysesResults()); } private async setAnalysisResults(analysesResults: AnalysisResults[]): Promise { From 3860247fcf9e852f4d5e8f69e4c949e02e4ff7b1 Mon Sep 17 00:00:00 2001 From: Charis Kyriakou Date: Mon, 24 Jan 2022 09:42:24 +0000 Subject: [PATCH 3/3] Update extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts Co-authored-by: Andrew Eisenberg --- .../ql-vscode/src/remote-queries/analyses-results-manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts index 80bb6d1c39e..81c6a3408a5 100644 --- a/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts +++ b/extensions/ql-vscode/src/remote-queries/analyses-results-manager.ts @@ -62,7 +62,7 @@ export class AnalysesResultsManager { const queryResults = await this.readResults(artifactPath); analysisResults = { nwo: analysis.nwo, results: queryResults }; } else { - void this.logger.log('Non-problem or problem-path queries are not currently fully supported.'); + void this.logger.log('Cannot download results. Only alert and path queries are fully supported.'); analysisResults = { nwo: analysis.nwo, results: [] }; }