Skip to content

Commit bf6806f

Browse files
Jason3SJason3S
authored
fix: Support remote dependencies in cache (#7642)
1 parent 64202ec commit bf6806f

File tree

7 files changed

+167
-27
lines changed

7 files changed

+167
-27
lines changed

‎packages/cspell/fixtures/features/cached-remote/README.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
This is a sample fixture for testing caching.

‎packages/cspell/fixtures/features/cached-remote/code.ts

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
/**
2+
* This is some sample code.
3+
*/
4+
5+
function main() {
6+
console.log('hello world');
7+
}
8+
9+
main();

‎packages/cspell/fixtures/features/cached-remote/cspell.json

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
{
2+
"version": "0.2",
3+
"import": ["https://cdn.jsdelivr.net/npm/@cspell/dict-typescript/cspell-ext.json"],
4+
"cache": {
5+
"useCache": true,
6+
"cacheLocation": ".cspellcache",
7+
"cacheFormat": "universal"
8+
},
9+
"files": ["*.md"]
10+
}

‎packages/cspell/src/application.test.ts

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -353,6 +353,27 @@ describe('Linter File Caching', () => {
353353
expect(result, `run #${r}`).toEqual(oc(expected));
354354
}
355355
});
356+
357+
test.each`
358+
runs | root | comment
359+
${[run(['*.ts'], WithCache, fc(1, 0)), run(['*.{md,ts}'], WithCache, fc(2, 1)), run(['*.{md,ts}'], WithCache, fc(2, 2))]} | ${fr('cached-remote')} | ${'cached changing glob three runs U WWW'}
360+
`('lint caching remote with $root $comment', { timeout: 60_000 }, async ({ runs, root }: TestCase) => {
361+
const reporter = new InMemoryReporter();
362+
const cacheLocation = tempLocation('.cspellcache');
363+
await fs.rm(cacheLocation, { recursive: true }).catch(() => undefined);
364+
365+
let r = 0;
366+
367+
for (const run of runs) {
368+
++r;
369+
const { fileGlobs, options, expected } = run;
370+
const useOptions = { ...options, cacheLocation };
371+
useOptions.root = root;
372+
const result = await App.lint(fileGlobs, useOptions, reporter);
373+
expect(reporter.errors).toEqual([]);
374+
expect(result, `run #${r}`).toEqual(oc(expected));
375+
}
376+
});
356377
});
357378

358379
function tempLocation(...parts: string[]): string {

‎packages/cspell/src/util/cache/DiskCache.test.ts

Lines changed: 34 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,6 @@ vi.mock('./file-entry-cache/index.js', () => ({
1414
Promise.resolve({
1515
getFileDescriptor: vi.fn(),
1616
reconcile: vi.fn(),
17-
analyzeFiles: vi.fn().mockReturnValue({
18-
changedFiles: [],
19-
notFoundFiles: [],
20-
notChangedFiles: [],
21-
}),
2217
destroy: vi.fn(() => Promise.resolve()),
2318
}),
2419
),
@@ -37,19 +32,20 @@ const RESULT_NO_ISSUES: CachedFileResult = {
3732
configErrors: 0,
3833
};
3934

35+
const urlCSpellReadme =
36+
'https://raw.githubusercontent.com/streetsidesoftware/vscode-spell-checker/refs/heads/main/README.md';
37+
4038
describe('DiskCache', () => {
4139
let diskCache: DiskCache;
4240
let _fileEntryCache: Promise<{
4341
getFileDescriptor: Mock;
4442
reconcile: Mock;
45-
analyzeFiles: Mock;
4643
destroy: Mock;
4744
}>;
4845

4946
function getFileEntryCache(): Promise<{
5047
getFileDescriptor: Mock;
5148
reconcile: Mock;
52-
analyzeFiles: Mock;
5349
destroy: Mock;
5450
}> {
5551
return _fileEntryCache;
@@ -130,12 +126,6 @@ describe('DiskCache', () => {
130126
const fileEntryCache = await getFileEntryCache();
131127
fileEntryCache.getFileDescriptor.mockReturnValue(entry(RESULT_NO_ISSUES, ['fileA', 'fileB']));
132128

133-
fileEntryCache.analyzeFiles.mockReturnValue({
134-
changedFiles: ['fileA', 'fileB'],
135-
notFoundFiles: [],
136-
notChangedFiles: [],
137-
});
138-
139129
expect(await diskCache.getCachedLintResults('file')).toBeUndefined();
140130
expect(await diskCache.getCachedLintResults('file')).toBeUndefined();
141131
});
@@ -180,6 +170,26 @@ describe('DiskCache', () => {
180170
expect(fileEntryCache.getFileDescriptor).toHaveBeenCalledWith('some-file');
181171
expect(descriptor.meta.data.r).toEqual(result);
182172
});
173+
174+
test('handles remote dependencies', { timeout: 60_000 }, async () => {
175+
const descriptor = { meta: { data: { r: undefined } } };
176+
const fileEntryCache = await getFileEntryCache();
177+
fileEntryCache.getFileDescriptor.mockReturnValue(descriptor);
178+
179+
const result = {
180+
processed: true,
181+
issues: [],
182+
errors: 0,
183+
configErrors: 0,
184+
};
185+
await diskCache.setCachedLintResults(
186+
{ ...result, fileInfo: { filename: 'some-file' }, elapsedTimeMs: 100 },
187+
[urlCSpellReadme, import.meta.url],
188+
);
189+
190+
expect(fileEntryCache.getFileDescriptor).toHaveBeenCalledWith('some-file');
191+
expect(descriptor.meta.data.r).toEqual(result);
192+
});
183193
});
184194

185195
describe('reconcile', () => {
@@ -203,6 +213,17 @@ describe('DiskCache', () => {
203213
});
204214
});
205215

216+
describe('getDependencyForUrl', () => {
217+
test('getDependencyForUrl', { timeout: 60_000 }, async () => {
218+
const url = urlCSpellReadme;
219+
const dep = await __testing__.getDependencyForUrl(url);
220+
expect(dep).toEqual({
221+
f: url,
222+
h: expect.any(String),
223+
});
224+
});
225+
});
226+
206227
function entry(result: CachedFileResult, dependencies: string[] = [], size = 100): { meta: CSpellCacheMeta } {
207228
return {
208229
meta: {

‎packages/cspell/src/util/cache/DiskCache.ts

Lines changed: 45 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
import assert from 'node:assert';
2-
import * as crypto from 'node:crypto';
3-
import * as fs from 'node:fs';
2+
import crypto from 'node:crypto';
3+
import fs from 'node:fs/promises';
44
import {
55
isAbsolute as isAbsolutePath,
66
relative as relativePath,
@@ -9,6 +9,8 @@ import {
99
} from 'node:path';
1010
import { fileURLToPath } from 'node:url';
1111

12+
import { isUrlLike, toFilePathOrHref } from '@cspell/url';
13+
1214
import { readFileInfo } from '../../util/fileHelper.js';
1315
import type { LintFileResult } from '../../util/LintFileResult.js';
1416
import type { CSpellLintResultCache } from './CSpellLintResultCache.js';
@@ -109,7 +111,7 @@ export class DiskCache implements CSpellLintResultCache {
109111
!meta ||
110112
!result ||
111113
!versionMatches ||
112-
!this.checkDependencies(data.d)
114+
!(await this.checkDependencies(data.d))
113115
) {
114116
return undefined;
115117
}
@@ -148,7 +150,7 @@ export class DiskCache implements CSpellLintResultCache {
148150
const data: CachedData = this.objectCollection.get({
149151
v: this.version,
150152
r: this.normalizeResult(result),
151-
d: this.calcDependencyHashes(dependsUponFiles),
153+
d: await this.calcDependencyHashes(dependsUponFiles),
152154
});
153155

154156
meta.data = data;
@@ -174,27 +176,27 @@ export class DiskCache implements CSpellLintResultCache {
174176
return this.ocCacheFileResult.get({ issues, processed, errors, configErrors, reportIssueOptions });
175177
}
176178

177-
private calcDependencyHashes(dependsUponFiles: string[]): Dependency[] {
179+
private async calcDependencyHashes(dependsUponFiles: string[]): Promise<Dependency[]> {
178180
dependsUponFiles.sort();
179181

180182
const c = getTreeEntry(this.dependencyCacheTree, dependsUponFiles);
181183
if (c?.d) {
182184
return c.d;
183185
}
184186

185-
const dependencies: Dependency[] = dependsUponFiles.map((f) => this.getDependency(f));
187+
const dependencies: Dependency[] = await Promise.all(dependsUponFiles.map((f) => this.getDependency(f)));
186188

187189
return setTreeEntry(this.dependencyCacheTree, dependencies);
188190
}
189191

190-
private checkDependency(dep: Dependency): boolean {
192+
private async checkDependency(dep: Dependency): Promise<boolean> {
191193
const depFile = this.resolveFile(dep.f);
192194
const cDep = this.dependencyCache.get(depFile);
193195

194196
if (cDep && compDep(dep, cDep)) return true;
195197
if (cDep) return false;
196198

197-
const d = this.getFileDep(depFile);
199+
const d = await this.getFileDep(depFile);
198200
if (compDep(dep, d)) {
199201
this.dependencyCache.set(depFile, dep);
200202
return true;
@@ -203,31 +205,37 @@ export class DiskCache implements CSpellLintResultCache {
203205
return false;
204206
}
205207

206-
private getDependency(file: string): Dependency {
208+
private async getDependency(file: string): Promise<Dependency> {
207209
const dep = this.dependencyCache.get(file);
208210
if (dep) return dep;
209-
const d = this.getFileDep(file);
211+
const d = await this.getFileDep(file);
210212
this.dependencyCache.set(file, d);
211213
return d;
212214
}
213215

214-
private getFileDep(file: string): Dependency {
216+
private async getFileDep(file: string): Promise<Dependency> {
217+
if (isUrlLike(file)) {
218+
if (!file.startsWith('file://')) {
219+
return getDependencyForUrl(file);
220+
}
221+
file = toFilePathOrHref(file);
222+
}
215223
assert(isAbsolutePath(file), `Dependency must be absolute "${file}"`);
216224
const f = this.toRelFile(file);
217225
let h: string;
218226
try {
219-
const buffer = fs.readFileSync(file);
227+
const buffer = await fs.readFile(file);
220228
h = this.getHash(buffer);
221229
} catch {
222230
return { f };
223231
}
224232
return { f, h };
225233
}
226234

227-
private checkDependencies(dependencies: Dependency[] | undefined): boolean {
235+
private async checkDependencies(dependencies: Dependency[] | undefined): Promise<boolean> {
228236
if (!dependencies) return false;
229237
for (const dep of dependencies) {
230-
if (!this.checkDependency(dep)) {
238+
if (!(await this.checkDependency(dep))) {
231239
return false;
232240
}
233241
}
@@ -239,6 +247,9 @@ export class DiskCache implements CSpellLintResultCache {
239247
}
240248

241249
private resolveFile(file: string): string {
250+
if (isUrlLike(file)) {
251+
return file;
252+
}
242253
return normalizePath(resolvePath(this.cacheDir, file));
243254
}
244255

@@ -247,6 +258,24 @@ export class DiskCache implements CSpellLintResultCache {
247258
}
248259
}
249260

261+
async function getDependencyForUrl(remoteUrl: string | URL): Promise<Dependency> {
262+
const url = new URL(remoteUrl);
263+
264+
try {
265+
// eslint-disable-next-line n/no-unsupported-features/node-builtins
266+
const response = await fetch(url, { method: 'HEAD' });
267+
const h =
268+
response.headers.get('etag') ||
269+
response.headers.get('last-modified') ||
270+
response.headers.get('content-length') ||
271+
'';
272+
return { f: url.href, h: h ? h.trim() : '' };
273+
} catch {
274+
// If the fetch fails, we cannot compute a hash, so we return an empty hash.
275+
return { f: url.href, h: '' };
276+
}
277+
}
278+
250279
export async function createDiskCache(
251280
cacheFileLocation: URL,
252281
useCheckSum: boolean,
@@ -304,6 +333,8 @@ export function normalizePath(filePath: string): string {
304333

305334
export const __testing__: {
306335
calcVersion: typeof calcVersion;
336+
getDependencyForUrl: typeof getDependencyForUrl;
307337
} = {
308338
calcVersion,
339+
getDependencyForUrl,
309340
};

‎packages/cspell/src/util/cache/DummyCache.test.ts

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
import { describe, expect, test } from 'vitest';
2+
3+
import { DummyCache } from './DummyCache.js';
4+
5+
describe('DummyCache', () => {
6+
test('should implement CSpellLintResultCache interface', () => {
7+
const cache = new DummyCache();
8+
expect(cache).toHaveProperty('getCachedLintResults');
9+
expect(cache).toHaveProperty('setCachedLintResults');
10+
expect(cache).toHaveProperty('reconcile');
11+
expect(cache).toHaveProperty('reset');
12+
});
13+
14+
describe('getCachedLintResults', () => {
15+
test('should return undefined', async () => {
16+
const cache = new DummyCache();
17+
18+
const result = await cache.getCachedLintResults();
19+
20+
expect(result).toBeUndefined();
21+
});
22+
});
23+
24+
describe('setCachedLintResults', () => {
25+
test('should resolve without errors', async () => {
26+
const cache = new DummyCache();
27+
28+
await expect(cache.setCachedLintResults()).resolves.toBeUndefined();
29+
});
30+
});
31+
32+
describe('reconcile', () => {
33+
test('should resolve without errors', async () => {
34+
const cache = new DummyCache();
35+
36+
await expect(cache.reconcile()).resolves.toBeUndefined();
37+
});
38+
});
39+
40+
describe('reset', () => {
41+
test('should resolve without errors', async () => {
42+
const cache = new DummyCache();
43+
44+
await expect(cache.reset()).resolves.toBeUndefined();
45+
});
46+
});
47+
});

0 commit comments

Comments
 (0)