diff --git a/end-to-end/package.json b/end-to-end/package.json index 9ea3475a..980588ee 100644 --- a/end-to-end/package.json +++ b/end-to-end/package.json @@ -18,6 +18,6 @@ "jest": "^29.4.1", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.2.2" } } diff --git a/end-to-end/tsconfig.json b/end-to-end/tsconfig.json index 7363a7c8..ba34870a 100644 --- a/end-to-end/tsconfig.json +++ b/end-to-end/tsconfig.json @@ -2,7 +2,6 @@ "compilerOptions": { "target": "ES2022", "module": "NodeNext", - "moduleResolution": "node", "allowJs": true, "rootDir": ".", "outDir": "./dist", diff --git a/examples/jest_typescript_integration/package.json b/examples/jest_typescript_integration/package.json index 2a5b2c03..d750aa4a 100644 --- a/examples/jest_typescript_integration/package.json +++ b/examples/jest_typescript_integration/package.json @@ -14,6 +14,6 @@ "jest": "^29.4.1", "ts-jest": "^29.0.5", "ts-node": "^10.9.1", - "typescript": "^4.9.5" + "typescript": "^5.2.2" } } diff --git a/examples/jest_typescript_integration/tsconfig.json b/examples/jest_typescript_integration/tsconfig.json index 7363a7c8..ba34870a 100644 --- a/examples/jest_typescript_integration/tsconfig.json +++ b/examples/jest_typescript_integration/tsconfig.json @@ -2,7 +2,6 @@ "compilerOptions": { "target": "ES2022", "module": "NodeNext", - "moduleResolution": "node", "allowJs": true, "rootDir": ".", "outDir": "./dist", diff --git a/examples/js-yaml/package.json b/examples/js-yaml/package.json index 93a0a4e6..37de7a06 100644 --- a/examples/js-yaml/package.json +++ b/examples/js-yaml/package.json @@ -10,7 +10,7 @@ "devDependencies": { "@jazzer.js/core": "file:../../packages/core", "@types/js-yaml": "^4.0.5", - "typescript": "^4.7.4" + "typescript": "^5.2.2" }, "dependencies": { "js-yaml": "^4.1.0" diff --git a/examples/js-yaml/tsconfig.json b/examples/js-yaml/tsconfig.json index dcbfa352..7e1e15e0 100644 --- a/examples/js-yaml/tsconfig.json +++ b/examples/js-yaml/tsconfig.json @@ -2,7 +2,6 @@ "compilerOptions": { "target": "ES2022", "module": "commonjs", - "moduleResolution": "node", "allowJs": true, "checkJs": true, "rootDir": ".", diff --git a/packages/jest-runner/index.ts b/packages/jest-runner/index.ts index 78bd6098..ccc1cd66 100644 --- a/packages/jest-runner/index.ts +++ b/packages/jest-runner/index.ts @@ -17,6 +17,7 @@ import type { JestEnvironment } from "@jest/environment"; import { TestResult } from "@jest/test-result"; import { Config } from "@jest/types"; +import * as libCoverage from "istanbul-lib-coverage"; import * as reports from "istanbul-reports"; import Runtime from "jest-runtime"; @@ -71,6 +72,7 @@ export default async function jazzerTestRunner( testPath, sendMessageToJest, ).then((result: TestResult) => { + includeImplicitElseBranches(environment.global.__coverage__); return cleanupTestResultDetails(result); }); } @@ -90,6 +92,52 @@ function cleanupTestResultDetails(result: TestResult) { return result; } +/** + * Coverage fix from https://github.com/vitest-dev/vitest/pull/2275 + * In our tests this seems to only affect the coverage of TypeScript files, + * hence including the fix in jest-runner should be sufficient. + * + * Original comment: + * Work-around for #1887 and #2239 while waiting for https://github.com/istanbuljs/istanbuljs/pull/706 + * Goes through all files in the coverage map and checks if branchMap's have + * if-statements with implicit else. When finds one, copies source location of + * the if-statement into the else statement. + */ +export function includeImplicitElseBranches( + coverageMapData: libCoverage.CoverageMapData, +) { + if (!coverageMapData) { + return; + } + function isEmptyCoverageRange(range: libCoverage.Range) { + return ( + range.start === undefined || + range.start.line === undefined || + range.start.column === undefined || + range.end === undefined || + range.end.line === undefined || + range.end.column === undefined + ); + } + const coverageMap = libCoverage.createCoverageMap(coverageMapData); + for (const file of coverageMap.files()) { + const fileCoverage = coverageMap.fileCoverageFor(file); + for (const branchMap of Object.values(fileCoverage.branchMap)) { + if (branchMap.type === "if") { + const lastIndex = branchMap.locations.length - 1; + if (lastIndex > 0) { + const elseLocation = branchMap.locations[lastIndex]; + if (elseLocation && isEmptyCoverageRange(elseLocation)) { + const ifLocation = branchMap.locations[0]; + elseLocation.start = { ...ifLocation.start }; + elseLocation.end = { ...ifLocation.end }; + } + } + } + } + } +} + // Global definition of the Jest fuzz test extension function. // This is required to allow the Typescript compiler to recognize it. declare global { diff --git a/packages/jest-runner/package.json b/packages/jest-runner/package.json index 34acb565..3667c564 100644 --- a/packages/jest-runner/package.json +++ b/packages/jest-runner/package.json @@ -1,7 +1,7 @@ { "name": "@jazzer.js/jest-runner", "version": "2.0.0", - "description": "", + "description": "Jazzer.js Jest runner", "homepage": "https://github.com/CodeIntelligenceTesting/jazzer.js#readme", "author": "Code Intelligence", "license": "Apache-2.0", @@ -20,6 +20,10 @@ "cosmiconfig": "^8.3.6", "istanbul-reports": "^3.1.6" }, + "peerDependencies": { + "@types/jest": "29.*", + "jest": "29.*" + }, "devDependencies": { "@types/tmp": "^0.2.4", "tmp": "^0.2.1" diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json index f868e98d..1d66342b 100644 --- a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+codeCoverage-fuzz.json @@ -104,7 +104,10 @@ "start": { "line": 24, "column": 1 }, "end": { "line": 26, "column": 2 } }, - { "start": {}, "end": {} } + { + "start": { "line": 24, "column": 1 }, + "end": { "line": 26, "column": 2 } + } ], "line": 24 } @@ -164,7 +167,10 @@ "start": { "line": 19, "column": 1 }, "end": { "line": 21, "column": 2 } }, - { "start": {}, "end": {} } + { + "start": { "line": 19, "column": 1 }, + "end": { "line": 21, "column": 2 } + } ], "line": 19 } diff --git a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json index 5c25053d..b4283dd1 100644 --- a/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json +++ b/tests/code_coverage/sample_fuzz_test/expected_coverage/fuzz+lib+otherCodeCoverage-fuzz.json @@ -113,7 +113,10 @@ "start": { "line": 24, "column": 1 }, "end": { "line": 26, "column": 2 } }, - { "start": {}, "end": {} } + { + "start": { "line": 24, "column": 1 }, + "end": { "line": 26, "column": 2 } + } ], "line": 24 } @@ -173,7 +176,10 @@ "start": { "line": 19, "column": 1 }, "end": { "line": 21, "column": 2 } }, - { "start": {}, "end": {} } + { + "start": { "line": 19, "column": 1 }, + "end": { "line": 21, "column": 2 } + } ], "line": 19 } diff --git a/tests/code_coverage/sample_fuzz_test/package.json b/tests/code_coverage/sample_fuzz_test/package.json index afa1d2e9..a7d9609b 100644 --- a/tests/code_coverage/sample_fuzz_test/package.json +++ b/tests/code_coverage/sample_fuzz_test/package.json @@ -10,7 +10,7 @@ "@jazzer.js/jest-runner": "file:../../../packages/jest-runner", "jest": "^29.4.1", "ts-jest": "^29.0.5", - "typescript": "^4.9.5" + "typescript": "^5.2.2" }, "jest": { "projects": [ diff --git a/tests/helpers.js b/tests/helpers.js index 84c17275..9a1b5b1f 100644 --- a/tests/helpers.js +++ b/tests/helpers.js @@ -245,8 +245,8 @@ class FuzzTestBuilder { * @param {boolean} logTestOutput - whether to print the output of the fuzz test to the console. * True if parameter is undefined. */ - logTestOutput(logTestOutput) { - this._logTestOutput = logTestOutput === undefined ? true : logTestOutput; + logTestOutput(logTestOutput = true) { + this._logTestOutput = logTestOutput; return this; } @@ -270,8 +270,8 @@ class FuzzTestBuilder { /** * @param {boolean} verbose - set verbose/debug output in fuzz test. */ - verbose(verbose) { - this._verbose = verbose === undefined ? true : verbose; + verbose(verbose = true) { + this._verbose = verbose; return this; } @@ -279,9 +279,8 @@ class FuzzTestBuilder { * @param {boolean} listFuzzTestNames - whether to list all fuzz test names on the console. * True if parameter is undefined. */ - listFuzzTestNames(listFuzzTestNames) { - this._listFuzzTestNames = - listFuzzTestNames === undefined ? true : listFuzzTestNames; + listFuzzTestNames(listFuzzTestNames = true) { + this._listFuzzTestNames = listFuzzTestNames; if (this._listFuzzTestNames) { this.jestTestName("__NOT_AN_ACTUAL_TESTNAME__"); } @@ -425,7 +424,7 @@ class FuzzTestBuilder { return this; } - coverage(coverage) { + coverage(coverage = true) { this._coverage = coverage; return this; } @@ -435,8 +434,8 @@ class FuzzTestBuilder { return this; } - asJson(asJson) { - this._asJson = asJson === undefined ? true : asJson; + asJson(asJson = true) { + this._asJson = asJson; return this; } diff --git a/tests/jest_integration/integration.test.js b/tests/jest_integration/integration.test.js index e7572aff..2e6f8562 100644 --- a/tests/jest_integration/integration.test.js +++ b/tests/jest_integration/integration.test.js @@ -27,20 +27,9 @@ const { describe("Jest integration", () => { const projectDir = path.join(__dirname, "jest_project"); const jestTestFile = "integration.fuzz"; + const expectCrashFileIn = expectCrashFileInProject(projectDir); - beforeEach(() => { - fs.rmSync(path.join(projectDir, ".jazzerjsrc.json"), { - force: true, - }); - fs.rmSync(path.join(projectDir, ".cifuzz-corpus"), { - force: true, - recursive: true, - }); - fs.rmSync(path.join(projectDir, jestTestFile), { - force: true, - recursive: true, - }); - }); + beforeEach(cleanupProjectFiles(projectDir, jestTestFile)); describe("Fuzzing mode", () => { let fuzzTestBuilder; @@ -421,12 +410,89 @@ describe("Jest integration", () => { expect(listFuzzTestNames.stdout).toBe(""); }); }); +}); - async function expectCrashFileIn(crashDir) { - const crashFiles = await cleanCrashFilesIn(projectDir); - expect(crashFiles).toHaveLength(1); - expect(crashFiles[0]).toContain(crashDir); - } +describe("Jest TS integration", () => { + const projectDir = path.join(__dirname, "jest_project_ts"); + const jestTsTestFile = "integration.fuzz"; + const expectCrashFileIn = expectCrashFileInProject(projectDir); + + beforeEach(cleanupProjectFiles(projectDir, jestTsTestFile)); + + describe("Fuzzing mode", () => { + let fuzzTestBuilder; + + beforeEach(() => { + fuzzTestBuilder = new FuzzTestBuilder() + .dir(projectDir) + .runs(1_000_000) + .jestRunInFuzzingMode(true) + .jestTestFile(jestTsTestFile + ".ts"); + }); + + describe("execute", () => { + it("execute sync test", async () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute sync test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + await expectCrashFileIn("execute_sync_test"); + }); + + it("execute async test", async () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + await expectCrashFileIn("execute_async_test"); + }); + + it("execute async test returning a promise", async () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test returning a promise") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + await expectCrashFileIn("execute_async_test_returning_a_promise"); + }); + + it("execute async test using a callback", async () => { + const fuzzTest = fuzzTestBuilder + .jestTestName("execute async test using a callback") + .build(); + expect(() => { + fuzzTest.execute(); + }).toThrow(JestRegressionExitCode); + await expectCrashFileIn("execute_async_test_using_a_callback"); + }); + }); + }); + + describe("Regression mode", () => { + const regressionTestBuilder = new FuzzTestBuilder() + .dir(projectDir) + .jestTestFile(jestTsTestFile + ".ts"); + + describe("mixed features", () => { + it("cover implicit else branch", async () => { + regressionTestBuilder + .jestTestName("execute sync test") + .coverage() + .build() + .execute(); + const coverage = readCoverageOf(projectDir, "target.ts"); + // Expect that every branch has two entries, one for the if and one for the else branch. + Object.keys(coverage.b).forEach((branch) => { + expect(coverage.b[branch]).toHaveLength(2); + }); + }); + }); + }); }); // Deflake the "timeout after N seconds" test to be more tolerant to small variations of N (+-1). @@ -444,3 +510,42 @@ function firstFailureMessage(result) { (result) => result.status === "failed", )[0].failureMessages[0]; } + +function expectCrashFileInProject(projectDir) { + return (crashDir) => async () => { + const crashFiles = await cleanCrashFilesIn(projectDir); + expect(crashFiles).toHaveLength(1); + expect(crashFiles[0]).toContain(crashDir); + }; +} + +function cleanupProjectFiles(projectDir, jestTestFile) { + return () => { + fs.rmSync(path.join(projectDir, ".jazzerjsrc.json"), { + force: true, + }); + fs.rmSync(path.join(projectDir, ".cifuzz-corpus"), { + force: true, + recursive: true, + }); + fs.rmSync(path.join(projectDir, jestTestFile), { + force: true, + recursive: true, + }); + }; +} + +function readCoverageOf(projectDir, fileName) { + const report = JSON.parse( + fs.readFileSync( + path.join(projectDir, "coverage", "coverage-final.json"), + "utf-8", + ), + ); + for (let path of Object.keys(report)) { + if (path.endsWith(fileName)) { + return report[path]; + } + } + throw new Error("Could not find coverage for " + fileName); +} diff --git a/tests/jest_integration/jest_project_with_single_test/.gitignore b/tests/jest_integration/jest_project_ts/.gitignore similarity index 100% rename from tests/jest_integration/jest_project_with_single_test/.gitignore rename to tests/jest_integration/jest_project_ts/.gitignore diff --git a/tests/jest_integration/jest_project_ts/integration.fuzz.ts b/tests/jest_integration/jest_project_ts/integration.fuzz.ts new file mode 100644 index 00000000..20a437ac --- /dev/null +++ b/tests/jest_integration/jest_project_ts/integration.fuzz.ts @@ -0,0 +1,40 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import "@jazzer.js/jest-runner"; + +import * as target from "./target"; + +describe("Jest TS Integration", () => { + it.fuzz("execute sync test", (data: Buffer) => { + target.fuzzMe(data); + }); + + it.fuzz("execute async test", async (data: Buffer) => { + await target.asyncFuzzMe(data); + }); + + it.fuzz("execute async test returning a promise", (data: Buffer) => { + return target.asyncFuzzMe(data); + }); + + it.fuzz( + "execute async test using a callback", + (data: Buffer, done: (e?: Error) => void) => { + target.callbackFuzzMe(data, done); + }, + ); +}); diff --git a/tests/jest_integration/jest_project_with_single_test/integration.fuzz.js b/tests/jest_integration/jest_project_ts/jest.config.ts similarity index 63% rename from tests/jest_integration/jest_project_with_single_test/integration.fuzz.js rename to tests/jest_integration/jest_project_ts/jest.config.ts index 946711a1..f27e0e8c 100644 --- a/tests/jest_integration/jest_project_with_single_test/integration.fuzz.js +++ b/tests/jest_integration/jest_project_ts/jest.config.ts @@ -14,10 +14,22 @@ * limitations under the License. */ -describe("Jest Integration", () => { - test.fuzz("one test only", (data) => { - if (data.toString() === "Welcome") { - throw Error("Welcome to Awesome Fuzzing!"); - } - }); -}); +import type { Config } from "jest"; + +const config: Config = { + projects: [ + { + displayName: { + name: "Jazzer.js", + color: "cyan", + }, + preset: "ts-jest", + testRunner: "@jazzer.js/jest-runner", + testEnvironment: "node", + testMatch: ["/*.fuzz.ts"], + }, + ], + collectCoverageFrom: ["*.ts"], +}; + +export default config; diff --git a/tests/jest_integration/jest_project_ts/package.json b/tests/jest_integration/jest_project_ts/package.json new file mode 100644 index 00000000..2cbe91f8 --- /dev/null +++ b/tests/jest_integration/jest_project_ts/package.json @@ -0,0 +1,17 @@ +{ + "name": "jazzerjs-jest-integration-tests-project-ts", + "version": "1.0.0", + "scripts": { + "build": "tsc", + "test": "jest", + "fuzz": "JAZZER_FUZZ=1 jest" + }, + "devDependencies": { + "@jazzer.js/jest-runner": "file:../../../packages/jest-runner", + "@types/jest": "^29.4.0", + "jest": "^29.4.1", + "ts-jest": "^29.0.5", + "ts-node": "^10.9.1", + "typescript": "^5.2.2" + } +} diff --git a/tests/jest_integration/jest_project_ts/target.ts b/tests/jest_integration/jest_project_ts/target.ts new file mode 100644 index 00000000..c561148c --- /dev/null +++ b/tests/jest_integration/jest_project_ts/target.ts @@ -0,0 +1,56 @@ +/* + * Copyright 2023 Code Intelligence GmbH + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export function fuzzMe(data: Buffer) { + if (typeof data === "object") { + if (data.toString() === "Awesome") { + throw Error("Welcome to Awesome Fuzzing!"); + } + return data; + } + // Implicit else block to test coverage error, + // see: https://github.com/vitest-dev/vitest/pull/2275 + return data; +} + +export function callbackFuzzMe(data: Buffer, done: (e?: Error) => void) { + // Use setImmediate here to unblock the event loop but still have better + // performance compared to setTimeout. + setImmediate(() => { + try { + fuzzMe(data); + done(); + } catch (e: unknown) { + if (e instanceof Error) { + done(e); + } else { + done(new Error(`Error: ${e}`)); + } + } + }); +} + +export async function asyncFuzzMe(data: Buffer) { + return new Promise((resolve, reject) => { + callbackFuzzMe(data, (e?: Error) => { + if (e) { + reject(e); + } else { + resolve(null); + } + }); + }); +} diff --git a/tests/jest_integration/jest_project_ts/tsconfig.json b/tests/jest_integration/jest_project_ts/tsconfig.json new file mode 100644 index 00000000..ba34870a --- /dev/null +++ b/tests/jest_integration/jest_project_ts/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "allowJs": true, + "rootDir": ".", + "outDir": "./dist", + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "strict": true, + "skipLibCheck": true, + "declaration": true, + "composite": true, + "sourceMap": true + } +} diff --git a/tests/jest_integration/jest_project_with_single_test/package.json b/tests/jest_integration/jest_project_with_single_test/package.json deleted file mode 100644 index ddb80201..00000000 --- a/tests/jest_integration/jest_project_with_single_test/package.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "name": "jazzerjs-jest-integration-tests-project-single-test", - "version": "1.0.0", - "scripts": { - "test": "jest", - "fuzz": "JAZZER_FUZZ=1 jest " - }, - "devDependencies": { - "@jazzer.js/jest-runner": "file:../../../packages/jest-runner", - "jest": "^29.3.1" - }, - "jest": { - "projects": [ - { - "displayName": "test" - }, - { - "testRunner": "@jazzer.js/jest-runner", - "displayName": { - "name": "Jazzer.js", - "color": "cyan" - }, - "testMatch": [ - "/**/*.fuzz.js" - ] - } - ] - } -} diff --git a/tests/jest_integration/package.json b/tests/jest_integration/package.json index 8d8c07fd..22d90606 100644 --- a/tests/jest_integration/package.json +++ b/tests/jest_integration/package.json @@ -7,9 +7,12 @@ }, "devDependencies": { "@types/jest": "^29.5.3", - "jest": "^29.6.2" + "jest": "^29.6.2", + "ts-jest": "^29.0.5", + "ts-node": "^10.9.1", + "typescript": "^5.2.2" }, "jest": { - "testTimeout": 30000 + "testTimeout": 60000 } } diff --git a/tsconfig.json b/tsconfig.json index d15f1423..beb467a8 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -2,7 +2,6 @@ "compilerOptions": { "target": "ES2022", "module": "NodeNext", - "moduleResolution": "NodeNext", "baseUrl": "./", "allowJs": true, "checkJs": true,