Add integration tests for coverage reports

- Refine the interplay between the flags "dry run" (-d) and "custom
hooks" (-h). Now the custom hooking works even in dry run mode.
- Simplify the FuzzedDataProvider test to decrease the chances of the
test failing on MacOS (this fuzz test is expected to find an exception
within some time)

Author: oetr

docs/fuzz-targets.md

@@ -191,9 +191,10 @@ Alternatively, you can add a new script to your package.json:
 }
 ```
 
-Only the files matched by the `--include` and `--exclude` flags are included in
-the coverage report. It is recommended to disable coverage report generation
-during fuzzing, because of a substantial overhead that it adds.
+Files matched by the flags `--include` and `--custom_hooks`, and not matched by
+the flag `--exclude` will be included in the coverage report. It is recommended
+to disable coverage report generation during fuzzing, because of a substantial
+overhead that it adds.
 
 ### Coverage report directory
 

docs/jest-integration.md

@@ -292,9 +292,10 @@ To generate a coverage report, run jest with the `--coverage` flag:
 » npx jest --coverage
 ```
 
-Only the files matched by the `--include` and `--exclude` flags are included in
-the coverage report. These can be specified in the `package.json` or
-`.jazzerjsrc.json` config files as described above.
+Files matched by the flags `--include` and `--custom_hooks`, and not matched by
+the flag `--exclude` will be included in the coverage report. These can be
+specified in the `package.json` or `.jazzerjsrc.json` config files as described
+above.
 
 By default, the coverage reports can be found in the `./coverage` directory.
 This default directory can be changed by setting the flag

jest.config.js

@@ -18,7 +18,11 @@
 module.exports = {
 	preset: "ts-jest",
 	testEnvironment: "node",
-	modulePathIgnorePatterns: ["dist", "packages/fuzzer/build"],
+	modulePathIgnorePatterns: [
+		"dist",
+		"packages/fuzzer/build",
+		"tests/code_coverage",
+	],
 	collectCoverageFrom: ["packages/**/*.ts"],
 	coveragePathIgnorePatterns: ["/node_modules/", "/dist/"],
 };

packages/core/cli.ts

@@ -185,7 +185,7 @@ yargs(process.argv.slice(2))
 				dryRun: args.dry_run,
 				sync: args.sync,
 				fuzzerOptions: args.corpus.concat(args._),
-				customHooks: args.custom_hooks.map(ensureFilepath),
+				customHooks: args.custom_hooks,
 				expectedErrors: args.expected_errors,
 				coverage: args.coverage,
 				coverageDirectory: args.coverageDirectory,

packages/core/core.test.ts

@@ -15,6 +15,7 @@
  */
 
 import { ensureFilepath } from "./core";
+
 import path from "path";
 
 describe("core", () => {

packages/core/core.ts

@@ -63,10 +63,11 @@ export async function initFuzzing(options: Options) {
 	registerInstrumentor(
 		options.includes,
 		options.excludes,
+		options.customHooks,
 		options.coverage,
 		options.dryRun
 	);
-	await Promise.all(options.customHooks.map(importModule));
+	await Promise.all(options.customHooks.map(ensureFilepath).map(importModule));
 }
 
 export function registerGlobals() {

packages/instrumentor/instrument.ts

@@ -63,7 +63,8 @@ export type FilePredicate = (filepath: string) => boolean;
 export function registerInstrumentor(
 	includes: string[],
 	excludes: string[],
-	shouldCollectCoverage: boolean,
+	customHooks: string[],
+	coverage: boolean,
 	dryRun: boolean
 ) {
 	installSourceMapSupport();
@@ -73,6 +74,11 @@ export function registerInstrumentor(
 
 	const shouldInstrument = shouldInstrumentFn(includes, excludes);
 	const shouldHook = hookManager.hasFunctionsToHook.bind(hookManager);
+	const shouldCollectCoverage = shouldCollectCoverageFn(
+		coverage,
+		customHooks,
+		shouldInstrument
+	);
 	hookRequire(
 		() => true,
 		(code: string, options: TransformerOptions): string =>
@@ -123,24 +129,38 @@ export function shouldInstrumentFn(
 	};
 }
 
+function shouldCollectCoverageFn(
+	coverage: boolean,
+	customHooks: string[],
+	shouldInstrument: FilePredicate = () => true
+): FilePredicate {
+	const shouldCoverCustomHooks = shouldInstrumentFn(customHooks, ["nothing"]);
+	return (filepath: string) => {
+		return (
+			coverage &&
+			(shouldCoverCustomHooks(filepath) || shouldInstrument(filepath))
+		);
+	};
+}
+
 function instrument(
 	code: string,
 	filename: string,
 	shouldInstrument: FilePredicate,
 	shouldHook: FilePredicate,
-	shouldCollectCoverage: boolean,
+	shouldCollectCoverage: FilePredicate,
 	dryRun: boolean
 ) {
 	const transformations: PluginItem[] = [];
 	if (!dryRun) {
 		if (shouldInstrument(filename)) {
 			transformations.push(codeCoverage, compareHooks);
 		}
-		if (shouldHook(filename)) {
-			transformations.push(functionHooks(filename));
-		}
 	}
-	if (shouldCollectCoverage && shouldInstrument(filename)) {
+	if (shouldHook(filename)) {
+		transformations.push(functionHooks(filename));
+	}
+	if (shouldCollectCoverage(filename)) {
 		transformations.push(istanbulCodeCoverage(filename));
 	}
 

tests/FuzzedDataProvider/fuzz.js

@@ -22,11 +22,10 @@ const { FuzzedDataProvider } = require("@jazzer.js/core");
  */
 module.exports.fuzz = function (fuzzerInputData) {
 	const data = new FuzzedDataProvider(fuzzerInputData);
-	const s1 = data.consumeString(data.consumeIntegralInRange(1, 15), "utf-8");
+	const s1 = data.consumeString(data.consumeIntegralInRange(10, 15), "utf-8");
 	const i1 = data.consumeIntegral(1);
 	const i2 = data.consumeIntegral(2);
-	let i3 = 0;
-	if (data.consumeBoolean()) i3 = data.consumeIntegral(4);
+	let i3 = data.consumeIntegral(4);
 
 	if (i3 === 1000) {
 		if (s1 === "Hello World!") {