Switch return value test to new test wrapper

Author: bertschneider

tests/helpers.js

@@ -44,6 +44,7 @@ class FuzzTest {
 		sync,
 		verbose,
 		coverage,
+		expectedErrors,
 	) {
 		this.customHooks = customHooks;
 		this.dictionaries = dictionaries;
@@ -61,6 +62,7 @@ class FuzzTest {
 		this.sync = sync;
 		this.verbose = verbose;
 		this.coverage = coverage;
+		this.expectedErrors = expectedErrors;
 	}
 
 	execute() {
@@ -71,17 +73,19 @@ class FuzzTest {
 		const options = ["jazzer", this.fuzzFile];
 		options.push("-f " + this.fuzzEntryPoint);
 		if (this.sync) options.push("--sync");
+		if (this.customHooks) options.push("--custom_hooks=" + this.customHooks);
+		if (this.dryRun) options.push("--dry_run=" + this.dryRun);
+		if (this.coverage) options.push("--coverage");
 		for (const bugDetector of this.disableBugDetectors) {
 			options.push("--disable_bug_detectors=" + bugDetector);
 		}
-
-		if (this.customHooks) {
-			options.push("--custom_hooks=" + this.customHooks);
+		for (const expectedError of this.expectedErrors) {
+			options.push("-x=" + expectedError);
 		}
-		options.push("--dry_run=" + this.dryRun);
-		if (this.coverage) options.push("--coverage");
+
 		options.push("--");
-		options.push("-runs=" + this.runs);
+
+		if (this.runs !== undefined) options.push("-runs=" + this.runs);
 		if (this.forkMode) options.push("-fork=" + this.forkMode);
 		options.push("-seed=" + this.seed);
 		for (const dictionary of this.dictionaries) {
@@ -92,7 +96,8 @@ class FuzzTest {
 
 	executeWithJest() {
 		// Put together the libfuzzer options.
-		const fuzzerOptions = ["-runs=" + this.runs, "-seed=" + this.seed];
+		const fuzzerOptions = ["-seed=" + this.seed];
+		if (this.runs !== undefined) fuzzerOptions.push("-runs=" + this.runs);
 		const dictionaries = this.dictionaries.map(
 			(dictionary) => "-dict=" + dictionary,
 		);
@@ -125,6 +130,9 @@ class FuzzTest {
 	}
 
 	runTest(cmd, options, env) {
+		if (this.verbose) {
+			console.log("Running: " + cmd + " " + options.join(" "));
+		}
 		const proc = spawnSync(cmd, options, {
 			stdio: "pipe",
 			cwd: this.dir,
@@ -150,7 +158,7 @@ class FuzzTestBuilder {
 	_sync = false;
 	_runs = 0;
 	_verbose = false;
-	_dryRun = false;
+	_dryRun = undefined;
 	_fuzzEntryPoint = "";
 	_dir = "";
 	_fuzzFile = "fuzz";
@@ -163,6 +171,7 @@ class FuzzTestBuilder {
 	_jestRunInFuzzingMode = false;
 	_dictionaries = [];
 	_coverage = false;
+	_expectedErrors = [];
 
 	/**
 	 * @param {boolean} sync - whether to run the fuzz test in synchronous mode.
@@ -302,6 +311,11 @@ class FuzzTestBuilder {
 		return this;
 	}
 
+	expectedErrors(...expectedError) {
+		this._expectedErrors = expectedError;
+		return this;
+	}
+
 	build() {
 		if (this._jestTestFile === "" && this._fuzzEntryPoint === "") {
 			throw new Error("fuzzEntryPoint or jestTestFile are not set.");
@@ -328,6 +342,7 @@ class FuzzTestBuilder {
 			this._sync,
 			this._verbose,
 			this._coverage,
+			this._expectedErrors,
 		);
 	}
 }

tests/return_values/return_values.test.js

@@ -14,67 +14,63 @@
  * limitations under the License.
  */
 
-const { spawnSync } = require("child_process");
 const path = require("path");
-const SyncInfo =
-	"Exclusively observed synchronous return values from fuzzed function. Fuzzing in synchronous mode seems beneficial!";
-const AsyncInfo =
-	"Observed asynchronous return values from fuzzed function. Fuzzing in asynchronous mode seems beneficial!";
+const { FuzzTestBuilder } = require("../helpers.js");
 
-// current working directory
 const testDirectory = __dirname;
+const syncInfo =
+	"Exclusively observed synchronous return values from fuzzed function. Fuzzing in synchronous mode seems beneficial!";
+const asyncInfo =
+	"Observed asynchronous return values from fuzzed function. Fuzzing in asynchronous mode seems beneficial!";
 
 describe("Execute a sync runner", () => {
 	it("Expect a hint due to async and sync return values", () => {
 		const testCaseDir = path.join(testDirectory, "syncRunnerMixedReturns");
-		const log = executeFuzzTest(true, false, testCaseDir);
-		expect(log).toContain(AsyncInfo.trim());
+		const log = executeFuzzTest(true, true, testCaseDir);
+		expect(log).toContain(asyncInfo.trim());
 	});
 	it("Expect a hint due to exclusively async return values", () => {
 		const testCaseDir = path.join(testDirectory, "syncRunnerAsyncReturns");
 		const log = executeFuzzTest(true, false, testCaseDir);
-		expect(log.trim()).toContain(AsyncInfo.trim());
+		expect(log.trim()).toContain(asyncInfo.trim());
 	});
 	it("Expect no hint due to strict synchronous return values", () => {
 		const testCaseDir = path.join(testDirectory, "syncRunnerSyncReturns");
 		const log = executeFuzzTest(true, false, testCaseDir);
-		expect(log.includes(SyncInfo)).toBeFalsy();
-		expect(log.includes(AsyncInfo)).toBeFalsy();
+		expect(log.includes(syncInfo)).toBeFalsy();
+		expect(log.includes(asyncInfo)).toBeFalsy();
 	});
 });
 
 describe("Execute a async runner", () => {
 	it("Expect no hint due to async and sync return values", () => {
 		const testCaseDir = path.join(testDirectory, "asyncRunnerMixedReturns");
 		const log = executeFuzzTest(false, false, testCaseDir);
-		expect(log.includes(SyncInfo)).toBeFalsy();
-		expect(log.includes(AsyncInfo)).toBeFalsy();
+		expect(log.includes(syncInfo)).toBeFalsy();
+		expect(log.includes(asyncInfo)).toBeFalsy();
 	});
 	it("Expect a hint due to exclusively sync return values", () => {
 		const testCaseDir = path.join(testDirectory, "asyncRunnerSyncReturns");
 		const log = executeFuzzTest(false, false, testCaseDir);
-		expect(log.trim()).toContain(SyncInfo.trim());
+		expect(log.trim()).toContain(syncInfo.trim());
 	});
 	it("Expect no hint due to strict asynchronous return values", () => {
 		const testCaseDir = path.join(testDirectory, "asyncRunnerAsyncReturns");
 		const log = executeFuzzTest(false, false, testCaseDir);
-		expect(log.includes(SyncInfo)).toBeFalsy();
-		expect(log.includes(AsyncInfo)).toBeFalsy();
+		expect(log.includes(syncInfo)).toBeFalsy();
+		expect(log.includes(asyncInfo)).toBeFalsy();
 	});
 });
 
 function executeFuzzTest(sync, verbose, dir) {
-	let options = ["jazzer", "fuzz"];
-	// Specify mode
-	if (sync) options.push("--sync");
-	options.push("--");
-	const process = spawnSync("npx", options, {
-		stdio: "pipe",
-		cwd: dir,
-		shell: true,
-		windowsHide: true,
-	});
-	let stdout = process.output.toString();
-	if (verbose) console.log(stdout);
-	return stdout;
+	const fuzzTest = new FuzzTestBuilder()
+		.fuzzEntryPoint("fuzz")
+		.runs(5000)
+		.dir(dir)
+		.sync(sync)
+		.verbose(verbose)
+		.expectedErrors("Error")
+		.build();
+	fuzzTest.execute();
+	return fuzzTest.stderr;
 }