GODRIVER-2550 Add fuzzer to bson packages (mongodb#1077)

Author: prestonvasquez

.evergreen/config.yml

@@ -15,9 +15,8 @@ command_type: setup
 # Fail builds when pre tasks fail.
 pre_error_fails_task: true
 
-# Protect ourself against rogue test case, or curl gone wild, that runs forever
-# 12 minutes is the longest we'll ever run
-exec_timeout_secs: 3600 # 12 minutes is the longest we'll ever run
+# Protect the CI from long or indefinite runtimes.
+exec_timeout_secs: 3600
 
 # What to do when evergreen hits the timeout (`post:` tasks are run automatically)
 timeout:
@@ -203,6 +202,16 @@ functions:
         permissions: public-read
         content_type: ${content_type|application/x-gzip}
         display_name: "mongodb-logs.tar.gz"
+    - command: s3.put
+      params:
+        aws_key: ${aws_key}
+        aws_secret: ${aws_secret}
+        local_file: ${PROJECT_DIRECTORY}/fuzz.tgz
+        remote_file: ${UPLOAD_BUCKET}/${build_variant}/${revision}/${version_id}/${build_id}/${task_id}-${execution}-fuzz.tgz
+        bucket: mciuploads
+        permissions: public-read
+        content_type: application/x-gzip
+        display_name: "fuzz.tgz"
 
   bootstrap-mongohoused:
     - command: shell.exec
@@ -1004,6 +1013,14 @@ functions:
           PKG_CONFIG_PATH=$PKG_CONFIG_PATH \
           LD_LIBRARY_PATH=$LD_LIBRARY_PATH
 
+  run-fuzz-tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: "src"
+        script: |
+          ${PREPARE_SHELL}
+          ${PROJECT_DIRECTORY}/.evergreen/run-fuzz.sh
 pre:
   - func: fetch-source
   - func: prepare-resources
@@ -1983,6 +2000,11 @@ tasks:
             EXPECT_ERROR='unable to retrieve GCP credentials' \
               ./testgcpkms
 
+  - name: "test-fuzz"
+    commands:
+      - func: bootstrap-mongo-orchestration
+      - func: run-fuzz-tests
+
 axes:
   - id: version
     display_name: MongoDB Version
@@ -2378,6 +2400,12 @@ buildvariants:
     tasks:
       - name: ".kms-kmip"
 
+  - matrix_name: "fuzz-test"
+    matrix_spec: { version: ["5.0"], os-ssl-40: ["ubuntu1804-64-go-1-18"] }
+    display_name: "Fuzz ${version} ${os-ssl-40}"
+    tasks:
+      - name: "test-fuzz"
+
   - name: testgcpkms-variant
     display_name: "GCP KMS"
     run_on:

.evergreen/run-fuzz.sh

@@ -0,0 +1,68 @@
+#!/bin/bash
+
+set -o errexit  # Exit the script with error if any of the commands fail
+
+FUZZTIME=10m
+
+# Change the working directory to the root of the mongo repository directory
+cd $PROJECT_DIRECTORY
+
+# Get all go test files that contain a fuzz test.
+FILES=$(grep -r --include='**_test.go' --files-with-matches 'func Fuzz' .)
+
+# For each file, run all of the fuzz tests in sequence, each for -fuzztime=FUZZTIME.
+for FILE in ${FILES}
+do
+	PARENTDIR="$(dirname -- "$FILE")"
+
+	# Get a list of all fuzz tests in the file.
+	FUNCS=$(grep -o 'func Fuzz[A-Za-z0-9]*' $FILE | cut -d' ' -f2)
+
+	# For each fuzz test in the file, run it for FUZZTIME.
+	for FUNC in ${FUNCS}
+	do
+		echo "Fuzzing \"${FUNC}\" in \"${FILE}\""
+
+		# Create a set of directories that are already in the subdirectories testdata/fuzz/$fuzzer corpus. This
+		# set will be used to differentiate between new and old corpus files.
+		declare -a cset
+
+		if [ -d $PARENTDIR/testdata/fuzz/$FUNC ]; then
+			# Iterate over the files in the corpus directory and add them to the set.
+			for SEED in $PARENTDIR/testdata/fuzz/$FUNC/*
+			do
+				cset+=("$SEED")
+			done
+		fi
+
+		go test ${PARENTDIR} -run=${FUNC} -fuzz=${FUNC} -fuzztime=${FUZZTIME} || true
+
+		# Check if any new corpus files were generated for the fuzzer. If there are new corpus files, move them
+		# to $PROJECT_DIRECTORY/fuzz/$FUNC/* so they can be tarred up and uploaded to S3.
+		if [ -d $PARENTDIR/testdata/fuzz/$FUNC ]; then
+			# Iterate over the files in the corpus directory and check if they are in the set.
+			for CORPUS_FILE in $PARENTDIR/testdata/fuzz/$FUNC/*
+			do
+				# Check to see if the value for CORPUS_FILE is in cset.
+				if [[ ! " ${cset[@]} " =~ " ${CORPUS_FILE} " ]]; then
+					# Create the directory if it doesn't exist.
+					if [ ! -d $PROJECT_DIRECTORY/fuzz/$FUNC ]; then
+						mkdir -p $PROJECT_DIRECTORY/fuzz/$FUNC
+					fi
+
+					# Move the file to the directory.
+					mv $CORPUS_FILE $PROJECT_DIRECTORY/fuzz/$FUNC
+
+					echo "Moved $CORPUS_FILE to $PROJECT_DIRECTORY/fuzz/$FUNC"
+				fi
+			done
+		fi
+	done
+done
+
+# If the fuzz directory exists, then tar it up in preparation to upload to S3.
+if [ -d $PROJECT_DIRECTORY/fuzz ]; then
+	echo "Tarring up fuzz directory"
+	tar -czf $PROJECT_DIRECTORY/fuzz.tgz $PROJECT_DIRECTORY/fuzz
+fi
+

bson/bson_corpus_spec_test.go

@@ -10,8 +10,8 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"math"
+	"os"
 	"path"
 	"strconv"
 	"strings"
@@ -60,11 +60,13 @@ type parseErrorTestCase struct {
 
 const dataDir = "../testdata/bson-corpus/"
 
-func findJSONFilesInDir(t *testing.T, dir string) []string {
+func findJSONFilesInDir(dir string) ([]string, error) {
 	files := make([]string, 0)
 
-	entries, err := ioutil.ReadDir(dir)
-	require.NoError(t, err)
+	entries, err := os.ReadDir(dir)
+	if err != nil {
+		return nil, err
+	}
 
 	for _, entry := range entries {
 		if entry.IsDir() || path.Ext(entry.Name()) != ".json" {
@@ -74,7 +76,65 @@ func findJSONFilesInDir(t *testing.T, dir string) []string {
 		files = append(files, entry.Name())
 	}
 
-	return files
+	return files, nil
+}
+
+// seedExtJSON will add the byte representation of the "extJSON" string to the fuzzer's coprus.
+func seedExtJSON(f *testing.F, extJSON string, extJSONType string, desc string) {
+	jbytes, err := jsonToBytes(extJSON, extJSONType, desc)
+	if err != nil {
+		f.Fatalf("failed to convert JSON to bytes: %v", err)
+	}
+
+	f.Add(jbytes)
+}
+
+// seedTestCase will add the byte representation for each "extJSON" string of each valid test case to the fuzzer's
+// corpus.
+func seedTestCase(f *testing.F, tcase *testCase) {
+	for _, vtc := range tcase.Valid {
+		seedExtJSON(f, vtc.CanonicalExtJSON, "canonical", vtc.Description)
+
+		// Seed the relaxed extended JSON.
+		if vtc.RelaxedExtJSON != nil {
+			seedExtJSON(f, *vtc.RelaxedExtJSON, "relaxed", vtc.Description)
+		}
+
+		// Seed the degenerate extended JSON.
+		if vtc.DegenerateExtJSON != nil {
+			seedExtJSON(f, *vtc.DegenerateExtJSON, "degenerate", vtc.Description)
+		}
+
+		// Seed the converted extended JSON.
+		if vtc.ConvertedExtJSON != nil {
+			seedExtJSON(f, *vtc.ConvertedExtJSON, "converted", vtc.Description)
+		}
+	}
+}
+
+// seedBSONCorpus will unmarshal the data from "testdata/bson-corpus" into a slice of "testCase" structs and then
+// marshal the "*_extjson" field of each "validityTestCase" into a slice of bytes to seed the fuzz corpus.
+func seedBSONCorpus(f *testing.F) {
+	fileNames, err := findJSONFilesInDir(dataDir)
+	if err != nil {
+		f.Fatalf("failed to find JSON files in directory %q: %v", dataDir, err)
+	}
+
+	for _, fileName := range fileNames {
+		filePath := path.Join(dataDir, fileName)
+
+		file, err := os.Open(filePath)
+		if err != nil {
+			f.Fatalf("failed to open file %q: %v", filePath, err)
+		}
+
+		var tcase testCase
+		if err := json.NewDecoder(file).Decode(&tcase); err != nil {
+			f.Fatal(err)
+		}
+
+		seedTestCase(f, &tcase)
+	}
 }
 
 func needsEscapedUnicode(bsonType string) bool {
@@ -196,11 +256,27 @@ func nativeToBSON(t *testing.T, cB []byte, doc D, testDesc, bType, docSrcDesc st
 }
 
 // jsonToNative decodes the extended JSON string (ej) into a native Document
-func jsonToNative(t *testing.T, ej, ejType, testDesc string) D {
+func jsonToNative(ej, ejType, testDesc string) (D, error) {
 	var doc D
-	err := UnmarshalExtJSON([]byte(ej), ejType != "relaxed", &doc)
-	expectNoError(t, err, fmt.Sprintf("%s: decoding %s extended JSON", testDesc, ejType))
-	return doc
+	if err := UnmarshalExtJSON([]byte(ej), ejType != "relaxed", &doc); err != nil {
+		return nil, fmt.Errorf("%s: decoding %s extended JSON: %w", testDesc, ejType, err)
+	}
+	return doc, nil
+}
+
+// jsonToBytes decodes the extended JSON string (ej) into canonical BSON and then encodes it into a byte slice.
+func jsonToBytes(ej, ejType, testDesc string) ([]byte, error) {
+	native, err := jsonToNative(ej, ejType, testDesc)
+	if err != nil {
+		return nil, err
+	}
+
+	b, err := Marshal(native)
+	if err != nil {
+		return nil, fmt.Errorf("%s: encoding %s BSON: %w", testDesc, ejType, err)
+	}
+
+	return b, nil
 }
 
 // nativeToJSON encodes the native Document (doc) into an extended JSON string
@@ -217,7 +293,7 @@ func nativeToJSON(t *testing.T, ej string, doc D, testDesc, ejType, ejShortName,
 
 func runTest(t *testing.T, file string) {
 	filepath := path.Join(dataDir, file)
-	content, err := ioutil.ReadFile(filepath)
+	content, err := os.ReadFile(filepath)
 	require.NoError(t, err)
 
 	// Remove ".json" from filename.
@@ -260,14 +336,16 @@ func runTest(t *testing.T, file string) {
 						nativeToJSON(t, rEJ, doc, v.Description, "relaxed", "rEJ", "bson_to_native(cB)")
 
 						/*** relaxed extended JSON round-trip tests (if exists) ***/
-						doc = jsonToNative(t, rEJ, "relaxed", v.Description)
+						doc, err = jsonToNative(rEJ, "relaxed", v.Description)
+						require.NoError(t, err)
 
 						// native_to_relaxed_extended_json(json_to_native(rEJ)) = rEJ
 						nativeToJSON(t, rEJ, doc, v.Description, "relaxed", "eJR", "json_to_native(rEJ)")
 					}
 
 					/*** canonical extended JSON round-trip tests ***/
-					doc = jsonToNative(t, cEJ, "canonical", v.Description)
+					doc, err = jsonToNative(cEJ, "canonical", v.Description)
+					require.NoError(t, err)
 
 					// native_to_canonical_extended_json(json_to_native(cEJ)) = cEJ
 					nativeToJSON(t, cEJ, doc, v.Description, "canonical", "cEJ", "json_to_native(cEJ)")
@@ -295,7 +373,8 @@ func runTest(t *testing.T, file string) {
 							dEJ = normalizeCanonicalDouble(t, *test.TestKey, dEJ)
 						}
 
-						doc = jsonToNative(t, dEJ, "degenerate canonical", v.Description)
+						doc, err = jsonToNative(dEJ, "degenerate canonical", v.Description)
+						require.NoError(t, err)
 
 						// native_to_canonical_extended_json(json_to_native(dEJ)) = cEJ
 						nativeToJSON(t, cEJ, doc, v.Description, "degenerate canonical", "cEJ", "json_to_native(dEJ)")
@@ -366,7 +445,12 @@ func runTest(t *testing.T, file string) {
 }
 
 func Test_BsonCorpus(t *testing.T) {
-	for _, file := range findJSONFilesInDir(t, dataDir) {
+	jsonFiles, err := findJSONFilesInDir(dataDir)
+	if err != nil {
+		t.Fatalf("error finding JSON files in %s: %v", dataDir, err)
+	}
+
+	for _, file := range jsonFiles {
 		runTest(t, file)
 	}
 }

bson/fuzz_test.go

@@ -0,0 +1,34 @@
+package bson
+
+import (
+	"testing"
+)
+
+func FuzzDecode(f *testing.F) {
+	seedBSONCorpus(f)
+
+	f.Fuzz(func(t *testing.T, data []byte) {
+		for _, typ := range []func() interface{}{
+			func() interface{} { return new(D) },
+			func() interface{} { return new([]E) },
+			func() interface{} { return new(M) },
+			func() interface{} { return new(interface{}) },
+			func() interface{} { return make(map[string]interface{}) },
+			func() interface{} { return new([]interface{}) },
+		} {
+			i := typ()
+			if err := Unmarshal(data, i); err != nil {
+				return
+			}
+
+			encoded, err := Marshal(i)
+			if err != nil {
+				t.Fatal("failed to marshal", err)
+			}
+
+			if err := Unmarshal(encoded, i); err != nil {
+				t.Fatal("failed to unmarshal", err)
+			}
+		}
+	})
+}

bson/testdata/fuzz/FuzzDecode/002ae7d43f636100116fede772a03d07726ed75c3c3b83da865fe9b718adf8ae

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\x10\x00\x00\x00\v\x00\x00\x00\b\x00\x00\v\x00\x00\x00\x00")

bson/testdata/fuzz/FuzzDecode/0de854041b0055ca1e5e6e54a7fb667ed38461db171af267665c21776f9a9ef4

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("0\\x00\\x00\\x00\\x0f\\x00000\\x8a00000000000000000000000000000000000000\n")

bson/testdata/fuzz/FuzzDecode/718592474a0a3626039f3471449b9aa374c746754d4925fcfe4ba747e7101504

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\\x80\\x00\\x00\\x00\\x03000000\\x00s\\x00\\x00\\x00\\x0300000\\x00g\\x00\\x00\\x00\\x100z\\x000000\\x11\\x00000\\x150000\\x020\\x00\\x02\\x00\\x00\\x000\\x12\\x00\\x050\\x00\\x01\\x00\\x00\\x0000\\x050\\x00\\x01\\x00\\x00\\x0000\\x040\\x00200000\\x00\\x000\\x02\\x00\\x10\\x0000000\\x110\\x0000000000\\x020\\x00\\x02\\x00\\x00\\x000\\x00\\x050\\x00\\x01\\x00\\x00\\x0000\\x050\\x00\\x01\\x00\\x00\\x0000\\x00\\x00\\x00\\x00\n")

bson/testdata/fuzz/FuzzDecode/93c43e3c1cf35c19b7618a618d128cea0ce05cef0711fdd91e403fe3b2f45628

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\\x59\\x01\\x00\\x00\\x01\\x64\\x6f\\x75\\x62\\x6c\\x65\\x00\\x9a\\x99\\x99\\x99\\x99\\x99\\xf1\\x3f\\x02\\x73\\x74\\x72\\x69\\x6e\\x67\\x00\\x06\\x00\\x00\\x00\\x68\\x65\\x6c\\x6c\\x6f\\x00\\x03\\x65\\x6d\\x62\\x65\\x64\\x64\\x65\\x64\\x00\\x4b\\x00\\x00\\x00\\x04\\x61\\x72\\x72\\x61\\x79\\x00\\x3f\\x00\\x00\\x00\\x10\\x30\\x00\\x01\\x00\\x00\\x00\\x01\\x31\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x40\\x02\\x32\\x00\\x02\\x00\\x00\\x00\\x33\\x00\\x04\\x33\\x00\\x0c\\x00\\x00\\x00\\x10\\x30\\x00\\x04\\x00\\x00\\x00\\x00\\x03\\x34\\x00\\x0d\\x00\\x00\\x00\\x03\\x35\\x00\\x05\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x05\\x62\\x69\\x6e\\x61\\x72\\x79\\x00\\x03\\x00\\x00\\x00\\x00\\x01\\x02\\x03\\x07\\x6f\\x62\\x6a\\x65\\x63\\x74\\x69\\x64\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x08\\x62\\x6f\\x6f\\x6c\\x65\\x61\\x6e\\x00\\x01\\x09\\x64\\x61\\x74\\x65\\x74\\x69\\x6d\\x65\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\x0a\\x6e\\x75\\x6c\\x6c\\x00\\x0b\\x72\\x65\\x67\\x65\\x78\\x00\\x68\\x65\\x6c\\x6c\\x6f\\x00\\x69\\x00\\x0d\\x6a\\x73\\x00\\x0e\\x00\\x00\\x00\\x66\\x75\\x6e\\x63\\x74\\x69\\x6f\\x6e\\x28\\x29\\x20\\x7b\\x7d\\x00\\x0f\\x73\\x63\\x6f\\x70\\x65\\x00\\x2c\\x00\\x00\\x00\\x0e\\x00\\x00\\x00\\x66\\x75\\x6e\\x63\\x74\\x69\\x6f\\x6e\\x28\\x29\\x20\\x7b\\x7d\\x00\\x16\\x00\\x00\\x00\\x02\\x68\\x65\\x6c\\x6c\\x6f\\x00\\x06\\x00\\x00\\x00\\x77\\x6f\\x72\\x6c\\x64\\x00\\x00\\x10\\x69\\x6e\\x74\\x33\\x32\\x00\\x20\\x00\\x00\\x00\\x11\\x74\\x69\\x6d\\x65\\x73\\x74\\x61\\x6d\\x70\\x00\\x02\\x00\\x00\\x00\\x01\\x00\\x00\\x00\\x12\\x69\\x6e\\x74\\x36\\x34\\x00\\x40\\x00\\x00\\x00\\x00\\x00\\x00\\x00\\xff\\x6d\\x69\\x6e\\x6b\\x65\\x79\\x00\\x7f\\x6d\\x61\\x78\\x6b\\x65\\x79\\x00\\x00\"\n")

bson/testdata/fuzz/FuzzDecode/c3ffbb42eb85b743ede396f00b7706e6ad0529c32689c63ca663dae37d072627

@@ -0,0 +1,2 @@
+go test fuzz v1
+[]byte("\\x05\\xf0\\xff\\x00\\x7f\n")