json: Fix panic when parsing malformed JSON

When scanning JSON, upon encountering an invalid token, we immediately
return. Previously this return happened without inserting an EOF token.
Since other functions assume that a token sequence always ends in EOF,
this could cause a panic.

This commit adds a synthetic EOF token after the invalid token before
returning. While this does not match the real end-of-file of the source
JSON, it is marking the end of the scanned bytes, so it seems reasonable.

Fixes #339

json/public_test.go

@@ -1,6 +1,7 @@
 package json
 
 import (
+	"strings"
 	"testing"
 
 	"github.com/hashicorp/hcl/v2"
@@ -95,3 +96,19 @@ func TestParseTemplateUnwrap(t *testing.T) {
 		t.Errorf("wrong result %#v; want %#v", val, cty.True)
 	}
 }
+
+func TestParse_malformed(t *testing.T) {
+	src := `{
+  "http_proxy_url: "http://xxxxxx",
+}`
+	file, diags := Parse([]byte(src), "")
+	if got, want := len(diags), 2; got != want {
+		t.Errorf("got %d diagnostics; want %d", got, want)
+	}
+	if err, want := diags.Error(), `Missing property value colon`; !strings.Contains(err, want) {
+		t.Errorf("diags are %q, but should contain %q", err, want)
+	}
+	if file == nil {
+		t.Errorf("got nil File; want actual file")
+	}
+}

json/scanner.go

@@ -107,6 +107,15 @@ func scan(buf []byte, start pos) []token {
 			})
 			// If we've encountered an invalid then we might as well stop
 			// scanning since the parser won't proceed beyond this point.
+			// We insert a synthetic EOF marker here to match the expectations
+			// of consumers of this data structure.
+			p.Pos.Column++
+			p.Pos.Byte++
+			tokens = append(tokens, token{
+				Type:  tokenEOF,
+				Bytes: nil,
+				Range: posRange(p, p),
+			})
 			return tokens
 		}
 	}

json/scanner_test.go

@@ -829,6 +829,21 @@ func TestScan(t *testing.T) {
 						},
 					},
 				},
+				{
+					Type: tokenEOF,
+					Range: hcl.Range{
+						Start: hcl.Pos{
+							Byte:   1,
+							Line:   1,
+							Column: 2,
+						},
+						End: hcl.Pos{
+							Byte:   1,
+							Line:   1,
+							Column: 2,
+						},
+					},
+				},
 			},
 		},
 	}