Catch all for Golang parser

[incognick]

First - huge thanks to Alec Thomas for this library. I've used it in the past for a custom SQL grammar and this is just amazing. Thank you!

Question

I'm trying to make a high level Go parser. I don't need to parse all the expressions. I just want the major blocks:

• Package name
• Imports
• Functions
  • Name
  • Args
  • Return
  • Body (full string capture)
• Types

Basically, I want to capture the function signature and then the full body should just be a string capture - all new lines, whitespaces, etc.

I've tried a few methods such as the Stateful lexer but didn't get very far. This example is the furthest I got but is unquoting strings. I have the parser set to unquote strings due to the import statement.

Does anyone have suggestions I can try? It seems like I need a new parser with different rules for just the function body such as capture everything (.*) including other curly brackets but break out once the function body is closed.

I appreciate anyone's help on this - exhausted a lot of stack overflow and GPT questions :)

package parser

import (
	"fmt"

	"github.com/alecthomas/participle/v2"
	"github.com/alecthomas/participle/v2/lexer"
)

type Parser struct {
}

func New() *Parser {
	return &Parser{}
}

var (
	golex = lexer.MustSimple([]lexer.SimpleRule{
		{Name: "Ident", Pattern: `[a-zA-Z_][a-zA-Z0-9_]*`},
		{Name: "String", Pattern: `"(?:\\.|[^"])*"`},
		{Name: "Number", Pattern: `[-+]?[0-9]*\.?[0-9]+`},
		{Name: "Punct", Pattern: `[-[!@#$%^&*(){}+_=\|:;"'<,>.?/]|]`},
		{Name: "EOL", Pattern: `\n`},
		{Name: "Whitespace", Pattern: `[ \t]+`},
	})

	gop = participle.MustBuild[GoFile](
		participle.Lexer(golex),
		participle.Unquote("String"),
		participle.UseLookahead(participle.MaxLookahead),
		participle.Elide("Whitespace", "EOL"),
	)
)

type GoFile struct {
	PackageName string           `parser:"'package' @Ident"`
	ImportStmt  *ImportStatement `parser:"('import' '('? @@ ')'?)?"`
	Decl        []Declaration    `parser:"@@*"`

	// Types       []*TypeDef `parser:"(@@)*"`
	// Functions   []*FuncDef `parser:"(@@)*"`
}

type Declaration struct {
	FuncDef *FuncDef `parser:"'func' @@"`
	// TypeDef *TypeDef `parser:"| 'type' @@"`
}

type FuncDef struct {
	Name      string         `parser:"@Ident '('"`
	Args      []FuncArg      `parser:"(@@ (',' @@)* )? ')'"`
	ReturnDef *FuncReturnDef `parser:"@@?"`
	Body      *FuncBody      `parser:"'{' @@ '}'"`
}

type FuncBody struct {
	Body string `parser:"@@"`
}

func (f *FuncBody) Parse(lex *lexer.PeekingLexer) error {
	count := 0

	start := lex.RawCursor()

	for {
		switch lex.Peek().Value {
		case "{":
			count++
		case "}":
			if count == 0 {
				end := lex.RawCursor()
				tokens := lex.Range(start, end)
				for _, tok := range tokens {
					f.Body += tok.Value
				}
				return nil
			}
			count--
		}
		lex.Next()
	}
}

type FuncArg struct {
	Name string `parser:"@Ident"`
	Type string `parser:"@Ident?"`
}

type FuncReturnDef struct {
	Type string `parser:"@Ident"`
}

// type TypeDef struct {
// 	Name string `parser:"@Ident"`
// }

type ImportStatement struct {
	Imports []ImportDef `parser:"@@+"`
}

type ImportDef struct {
	Name    string `parser:"@Ident?"`
	Package string `parser:"@String"`
}

func (p *Parser) Parse(code string) (*GoFile, error) {
	ast, err := gop.ParseString("", code)
	if err != nil {
		return nil, err
	}
	return ast, nil
}

Test code:

package parser

import (
	"testing"

	"github.com/stretchr/testify/require"
)

const focus = true

func TestParser(t *testing.T) {
	tests := []struct {
		name        string
		src         string
		wantPackage string
		wantImports *ImportStatement
		wantDecl    []Declaration
		wantErr     bool
		focus       bool
	}{
		{
			name:        "just package",
			src:         `package main`,
			wantPackage: "main",
		},
		{
			name: "single import",
			src: `package main
			import "fmt"`,
			wantPackage: "main",
			wantImports: &ImportStatement{
				Imports: []ImportDef{{Package: "fmt"}},
			},
		},
		{
			name: "multi import",
			src: `package main
			import (
				"fmt"
				"io"
			)`,
			wantPackage: "main",
			wantImports: &ImportStatement{
				Imports: []ImportDef{{Package: "fmt"}, {Package: "io"}},
			},
		},
		{
			name: "named imports",
			src: `package main
		import (
			"fmt"
			"io"
			_ "database/driver"

			myconfig "github.com/mine/pkg/config"
		)`,
			wantPackage: "main",
			wantImports: &ImportStatement{
				Imports: []ImportDef{
					{Package: "fmt"},
					{Package: "io"},
					{Name: "_", Package: "database/driver"},
					{Name: "myconfig", Package: "github.com/mine/pkg/config"},
				},
			},
		},
		{
			name: "empty function",
			src: `package main
		func Hello()  {  }`,
			wantPackage: "main",
			wantDecl: []Declaration{
				{
					FuncDef: &FuncDef{
						Name: "Hello",
						Body: &FuncBody{},
					},
				},
			},
		},
		{

			name: "function with body",
			src: `package main
		func Hello()  { hello world not real go code }`,
			wantPackage: "main",
			wantDecl: []Declaration{
				{
					FuncDef: &FuncDef{
						Name: "Hello",
						Body: &FuncBody{Body: " hello world not real go code"},
					},
				},
			},
		},
		{
			focus: true,
			name:  "function with multi line body",
			src: `package main
func Hello()  {
	var a = 1
	var b = 2
	fmt.Println("hello world")
	return a + b
}
		`,
			wantPackage: "main",
			wantDecl: []Declaration{
				{
					FuncDef: &FuncDef{
						Name: "Hello",
						Body: &FuncBody{Body: `
	var a = 1
	var b = 2
	fmt.Println("hello world")
	return a + b`},
					},
				},
			},
		},
		// 	{
		// 		name: "function with multiple returns",
		// 		src: `package main
		// func Divide(x, y float64) (float64, error) { return x/y, nil }`,
		// 		funcName:      "Divide",
		// 		wantSignature: "func Divide(x, y float64) (float64, error)",
		// 	},
		// 	{
		// 		name: "missing function",
		// 		src: `package main
		// func Other() {}`,
		// 		funcName: "Missing",
		// 		wantErr:  true,
		// 	},
		// 	{
		// 		name: "function with named returns",
		// 		src: `package main
		// func Split(s string) (first, second string) { return "", "" }`,
		// 		funcName:      "Split",
		// 		wantSignature: "func Split(s string) (first string, second string)",
		// 	},
		// 	{
		// 		name: "function with complex params",
		// 		src: `package main
		// func Process(ch chan<- string, fn func(int) bool) error { return nil }`,
		// 		funcName:      "Process",
		// 		wantSignature: "func Process(ch chan<- string, fn func(int) bool) error",
		// 	},
	}

	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if focus && !tt.focus {
				t.Skip()
			}

			p := New()
			gf, err := p.Parse(tt.src)
			require.NoError(t, err)
			require.Equal(t, tt.wantPackage, gf.PackageName)

			if tt.wantImports == nil {
				require.Nil(t, gf.ImportStmt)
			} else {
				require.NotNil(t, gf.ImportStmt)
				require.Equal(t, tt.wantImports.Imports, gf.ImportStmt.Imports)
			}

			// require.Len(t, gf.Imports.Imports, len(tt.wantImports))
			// for i, wantImport := range tt.wantImports {
			// 	assert.Equal(t, wantImport, gf.Imports.Imports[i])
			// }

			require.Len(t, gf.Decl, len(tt.wantDecl))
			for i, wantDecl := range tt.wantDecl {
				if wantDecl.FuncDef != nil {
					require.NotNil(t, gf.Decl[i].FuncDef)
					require.Equal(t, wantDecl.FuncDef.Name, gf.Decl[i].FuncDef.Name)
					require.Equal(t, wantDecl.FuncDef.Body.Body, gf.Decl[i].FuncDef.Body.Body)
				} else {
					require.Fail(t, "unexpected declaration type")
				}
			}

			// assert.DeepEqual(t, tt.wantDecl, gf.Decl)

			// for i, wantFunc := range tt.wantFuncs {
			// 	assert.DeepEqual(t, wantFunc, gf.Functions[i])
			// }
		})
	}

}

[alecthomas]

I'm out and about, but you should almost certainly use the default Go lexer.

For recursive constructs like {} I would probably capture them into some throwaway field. Use ! (IIRC) to match until characters you care about.