cmd/pkgsite: add multi-module support, invalidation, and search

This change adds a new goPackagesModuleGetter, which uses
x/tools/go/packages to query package information for modules requested
by the pkgsite command.

Additionally, add new extension interfaces that allow getters to add
support for search and content invalidation. The go/packages getter uses
these extensions to implement search (via a simple fuzzy-matching
algorithm copied from x/tools), and invalidation via statting package
files.

Along the way, refactor slightly for testing ergonomics.

Updates golang/go#40371
Updates golang/go#50229
Fixes golang/go#54479

Change-Id: Iea91a4d6327707733cbbc4f74a9d93052f33e295
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/474295
TryBot-Result: kokoro <noreply+kokoro@google.com>
Run-TryBot: Robert Findley <rfindley@google.com>
Reviewed-by: Jamal Carvalho <jamal@golang.org>

Author: findleyr

cmd/pkgsite/main.go

@@ -16,10 +16,10 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"golang.org/x/net/html"
-	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/proxy"
 	"golang.org/x/pkgsite/internal/proxy/proxytest"
 	"golang.org/x/pkgsite/internal/testing/htmlcheck"
+	"golang.org/x/pkgsite/internal/testing/testhelper"
 )
 
 var (
@@ -51,48 +51,45 @@ func TestBuildGetters(t *testing.T) {
 	prox, teardown := proxytest.SetupTestClient(t, testModules)
 	defer teardown()
 
-	localGetter := "Dir(example.com/testmod, " + abs(localModule) + ")"
+	localGetter := "Dir(" + abs(localModule) + ")"
 	cacheGetter := "FSProxy(" + abs(cacheDir) + ")"
 	for _, test := range []struct {
 		name     string
-		paths    []string
-		cmods    []internal.Modver
+		dirs     []string
 		cacheDir string
-		prox     *proxy.Client
+		proxy    *proxy.Client
 		want     []string
 	}{
 		{
-			name:  "local only",
-			paths: []string{localModule},
-			want:  []string{localGetter},
+			name: "local only",
+			dirs: []string{localModule},
+			want: []string{localGetter},
 		},
 		{
 			name:     "cache",
 			cacheDir: cacheDir,
 			want:     []string{cacheGetter},
 		},
 		{
-			name: "proxy",
-			prox: prox,
-			want: []string{"Proxy"},
+			name:  "proxy",
+			proxy: prox,
+			want:  []string{"Proxy"},
 		},
 		{
 			name:     "all three",
-			paths:    []string{localModule},
+			dirs:     []string{localModule},
 			cacheDir: cacheDir,
-			prox:     prox,
+			proxy:    prox,
 			want:     []string{localGetter, cacheGetter, "Proxy"},
 		},
-		{
-			name:     "list",
-			paths:    []string{localModule},
-			cacheDir: cacheDir,
-			cmods:    []internal.Modver{{Path: "foo", Version: "v1.2.3"}},
-			want:     []string{localGetter, "FSProxy(" + abs(cacheDir) + ", foo@v1.2.3)"},
-		},
 	} {
 		t.Run(test.name, func(t *testing.T) {
-			getters, err := buildGetters(ctx, test.paths, false, test.cacheDir, test.cmods, test.prox)
+			getters, err := buildGetters(ctx, getterConfig{
+				dirs:        test.dirs,
+				pattern:     "./...",
+				modCacheDir: test.cacheDir,
+				proxy:       test.proxy,
+			})
 			if err != nil {
 				t.Fatal(err)
 			}
@@ -119,56 +116,91 @@ func TestServer(t *testing.T) {
 		return a
 	}
 
-	localModule := repoPath("internal/fetch/testdata/has_go_mod")
+	localModule, _ := testhelper.WriteTxtarToTempDir(t, `
+-- go.mod --
+module example.com/testmod
+-- a.go --
+package a
+`)
 	cacheDir := repoPath("internal/fetch/testdata/modcache")
 	testModules := proxytest.LoadTestModules(repoPath("internal/proxy/testdata"))
 	prox, teardown := proxytest.SetupTestClient(t, testModules)
 	defer teardown()
 
-	getters, err := buildGetters(context.Background(), []string{localModule}, false, cacheDir, nil, prox)
-	if err != nil {
-		t.Fatal(err)
-	}
-	server, err := newServer(getters, prox)
-	if err != nil {
-		t.Fatal(err)
+	defaultConfig := serverConfig{
+		paths:         []string{localModule},
+		gopathMode:    false,
+		useListedMods: true,
+		useCache:      true,
+		cacheDir:      cacheDir,
+		proxy:         prox,
 	}
-	mux := http.NewServeMux()
-	server.Install(mux.Handle, nil, nil)
 
 	modcacheChecker := in("",
 		in(".Documentation", hasText("var V = 1")),
 		sourceLinks(path.Join(abs(cacheDir), "modcache.com@v1.0.0"), "a.go"))
 
+	ctx := context.Background()
 	for _, test := range []struct {
 		name string
+		cfg  serverConfig
 		url  string
 		want htmlcheck.Checker
 	}{
 		{
 			"local",
+			defaultConfig,
 			"example.com/testmod",
 			in("",
 				in(".Documentation", hasText("There is no documentation for this package.")),
 				sourceLinks(path.Join(abs(localModule), "example.com/testmod"), "a.go")),
 		},
 		{
 			"modcache",
+			defaultConfig,
 			"modcache.com@v1.0.0",
 			modcacheChecker,
 		},
 		{
 			"modcache latest",
+			defaultConfig,
 			"modcache.com",
 			modcacheChecker,
 		},
 		{
 			"proxy",
+			defaultConfig,
 			"example.com/single/pkg",
 			hasText("G is new in v1.1.0"),
 		},
+		{
+			"search",
+			defaultConfig,
+			"search?q=a",
+			in(".SearchResults",
+				hasText("example.com/testmod"),
+			),
+		},
+		{
+			"search",
+			defaultConfig,
+			"search?q=zzz",
+			in(".SearchResults",
+				hasText("no matches"),
+			),
+		},
+		// TODO(rfindley): add more tests, including a test for the standard
+		// library once it doesn't go through the stdlib package.
+		// See also golang/go#58923.
 	} {
 		t.Run(test.name, func(t *testing.T) {
+			server, err := buildServer(ctx, test.cfg)
+			if err != nil {
+				t.Fatal(err)
+			}
+			mux := http.NewServeMux()
+			server.Install(mux.Handle, nil, nil)
+
 			w := httptest.NewRecorder()
 			mux.ServeHTTP(w, httptest.NewRequest("GET", "/"+test.url, nil))
 			if w.Code != http.StatusOK {
@@ -203,36 +235,3 @@ func TestCollectPaths(t *testing.T) {
 		t.Errorf("got %v, want %v", got, want)
 	}
 }
-
-func TestListModsForPaths(t *testing.T) {
-	listModules = func(string) ([]listedMod, error) {
-		return []listedMod{
-			{
-				internal.Modver{Path: "m1", Version: "v1.2.3"},
-				"/dir/cache/download/m1/@v/v1.2.3.mod",
-				false,
-			},
-			{
-				internal.Modver{Path: "m2", Version: "v1.0.0"},
-				"/repos/m2/go.mod",
-				false,
-			},
-			{
-				internal.Modver{Path: "indir", Version: "v2.3.4"},
-				"",
-				true,
-			},
-		}, nil
-	}
-	defer func() { listModules = _listModules }()
-
-	gotPaths, gotCacheMods, err := listModsForPaths([]string{"m1"}, "/dir")
-	if err != nil {
-		t.Fatal(err)
-	}
-	wantPaths := []string{"/repos/m2"}
-	wantCacheMods := []internal.Modver{{Path: "m1", Version: "v1.2.3"}}
-	if !cmp.Equal(gotPaths, wantPaths) || !cmp.Equal(gotCacheMods, wantCacheMods) {
-		t.Errorf("got\n%v, %v\nwant\n%v, %v", gotPaths, gotCacheMods, wantPaths, wantCacheMods)
-	}
-}

cmd/pkgsite/main_test.go

@@ -4,7 +4,73 @@
 
 package internal
 
-import "context"
+import (
+	"context"
+	"time"
+)
+
+// SearchOptions provide information used by db.Search.
+type SearchOptions struct {
+	// Maximum number of results to return (page size).
+	MaxResults int
+
+	// Offset for DB query.
+	Offset int
+
+	// Maximum number to use for total result count.
+	MaxResultCount int
+
+	// If true, perform a symbol search.
+	SearchSymbols bool
+
+	// SymbolFilter is the word in a search query with a # prefix.
+	SymbolFilter string
+}
+
+// SearchResult represents a single search result from SearchDocuments.
+type SearchResult struct {
+	Name        string
+	PackagePath string
+	ModulePath  string
+	Version     string
+	Synopsis    string
+	Licenses    []string
+
+	CommitTime time.Time
+
+	// Score is used to sort items in an array of SearchResult.
+	Score float64
+
+	// NumImportedBy is the number of packages that import PackagePath.
+	NumImportedBy uint64
+
+	// SameModule is a list of SearchResults from the same module as this one,
+	// with lower scores.
+	SameModule []*SearchResult
+
+	// OtherMajor is a map from module paths with the same series path but at
+	// different major versions of this module, to major version.
+	// The major version for a non-vN module path (either 0 or 1) is computed
+	// based on the version in search documents.
+	OtherMajor map[string]int
+
+	// NumResults is the total number of packages that were returned for this
+	// search.
+	NumResults uint64
+
+	// Symbol information returned by a search request.
+	// Only populated for symbol search mode.
+	SymbolName     string
+	SymbolKind     SymbolKind
+	SymbolSynopsis string
+	SymbolGOOS     string
+	SymbolGOARCH   string
+
+	// Offset is the 0-based number of this row in the DB query results, which
+	// is the value to use in a SQL OFFSET clause to have this row be the first
+	// one returned.
+	Offset int
+}
 
 // DataSource is the interface used by the frontend to interact with module data.
 type DataSource interface {
@@ -22,10 +88,14 @@ type DataSource interface {
 	GetUnitMeta(ctx context.Context, path, requestedModulePath, requestedVersion string) (_ *UnitMeta, err error)
 	// GetModuleReadme gets the readme for the module.
 	GetModuleReadme(ctx context.Context, modulePath, resolvedVersion string) (*Readme, error)
-
 	// GetLatestInfo gets information about the latest versions of a unit and module.
 	// See LatestInfo for documentation.
 	GetLatestInfo(ctx context.Context, unitPath, modulePath string, latestUnitMeta *UnitMeta) (LatestInfo, error)
+
+	// SupportsSearch reports whether this data source supports search.
+	SupportsSearch() bool
+	// Search searches for packages matching the given query.
+	Search(ctx context.Context, q string, opts SearchOptions) (_ []*SearchResult, err error)
 }
 
 // LatestInfo holds information about the latest versions and paths.