internal/fetch: create FetchLocalModule

Create FetchLocalModule to fetch modules from local directories without
needing a proxy instance.

Add tests for FetchLocalModule and update helper functions.

Updates golang/go#40159

Change-Id: Ie1296e6a2008bf8d7c3811864d5d948e042fcb38
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/260777
Reviewed-by: Jonathan Amsterdam <jba@google.com>
Reviewed-by: Julie Qiu <julie@golang.org>
Trust: Jonathan Amsterdam <jba@google.com>
Run-TryBot: Jonathan Amsterdam <jba@google.com>

Author: sudo-sturbia

internal/fetch/fetch_test.go

@@ -7,6 +7,7 @@ package fetch
 import (
 	"context"
 	"errors"
+	"fmt"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -18,7 +19,7 @@ import (
 	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/godoc"
-	"golang.org/x/pkgsite/internal/proxy"
+	"golang.org/x/pkgsite/internal/licenses"
 	"golang.org/x/pkgsite/internal/source"
 	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/testing/sample"
@@ -48,6 +49,7 @@ func TestFetchModule(t *testing.T) {
 		name         string
 		mod          *testModule
 		fetchVersion string
+		proxyOnly    bool
 	}{
 		{name: "basic", mod: moduleNoGoMod},
 		{name: "wasm", mod: moduleWasm},
@@ -63,54 +65,67 @@ func TestFetchModule(t *testing.T) {
 		{name: "module with type example", mod: moduleTypeExample},
 		{name: "module with method example", mod: moduleMethodExample},
 		{name: "module with nonredistributable packages", mod: moduleNonRedist},
-		{name: "stdlib module", mod: moduleStd},
-		{name: "master version of module", mod: moduleMaster, fetchVersion: "master"},
-		{name: "latest version of module", mod: moduleLatest, fetchVersion: "latest"},
+		// Proxy only as stdlib is not accounted for in local mode
+		{name: "stdlib module", mod: moduleStd, proxyOnly: true},
+		// Proxy only as version is pre specified in local mode
+		{name: "master version of module", mod: moduleMaster, fetchVersion: "master", proxyOnly: true},
+		// Proxy only as version is pre specified in local mode
+		{name: "latest version of module", mod: moduleLatest, fetchVersion: "latest", proxyOnly: true},
 	} {
-		t.Run(test.name, func(t *testing.T) {
-			ctx := context.Background()
-			ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
-			defer cancel()
-
-			modulePath := test.mod.mod.ModulePath
-			version := test.mod.mod.Version
-			fetchVersion := test.fetchVersion
-			if version == "" {
-				version = "v1.0.0"
-			}
-			if fetchVersion == "" {
-				fetchVersion = version
-			}
-			sourceClient := source.NewClient(sourceTimeout)
-			proxyClient, teardownProxy := proxy.SetupTestClient(t, []*proxy.Module{{
-				ModulePath: modulePath,
-				Version:    version,
-				Files:      test.mod.mod.Files,
-			}})
-			defer teardownProxy()
-			got := FetchModule(ctx, modulePath, fetchVersion, proxyClient, sourceClient)
-			defer got.Defer()
-			if got.Error != nil {
-				t.Fatal(got.Error)
-			}
-			d := licenseDetector(ctx, t, modulePath, got.ResolvedVersion, proxyClient)
-			fr := cleanFetchResult(test.mod.fr, d)
-			sortFetchResult(fr)
-			sortFetchResult(got)
-			opts := []cmp.Option{
-				cmpopts.IgnoreFields(internal.LegacyPackage{}, "DocumentationHTML"),
-				cmpopts.IgnoreFields(internal.Documentation{}, "HTML"),
-				cmpopts.IgnoreFields(internal.PackageVersionState{}, "Error"),
-				cmpopts.IgnoreFields(FetchResult{}, "Defer"),
-				cmp.AllowUnexported(source.Info{}),
-				cmpopts.EquateEmpty(),
-			}
-			opts = append(opts, sample.LicenseCmpOpts...)
-			if diff := cmp.Diff(fr, got, opts...); diff != "" {
-				t.Fatalf("mismatch (-want +got):\n%s", diff)
+		for _, fetcher := range []struct {
+			name  string
+			fetch func(t *testing.T, withLicenseDetector bool, ctx context.Context, mod *testModule, fetchVersion string) (*FetchResult, *licenses.Detector)
+		}{
+			{name: "proxy", fetch: proxyFetcher},
+			{name: "local", fetch: localFetcher},
+		} {
+			if test.proxyOnly && fetcher.name == "local" {
+				continue
 			}
-			validateDocumentationHTML(t, got.Module, fr.Module)
-		})
+			t.Run(fmt.Sprintf("%s:%s", fetcher.name, test.name), func(t *testing.T) {
+				ctx := context.Background()
+				ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
+				defer cancel()
+
+				got, d := fetcher.fetch(t, true, ctx, test.mod, test.fetchVersion)
+				defer got.Defer()
+				if got.Error != nil {
+					t.Fatal("fetching failed: %w", got.Error)
+				}
+
+				if fetcher.name == "proxy" {
+					test.mod.fr = cleanFetchResult(t, test.mod.fr, d)
+				}
+				fr := updateFetchResultVersions(t, test.mod.fr, fetcher.name == "local")
+				sortFetchResult(fr)
+				sortFetchResult(got)
+				opts := []cmp.Option{
+					cmpopts.IgnoreFields(internal.LegacyPackage{}, "DocumentationHTML"),
+					cmpopts.IgnoreFields(internal.Documentation{}, "HTML"),
+					cmpopts.IgnoreFields(internal.PackageVersionState{}, "Error"),
+					cmpopts.IgnoreFields(FetchResult{}, "Defer"),
+					cmp.AllowUnexported(source.Info{}),
+					cmpopts.EquateEmpty(),
+				}
+				if fetcher.name == "local" {
+					opts = append(opts,
+						[]cmp.Option{
+							// Pre specified for all modules
+							cmpopts.IgnoreFields(internal.Module{}, "SourceInfo"),
+							cmpopts.IgnoreFields(internal.Module{}, "Version"),
+							cmpopts.IgnoreFields(FetchResult{}, "RequestedVersion"),
+							cmpopts.IgnoreFields(FetchResult{}, "ResolvedVersion"),
+							cmpopts.IgnoreFields(internal.Module{}, "CommitTime"),
+						}...)
+				}
+
+				opts = append(opts, sample.LicenseCmpOpts...)
+				if diff := cmp.Diff(fr, got, opts...); diff != "" {
+					t.Fatalf("mismatch (-want +got):\n%s", diff)
+				}
+				validateDocumentationHTML(t, got.Module, fr.Module)
+			})
+		}
 	}
 }
 func TestFetchModule_Errors(t *testing.T) {
@@ -125,29 +140,25 @@ func TestFetchModule_Errors(t *testing.T) {
 		{name: "alternative", mod: moduleAlternative, wantErr: derrors.AlternativeModule, wantGoModPath: "canonical"},
 		{name: "empty module", mod: moduleEmpty, wantErr: derrors.BadModule},
 	} {
-		t.Run(test.name, func(t *testing.T) {
-			modulePath := test.mod.mod.ModulePath
-			version := test.mod.mod.Version
-			if version == "" {
-				version = "v1.0.0"
-			}
-			proxyClient, teardownProxy := proxy.SetupTestClient(t, []*proxy.Module{{
-				ModulePath: modulePath,
-				Files:      test.mod.mod.Files,
-			}})
-			defer teardownProxy()
-
-			sourceClient := source.NewClient(sourceTimeout)
-			got := FetchModule(ctx, modulePath, "v1.0.0", proxyClient, sourceClient)
-			defer got.Defer()
-			if !errors.Is(got.Error, test.wantErr) {
-				t.Fatalf("FetchModule(ctx, %q, v1.0.0, proxyClient, sourceClient): %v; wantErr = %v)", modulePath, got.Error, test.wantErr)
-			}
-			if test.wantGoModPath != "" {
-				if got == nil || got.GoModPath != test.wantGoModPath {
-					t.Errorf("got %+v, wanted GoModPath %q", got, test.wantGoModPath)
+		for _, fetcher := range []struct {
+			name  string
+			fetch func(t *testing.T, withLicenseDetector bool, ctx context.Context, mod *testModule, fetchVersion string) (*FetchResult, *licenses.Detector)
+		}{
+			{name: "proxy", fetch: proxyFetcher},
+			{name: "local", fetch: localFetcher},
+		} {
+			t.Run(fmt.Sprintf("%s:%s", fetcher.name, test.name), func(t *testing.T) {
+				got, _ := fetcher.fetch(t, false, ctx, test.mod, "")
+				defer got.Defer()
+				if !errors.Is(got.Error, test.wantErr) {
+					t.Fatalf("got error = %v; wantErr = %v)", got.Error, test.wantErr)
 				}
-			}
-		})
+				if test.wantGoModPath != "" {
+					if got == nil || got.GoModPath != test.wantGoModPath {
+						t.Errorf("got %+v, wanted GoModPath %q", got, test.wantGoModPath)
+					}
+				}
+			})
+		}
 	}
 }

internal/fetch/fetchlocal.go

@@ -0,0 +1,165 @@
+// Copyright 2020 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package fetch
+
+import (
+	"archive/zip"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"golang.org/x/mod/modfile"
+	"golang.org/x/pkgsite/internal/derrors"
+	"golang.org/x/pkgsite/internal/log"
+	"golang.org/x/pkgsite/internal/source"
+)
+
+// Version and commit time are pre specified when fetching a local module, as these
+// fields are normally obtained from a proxy.
+var (
+	LocalVersion    = "latest"
+	LocalCommitTime = time.Time{}
+)
+
+// FetchLocalModule fetches a module from a local directory and process its contents
+// to return an internal.Module and other related information. modulePath is not necessary
+// if the module has a go.mod file, but if both exist, then they must match.
+// FetchResult.Error should be checked to verify that the fetch succeeded. Even if the
+// error is non-nil the result may contain useful data.
+func FetchLocalModule(ctx context.Context, modulePath, localPath string, sourceClient *source.Client) *FetchResult {
+	fr := &FetchResult{
+		ModulePath:       modulePath,
+		RequestedVersion: LocalVersion,
+		ResolvedVersion:  LocalVersion,
+		Defer:            func() {},
+	}
+
+	var fi *FetchInfo
+	defer func() {
+		if fr.Error != nil {
+			derrors.Wrap(&fr.Error, "FetchLocalModule(%q, %q)", modulePath, localPath)
+			fr.Status = derrors.ToStatus(fr.Error)
+		}
+		if fr.Status == 0 {
+			fr.Status = http.StatusOK
+		}
+		if fi != nil {
+			finishFetchInfo(fi, fr.Status, fr.Error)
+		}
+		log.Debugf(ctx, "memory after fetch of %s: %dM", fr.ModulePath, allocMeg())
+
+	}()
+
+	info, err := os.Stat(localPath)
+	if err != nil {
+		fr.Error = fmt.Errorf("%s: %w", err.Error(), derrors.NotFound)
+		return fr
+	}
+
+	if !info.IsDir() {
+		fr.Error = fmt.Errorf("%s not a directory: %w", localPath, derrors.NotFound)
+		return fr
+	}
+
+	fi = &FetchInfo{
+		ModulePath: fr.ModulePath,
+		Version:    fr.ResolvedVersion,
+		Start:      time.Now(),
+	}
+	startFetchInfo(fi)
+
+	// Options for module path are either the modulePath parameter or go.mod file.
+	// Accepted cases:
+	//   - Both are given and are the same.
+	//   - Only one is given. Note that: if modulePath is given and there's no go.mod
+	//     file, then the package is assumed to be using GOPATH.
+	// Errors:
+	//   - Both are given and are different.
+	//   - Neither is given.
+	if goModBytes, err := ioutil.ReadFile(filepath.Join(localPath, "go.mod")); err != nil {
+		fr.GoModPath = modulePath
+	} else {
+		fr.GoModPath = modfile.ModulePath(goModBytes)
+		if fr.GoModPath != modulePath && modulePath != "" {
+			fr.Error = fmt.Errorf("module path=%s, go.mod path=%s: %w", modulePath, fr.GoModPath, derrors.AlternativeModule)
+			return fr
+		}
+	}
+
+	if fr.GoModPath == "" {
+		fr.Error = fmt.Errorf("no module path: %w", derrors.BadModule)
+		return fr
+	}
+	fr.ModulePath = fr.GoModPath
+
+	zipReader, err := createZipReader(localPath, fr.GoModPath, LocalVersion)
+	if err != nil {
+		fr.Error = fmt.Errorf("couldn't create a zip: %s, %w", err.Error(), derrors.BadModule)
+		return fr
+	}
+
+	mod, pvs, err := processZipFile(ctx, fr.GoModPath, LocalVersion, LocalCommitTime, zipReader, sourceClient)
+	if err != nil {
+		fr.Error = err
+		return fr
+	}
+
+	fr.Module = mod
+	fr.PackageVersionStates = pvs
+	fr.Module.SourceInfo = nil // version is not known, so even if info is found it most likely is wrong.
+	for _, state := range fr.PackageVersionStates {
+		if state.Status != http.StatusOK {
+			fr.Status = derrors.ToStatus(derrors.HasIncompletePackages)
+		}
+	}
+	return fr
+}
+
+// createZipReader creates a zip file from a directory given a local path and
+// returns a zip.Reader to be passed to processZipFile. The purpose of the
+// function is to transform a local go module into a zip file to be processed by
+// existing functions.
+func createZipReader(localPath, modulePath, version string) (*zip.Reader, error) {
+	buf := new(bytes.Buffer)
+	w := zip.NewWriter(buf)
+	err := filepath.Walk(localPath, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+
+		readFrom, err := os.Open(path)
+		if err != nil {
+			return err
+		}
+		defer readFrom.Close()
+
+		writeTo, err := w.Create(filepath.Join(moduleVersionDir(modulePath, version), strings.TrimPrefix(path, localPath)))
+		if err != nil {
+			return err
+		}
+
+		_, err = io.Copy(writeTo, readFrom)
+		return err
+	})
+	if err != nil {
+		return nil, err
+	}
+	if err := w.Close(); err != nil {
+		return nil, err
+	}
+
+	reader := bytes.NewReader(buf.Bytes())
+	return zip.NewReader(reader, reader.Size())
+}